make backup.js deployable and add cron timers to it
This commit is contained in:
parent
759bffbf1f
commit
e6c6b45a6b
|
@ -15,16 +15,25 @@ import fs from 'fs';
|
|||
import redis from 'redis';
|
||||
import bluebird from 'bluebird';
|
||||
|
||||
import process from 'process';
|
||||
import { spawn } from 'child_process';
|
||||
|
||||
import { DailyCron } from './utils/cron';
|
||||
import {
|
||||
updateBackupRedis,
|
||||
createPngBackup,
|
||||
incrementialBackupRedis,
|
||||
} from './core/tilesBackup';
|
||||
import canvases from './canvases.json';
|
||||
|
||||
/*
|
||||
* use low cpu priority
|
||||
*/
|
||||
import process from 'process';
|
||||
import { spawn } from 'child_process';
|
||||
const priority = 15;
|
||||
const proc= spawn("renice", [priority, process.pid]);
|
||||
proc.on('exit', function (code) {
|
||||
const proc = spawn('renice', [priority, process.pid]);
|
||||
proc.on('exit', (code) => {
|
||||
if (code !== 0) {
|
||||
console.log("renice failed with code - " +code);
|
||||
console.log(`renice failed with code ${code}`);
|
||||
}
|
||||
console.log('Useing low cpu priority');
|
||||
});
|
||||
|
@ -34,13 +43,6 @@ proc.on('exit', function (code) {
|
|||
bluebird.promisifyAll(redis.RedisClient.prototype);
|
||||
bluebird.promisifyAll(redis.Multi.prototype);
|
||||
|
||||
import canvases from './canvases.json';
|
||||
import {
|
||||
updateBackupRedis,
|
||||
createPngBackup,
|
||||
incrementialBackupRedis,
|
||||
} from './core/tilesBackup';
|
||||
|
||||
const {
|
||||
CANVAS_REDIS_URL,
|
||||
BACKUP_REDIS_URL,
|
||||
|
@ -64,23 +66,60 @@ backupRedis.on('error', () => {
|
|||
});
|
||||
|
||||
|
||||
function dailyBackup() {
|
||||
function getDateFolder() {
|
||||
if (!fs.existsSync(BACKUP_DIR)) {
|
||||
throw new Error(`Backup directory ${backupDir} does not exist!`);
|
||||
throw new Error(`Backup directory ${BACKUP_DIR} does not exist!`);
|
||||
}
|
||||
const date = new Date();
|
||||
// eslint-disable-next-line max-len
|
||||
const dayDir = `${date.getFullYear()}${date.getMonth() + 1}${date.getDate()}`;
|
||||
const backupDir = `${BACKUP_DIR}/${dayDir}`;
|
||||
return backupDir;
|
||||
}
|
||||
|
||||
function dailyBackup() {
|
||||
const backupDir = getDateFolder();
|
||||
if (!fs.existsSync(backupDir)) {
|
||||
fs.mkdirSync(backupDir);
|
||||
}
|
||||
|
||||
backupRedis.flushall('ASYNC', async () => {
|
||||
const date = new Date();
|
||||
const dayDir = `${date.getFullYear()}${date.getMonth() + 1}${date.getDate()}`;
|
||||
const backupDir = `${BACKUP_DIR}/${dayDir}`;
|
||||
if (!fs.existsSync(backupDir)) {
|
||||
fs.mkdirSync(backupDir);
|
||||
}
|
||||
await updateBackupRedis(canvasRedis, backupRedis, canvases);
|
||||
await createPngBackup(backupRedis, canvases, backupDir);
|
||||
await incrementialBackupRedis(canvasRedis, backupRedis, canvases, backupDir);
|
||||
console.log(`Daily backup ${dayDir} done`);
|
||||
console.log('Daily backup done');
|
||||
});
|
||||
}
|
||||
|
||||
dailyBackup();
|
||||
function incrementialBackup() {
|
||||
const backupDir = getDateFolder();
|
||||
if (!fs.existsSync(backupDir)) {
|
||||
fs.mkdirSync(backupDir);
|
||||
}
|
||||
incrementialBackupRedis(
|
||||
canvasRedis,
|
||||
backupRedis,
|
||||
canvases,
|
||||
backupDir,
|
||||
);
|
||||
}
|
||||
|
||||
async function startCronJobs() {
|
||||
if (!fs.existsSync(BACKUP_DIR)) {
|
||||
throw new Error(`Backup directory ${BACKUP_DIR} does not exist!`);
|
||||
}
|
||||
const backupDir = getDateFolder();
|
||||
if (!fs.existsSync(backupDir)) {
|
||||
await dailyBackup();
|
||||
}
|
||||
|
||||
console.log(
|
||||
'Creating one full PNG backup per day and incremential backups every hour.',
|
||||
);
|
||||
DailyCron.hook(dailyBackup);
|
||||
setInterval(incrementialBackup, 15 * 60 * 1000);
|
||||
}
|
||||
|
||||
startCronJobs();
|
||||
|
|
|
@ -72,8 +72,8 @@ export async function incrementialBackupRedis(
|
|||
if (!fs.existsSync(canvasBackupDir)) {
|
||||
fs.mkdirSync(canvasBackupDir);
|
||||
}
|
||||
const hourOfDay = new Date().getHours();
|
||||
const canvasTileBackupDir = `${canvasBackupDir}/${hourOfDay}`;
|
||||
const date = new Date();
|
||||
const canvasTileBackupDir = `${canvasBackupDir}/${date.getHours()}${date.getMinutes()}`;
|
||||
if (!fs.existsSync(canvasTileBackupDir)) {
|
||||
fs.mkdirSync(canvasTileBackupDir);
|
||||
}
|
||||
|
|
|
@ -9,27 +9,27 @@ import { HOUR } from '../core/constants';
|
|||
import logger from '../core/logger';
|
||||
|
||||
class Cron {
|
||||
last_run: number;
|
||||
lastRun: number;
|
||||
interval: number;
|
||||
functions: Array;
|
||||
timeout;
|
||||
|
||||
// interval = how many hours between runs
|
||||
// last_run = when this cron job was last run
|
||||
constructor(interval: number, last_run: number = 0) {
|
||||
this.check_for_execution = this.check_for_execution.bind(this);
|
||||
// lastRun = when this cron job was last run
|
||||
constructor(interval: number, lastRun: number = 0) {
|
||||
this.checkForExecution = this.checkForExecution.bind(this);
|
||||
this.interval = interval;
|
||||
this.last_run = last_run;
|
||||
this.lastRun = lastRun;
|
||||
this.functions = [];
|
||||
|
||||
this.timeout = setInterval(this.check_for_execution, HOUR);
|
||||
this.timeout = setInterval(this.checkForExecution, HOUR);
|
||||
}
|
||||
|
||||
check_for_execution() {
|
||||
const cur_time = Date.now();
|
||||
if (cur_time > this.last_run + this.interval * HOUR) {
|
||||
checkForExecution() {
|
||||
const curTime = Date.now();
|
||||
if (curTime > this.lastRun + this.interval * HOUR) {
|
||||
logger.info(`Run cron events for interval: ${this.interval}h`);
|
||||
this.last_run = cur_time;
|
||||
this.lastRun = curTime;
|
||||
this.functions.forEach(async (item) => {
|
||||
item();
|
||||
});
|
||||
|
@ -42,19 +42,19 @@ class Cron {
|
|||
}
|
||||
|
||||
|
||||
function initialize_daily_cron() {
|
||||
function initializeDailyCron() {
|
||||
const now = new Date();
|
||||
// make it first run at midnight
|
||||
const last_run = now.getTime() - now.getHours() * HOUR;
|
||||
const cron = new Cron(24, last_run);
|
||||
const lastRun = now.getTime() - now.getHours() * HOUR;
|
||||
const cron = new Cron(24, lastRun);
|
||||
return cron;
|
||||
}
|
||||
|
||||
function initialize_hourly_cron() {
|
||||
function initializeHourlyCron() {
|
||||
const cron = new Cron(1, Date.now());
|
||||
return cron;
|
||||
}
|
||||
|
||||
export const DailyCron = initialize_daily_cron();
|
||||
export const DailyCron = initializeDailyCron();
|
||||
|
||||
export const HourlyCron = initialize_hourly_cron();
|
||||
export const HourlyCron = initializeHourlyCron();
|
||||
|
|
|
@ -15,6 +15,7 @@ async function copy() {
|
|||
copyFile('LICENSE', 'build/LICENSE'),
|
||||
copyDir('public', 'build/public'),
|
||||
copyFile('tools/example-ecosystem.yml', 'build/ecosystem.example.yml'),
|
||||
copyFile('tools/example-ecosystem-backup.yml', 'build/ecosystem-backup.example.yml'),
|
||||
]);
|
||||
}
|
||||
|
||||
|
|
8
tools/example-ecosystem-backup.yml
Normal file
8
tools/example-ecosystem-backup.yml
Normal file
|
@ -0,0 +1,8 @@
|
|||
apps:
|
||||
- script : ./backup.js
|
||||
name : 'backup'
|
||||
node_args: --nouse-idle-notification --expose-gc
|
||||
env:
|
||||
CANVAS_REDIS_URL: 'redis://localhost:6379'
|
||||
BACKUP_REDIS_URL: 'redis://localhost:6380'
|
||||
BACKUP_DIR: '/backups'
|
|
@ -299,12 +299,13 @@ const webConfig = {
|
|||
|
||||
entry: {
|
||||
web: ['./src/web.js'],
|
||||
backup: ['./src/backup.js'],
|
||||
},
|
||||
|
||||
output: {
|
||||
...config.output,
|
||||
path: path.resolve(__dirname, '../build'),
|
||||
filename: './web.js',
|
||||
filename: '[name].js',
|
||||
libraryTarget: 'commonjs2',
|
||||
},
|
||||
|
||||
|
|
Loading…
Reference in New Issue
Block a user