make backup.js deployable and add cron timers to it
This commit is contained in:
parent
759bffbf1f
commit
e6c6b45a6b
|
@ -15,16 +15,25 @@ import fs from 'fs';
|
||||||
import redis from 'redis';
|
import redis from 'redis';
|
||||||
import bluebird from 'bluebird';
|
import bluebird from 'bluebird';
|
||||||
|
|
||||||
|
import process from 'process';
|
||||||
|
import { spawn } from 'child_process';
|
||||||
|
|
||||||
|
import { DailyCron } from './utils/cron';
|
||||||
|
import {
|
||||||
|
updateBackupRedis,
|
||||||
|
createPngBackup,
|
||||||
|
incrementialBackupRedis,
|
||||||
|
} from './core/tilesBackup';
|
||||||
|
import canvases from './canvases.json';
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* use low cpu priority
|
* use low cpu priority
|
||||||
*/
|
*/
|
||||||
import process from 'process';
|
const priority = 15;
|
||||||
import { spawn } from 'child_process';
|
const proc = spawn('renice', [priority, process.pid]);
|
||||||
const priority = 15;
|
proc.on('exit', (code) => {
|
||||||
const proc= spawn("renice", [priority, process.pid]);
|
if (code !== 0) {
|
||||||
proc.on('exit', function (code) {
|
console.log(`renice failed with code ${code}`);
|
||||||
if (code !== 0){
|
|
||||||
console.log("renice failed with code - " +code);
|
|
||||||
}
|
}
|
||||||
console.log('Useing low cpu priority');
|
console.log('Useing low cpu priority');
|
||||||
});
|
});
|
||||||
|
@ -34,13 +43,6 @@ proc.on('exit', function (code) {
|
||||||
bluebird.promisifyAll(redis.RedisClient.prototype);
|
bluebird.promisifyAll(redis.RedisClient.prototype);
|
||||||
bluebird.promisifyAll(redis.Multi.prototype);
|
bluebird.promisifyAll(redis.Multi.prototype);
|
||||||
|
|
||||||
import canvases from './canvases.json';
|
|
||||||
import {
|
|
||||||
updateBackupRedis,
|
|
||||||
createPngBackup,
|
|
||||||
incrementialBackupRedis,
|
|
||||||
} from './core/tilesBackup';
|
|
||||||
|
|
||||||
const {
|
const {
|
||||||
CANVAS_REDIS_URL,
|
CANVAS_REDIS_URL,
|
||||||
BACKUP_REDIS_URL,
|
BACKUP_REDIS_URL,
|
||||||
|
@ -64,23 +66,60 @@ backupRedis.on('error', () => {
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
||||||
function dailyBackup() {
|
function getDateFolder() {
|
||||||
if (!fs.existsSync(BACKUP_DIR)) {
|
if (!fs.existsSync(BACKUP_DIR)) {
|
||||||
throw new Error(`Backup directory ${backupDir} does not exist!`);
|
throw new Error(`Backup directory ${BACKUP_DIR} does not exist!`);
|
||||||
|
}
|
||||||
|
const date = new Date();
|
||||||
|
// eslint-disable-next-line max-len
|
||||||
|
const dayDir = `${date.getFullYear()}${date.getMonth() + 1}${date.getDate()}`;
|
||||||
|
const backupDir = `${BACKUP_DIR}/${dayDir}`;
|
||||||
|
return backupDir;
|
||||||
|
}
|
||||||
|
|
||||||
|
function dailyBackup() {
|
||||||
|
const backupDir = getDateFolder();
|
||||||
|
if (!fs.existsSync(backupDir)) {
|
||||||
|
fs.mkdirSync(backupDir);
|
||||||
}
|
}
|
||||||
|
|
||||||
backupRedis.flushall('ASYNC', async () => {
|
backupRedis.flushall('ASYNC', async () => {
|
||||||
const date = new Date();
|
|
||||||
const dayDir = `${date.getFullYear()}${date.getMonth() + 1}${date.getDate()}`;
|
|
||||||
const backupDir = `${BACKUP_DIR}/${dayDir}`;
|
|
||||||
if (!fs.existsSync(backupDir)) {
|
if (!fs.existsSync(backupDir)) {
|
||||||
fs.mkdirSync(backupDir);
|
fs.mkdirSync(backupDir);
|
||||||
}
|
}
|
||||||
await updateBackupRedis(canvasRedis, backupRedis, canvases);
|
await updateBackupRedis(canvasRedis, backupRedis, canvases);
|
||||||
await createPngBackup(backupRedis, canvases, backupDir);
|
await createPngBackup(backupRedis, canvases, backupDir);
|
||||||
await incrementialBackupRedis(canvasRedis, backupRedis, canvases, backupDir);
|
console.log('Daily backup done');
|
||||||
console.log(`Daily backup ${dayDir} done`);
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
dailyBackup();
|
function incrementialBackup() {
|
||||||
|
const backupDir = getDateFolder();
|
||||||
|
if (!fs.existsSync(backupDir)) {
|
||||||
|
fs.mkdirSync(backupDir);
|
||||||
|
}
|
||||||
|
incrementialBackupRedis(
|
||||||
|
canvasRedis,
|
||||||
|
backupRedis,
|
||||||
|
canvases,
|
||||||
|
backupDir,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function startCronJobs() {
|
||||||
|
if (!fs.existsSync(BACKUP_DIR)) {
|
||||||
|
throw new Error(`Backup directory ${BACKUP_DIR} does not exist!`);
|
||||||
|
}
|
||||||
|
const backupDir = getDateFolder();
|
||||||
|
if (!fs.existsSync(backupDir)) {
|
||||||
|
await dailyBackup();
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(
|
||||||
|
'Creating one full PNG backup per day and incremential backups every hour.',
|
||||||
|
);
|
||||||
|
DailyCron.hook(dailyBackup);
|
||||||
|
setInterval(incrementialBackup, 15 * 60 * 1000);
|
||||||
|
}
|
||||||
|
|
||||||
|
startCronJobs();
|
||||||
|
|
|
@ -72,8 +72,8 @@ export async function incrementialBackupRedis(
|
||||||
if (!fs.existsSync(canvasBackupDir)) {
|
if (!fs.existsSync(canvasBackupDir)) {
|
||||||
fs.mkdirSync(canvasBackupDir);
|
fs.mkdirSync(canvasBackupDir);
|
||||||
}
|
}
|
||||||
const hourOfDay = new Date().getHours();
|
const date = new Date();
|
||||||
const canvasTileBackupDir = `${canvasBackupDir}/${hourOfDay}`;
|
const canvasTileBackupDir = `${canvasBackupDir}/${date.getHours()}${date.getMinutes()}`;
|
||||||
if (!fs.existsSync(canvasTileBackupDir)) {
|
if (!fs.existsSync(canvasTileBackupDir)) {
|
||||||
fs.mkdirSync(canvasTileBackupDir);
|
fs.mkdirSync(canvasTileBackupDir);
|
||||||
}
|
}
|
||||||
|
|
|
@ -9,27 +9,27 @@ import { HOUR } from '../core/constants';
|
||||||
import logger from '../core/logger';
|
import logger from '../core/logger';
|
||||||
|
|
||||||
class Cron {
|
class Cron {
|
||||||
last_run: number;
|
lastRun: number;
|
||||||
interval: number;
|
interval: number;
|
||||||
functions: Array;
|
functions: Array;
|
||||||
timeout;
|
timeout;
|
||||||
|
|
||||||
// interval = how many hours between runs
|
// interval = how many hours between runs
|
||||||
// last_run = when this cron job was last run
|
// lastRun = when this cron job was last run
|
||||||
constructor(interval: number, last_run: number = 0) {
|
constructor(interval: number, lastRun: number = 0) {
|
||||||
this.check_for_execution = this.check_for_execution.bind(this);
|
this.checkForExecution = this.checkForExecution.bind(this);
|
||||||
this.interval = interval;
|
this.interval = interval;
|
||||||
this.last_run = last_run;
|
this.lastRun = lastRun;
|
||||||
this.functions = [];
|
this.functions = [];
|
||||||
|
|
||||||
this.timeout = setInterval(this.check_for_execution, HOUR);
|
this.timeout = setInterval(this.checkForExecution, HOUR);
|
||||||
}
|
}
|
||||||
|
|
||||||
check_for_execution() {
|
checkForExecution() {
|
||||||
const cur_time = Date.now();
|
const curTime = Date.now();
|
||||||
if (cur_time > this.last_run + this.interval * HOUR) {
|
if (curTime > this.lastRun + this.interval * HOUR) {
|
||||||
logger.info(`Run cron events for interval: ${this.interval}h`);
|
logger.info(`Run cron events for interval: ${this.interval}h`);
|
||||||
this.last_run = cur_time;
|
this.lastRun = curTime;
|
||||||
this.functions.forEach(async (item) => {
|
this.functions.forEach(async (item) => {
|
||||||
item();
|
item();
|
||||||
});
|
});
|
||||||
|
@ -42,19 +42,19 @@ class Cron {
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
function initialize_daily_cron() {
|
function initializeDailyCron() {
|
||||||
const now = new Date();
|
const now = new Date();
|
||||||
// make it first run at midnight
|
// make it first run at midnight
|
||||||
const last_run = now.getTime() - now.getHours() * HOUR;
|
const lastRun = now.getTime() - now.getHours() * HOUR;
|
||||||
const cron = new Cron(24, last_run);
|
const cron = new Cron(24, lastRun);
|
||||||
return cron;
|
return cron;
|
||||||
}
|
}
|
||||||
|
|
||||||
function initialize_hourly_cron() {
|
function initializeHourlyCron() {
|
||||||
const cron = new Cron(1, Date.now());
|
const cron = new Cron(1, Date.now());
|
||||||
return cron;
|
return cron;
|
||||||
}
|
}
|
||||||
|
|
||||||
export const DailyCron = initialize_daily_cron();
|
export const DailyCron = initializeDailyCron();
|
||||||
|
|
||||||
export const HourlyCron = initialize_hourly_cron();
|
export const HourlyCron = initializeHourlyCron();
|
||||||
|
|
|
@ -15,6 +15,7 @@ async function copy() {
|
||||||
copyFile('LICENSE', 'build/LICENSE'),
|
copyFile('LICENSE', 'build/LICENSE'),
|
||||||
copyDir('public', 'build/public'),
|
copyDir('public', 'build/public'),
|
||||||
copyFile('tools/example-ecosystem.yml', 'build/ecosystem.example.yml'),
|
copyFile('tools/example-ecosystem.yml', 'build/ecosystem.example.yml'),
|
||||||
|
copyFile('tools/example-ecosystem-backup.yml', 'build/ecosystem-backup.example.yml'),
|
||||||
]);
|
]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
8
tools/example-ecosystem-backup.yml
Normal file
8
tools/example-ecosystem-backup.yml
Normal file
|
@ -0,0 +1,8 @@
|
||||||
|
apps:
|
||||||
|
- script : ./backup.js
|
||||||
|
name : 'backup'
|
||||||
|
node_args: --nouse-idle-notification --expose-gc
|
||||||
|
env:
|
||||||
|
CANVAS_REDIS_URL: 'redis://localhost:6379'
|
||||||
|
BACKUP_REDIS_URL: 'redis://localhost:6380'
|
||||||
|
BACKUP_DIR: '/backups'
|
|
@ -299,12 +299,13 @@ const webConfig = {
|
||||||
|
|
||||||
entry: {
|
entry: {
|
||||||
web: ['./src/web.js'],
|
web: ['./src/web.js'],
|
||||||
|
backup: ['./src/backup.js'],
|
||||||
},
|
},
|
||||||
|
|
||||||
output: {
|
output: {
|
||||||
...config.output,
|
...config.output,
|
||||||
path: path.resolve(__dirname, '../build'),
|
path: path.resolve(__dirname, '../build'),
|
||||||
filename: './web.js',
|
filename: '[name].js',
|
||||||
libraryTarget: 'commonjs2',
|
libraryTarget: 'commonjs2',
|
||||||
},
|
},
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue
Block a user