2020-01-07 17:50:56 +00:00
|
|
|
/*
|
|
|
|
* Creates regular backups of the canvas in png tiles
|
|
|
|
* In order to run huge redis operations, you have to allow redis to use
|
|
|
|
* more virtual memory, with:
|
|
|
|
* vm.overcommit_memory = 1 in /etc/sysctl.conf and `sysctl vm.overcommit_memory=1`
|
|
|
|
* also:
|
|
|
|
* echo never > /sys/kernel/mm/transparent_hugepage/enabled
|
|
|
|
*
|
|
|
|
*/
|
|
|
|
|
|
|
|
/* eslint-disable no-console */
|
|
|
|
|
|
|
|
import fs from 'fs';
|
2022-04-04 00:15:24 +00:00
|
|
|
import os from 'os';
|
|
|
|
import { spawn } from 'child_process';
|
2020-01-08 19:22:52 +00:00
|
|
|
import path from 'path';
|
2022-04-06 19:50:34 +00:00
|
|
|
import { createClient } from 'redis';
|
2020-01-07 17:50:56 +00:00
|
|
|
|
2020-01-08 00:29:52 +00:00
|
|
|
|
|
|
|
import {
|
|
|
|
updateBackupRedis,
|
|
|
|
createPngBackup,
|
|
|
|
incrementialBackupRedis,
|
|
|
|
} from './core/tilesBackup';
|
2022-08-16 00:43:48 +00:00
|
|
|
import canvases from './core/canvases';
|
2020-01-08 00:29:52 +00:00
|
|
|
|
2020-01-07 17:50:56 +00:00
|
|
|
/*
|
|
|
|
* use low cpu priority
|
|
|
|
*/
|
2022-04-04 00:15:24 +00:00
|
|
|
const PRIORITY = 15;
|
|
|
|
console.log(`Setting priority for the current process to ${PRIORITY}`);
|
|
|
|
try {
|
|
|
|
os.setPriority(PRIORITY);
|
|
|
|
} catch (err) {
|
|
|
|
console.log(`: error occurred${err}`);
|
|
|
|
}
|
2020-01-07 17:50:56 +00:00
|
|
|
|
|
|
|
|
2020-01-08 01:53:10 +00:00
|
|
|
const [
|
2020-01-07 17:50:56 +00:00
|
|
|
CANVAS_REDIS_URL,
|
|
|
|
BACKUP_REDIS_URL,
|
|
|
|
BACKUP_DIR,
|
2020-01-08 01:53:10 +00:00
|
|
|
INTERVAL,
|
2020-01-08 19:59:24 +00:00
|
|
|
CMD,
|
2020-01-08 01:53:10 +00:00
|
|
|
] = process.argv.slice(2);
|
|
|
|
|
2020-01-07 17:50:56 +00:00
|
|
|
if (!CANVAS_REDIS_URL || !BACKUP_REDIS_URL || !BACKUP_DIR) {
|
2020-01-08 01:53:10 +00:00
|
|
|
console.error(
|
|
|
|
'Usage: node backup.js original_canvas backup_canvas backup_directory',
|
2020-01-07 17:50:56 +00:00
|
|
|
);
|
2020-01-08 01:53:10 +00:00
|
|
|
process.exit(1);
|
2020-01-07 17:50:56 +00:00
|
|
|
}
|
|
|
|
|
2022-04-06 19:50:34 +00:00
|
|
|
const canvasRedis = createClient(CANVAS_REDIS_URL
|
|
|
|
.startsWith('redis://')
|
|
|
|
? {
|
|
|
|
url: CANVAS_REDIS_URL,
|
|
|
|
}
|
|
|
|
: {
|
|
|
|
socket: {
|
|
|
|
path: CANVAS_REDIS_URL,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
);
|
|
|
|
const backupRedis = createClient(BACKUP_REDIS_URL
|
|
|
|
.startsWith('redis://')
|
|
|
|
? {
|
|
|
|
url: BACKUP_REDIS_URL,
|
|
|
|
}
|
|
|
|
: {
|
|
|
|
socket: {
|
|
|
|
path: BACKUP_REDIS_URL,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
);
|
|
|
|
//
|
2020-01-07 17:50:56 +00:00
|
|
|
canvasRedis.on('error', () => {
|
2020-01-08 19:59:24 +00:00
|
|
|
console.error('Could not connect to canvas redis');
|
|
|
|
process.exit(1);
|
2020-01-07 17:50:56 +00:00
|
|
|
});
|
|
|
|
backupRedis.on('error', () => {
|
2020-01-08 19:59:24 +00:00
|
|
|
console.error('Could not connect to backup redis');
|
|
|
|
process.exit(1);
|
2020-01-07 17:50:56 +00:00
|
|
|
});
|
|
|
|
|
|
|
|
|
2022-08-01 15:44:18 +00:00
|
|
|
function runCmd(cmd) {
|
2020-01-08 19:59:24 +00:00
|
|
|
const startTime = Date.now();
|
|
|
|
console.log(`Executing ${cmd}`);
|
|
|
|
const cmdproc = spawn(cmd);
|
|
|
|
cmdproc.on('exit', (code) => {
|
|
|
|
if (code !== 0) {
|
|
|
|
console.log(`${cmd} failed with code ${code}`);
|
|
|
|
}
|
|
|
|
const time = Date.now() - startTime;
|
|
|
|
console.log(`${cmd} done in ${time}ms`);
|
|
|
|
});
|
|
|
|
cmdproc.stdout.on('data', (data) => {
|
2020-01-08 20:54:32 +00:00
|
|
|
console.log(`${cmd}: ${data}`);
|
2020-01-08 19:59:24 +00:00
|
|
|
});
|
|
|
|
cmdproc.stderr.on('data', (data) => {
|
2020-01-08 20:54:32 +00:00
|
|
|
console.log(`${cmd} error: ${data}`);
|
2020-01-08 19:59:24 +00:00
|
|
|
});
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2020-01-08 00:29:52 +00:00
|
|
|
function getDateFolder() {
|
2020-01-08 19:22:52 +00:00
|
|
|
const dir = path.resolve(__dirname, BACKUP_DIR);
|
|
|
|
if (!fs.existsSync(dir)) {
|
2020-06-13 09:24:16 +00:00
|
|
|
// eslint-disable-next-line max-len
|
2020-06-13 19:02:16 +00:00
|
|
|
console.info(`Backup directory ${BACKUP_DIR} does not exist! Trying to create it`);
|
|
|
|
try {
|
|
|
|
fs.mkdirSync(dir);
|
|
|
|
} catch {
|
|
|
|
console.error('Couldn\'t create backup dir');
|
|
|
|
process.exit(1);
|
|
|
|
}
|
2020-01-08 00:29:52 +00:00
|
|
|
}
|
|
|
|
const date = new Date();
|
2022-06-20 09:46:28 +00:00
|
|
|
let month = date.getUTCMonth() + 1;
|
|
|
|
let day = date.getUTCDate();
|
2020-01-08 19:59:24 +00:00
|
|
|
if (month < 10) month = `0${month}`;
|
|
|
|
if (day < 10) day = `0${day}`;
|
2022-06-20 09:46:28 +00:00
|
|
|
const dayDir = `${date.getUTCFullYear()}/${month}/${day}`;
|
2020-01-08 19:22:52 +00:00
|
|
|
const backupDir = `${dir}/${dayDir}`;
|
2020-01-08 00:29:52 +00:00
|
|
|
return backupDir;
|
|
|
|
}
|
|
|
|
|
2020-01-08 01:53:10 +00:00
|
|
|
async function dailyBackup() {
|
2020-01-08 00:29:52 +00:00
|
|
|
const backupDir = getDateFolder();
|
|
|
|
if (!fs.existsSync(backupDir)) {
|
2021-05-31 20:51:10 +00:00
|
|
|
fs.mkdirSync(backupDir, { recursive: true });
|
2020-01-07 17:50:56 +00:00
|
|
|
}
|
|
|
|
|
2022-04-08 12:59:59 +00:00
|
|
|
await backupRedis.flushAll('ASYNC');
|
2021-05-31 20:51:10 +00:00
|
|
|
|
2020-05-02 02:44:26 +00:00
|
|
|
try {
|
|
|
|
await updateBackupRedis(canvasRedis, backupRedis, canvases);
|
|
|
|
await createPngBackup(backupRedis, canvases, backupDir);
|
|
|
|
} catch (e) {
|
|
|
|
console.log('Error occured during daily backup', e);
|
|
|
|
}
|
2020-01-08 20:39:24 +00:00
|
|
|
console.log('Daily full backup done');
|
2020-01-07 17:50:56 +00:00
|
|
|
}
|
|
|
|
|
2020-01-08 20:54:32 +00:00
|
|
|
async function incrementialBackup() {
|
2020-01-08 00:29:52 +00:00
|
|
|
const backupDir = getDateFolder();
|
|
|
|
if (!fs.existsSync(backupDir)) {
|
2021-05-31 20:51:10 +00:00
|
|
|
fs.mkdirSync(backupDir, { recursive: true });
|
2020-01-08 00:29:52 +00:00
|
|
|
}
|
2020-05-02 02:44:26 +00:00
|
|
|
try {
|
|
|
|
await incrementialBackupRedis(
|
|
|
|
canvasRedis,
|
|
|
|
backupRedis,
|
|
|
|
canvases,
|
|
|
|
backupDir,
|
|
|
|
);
|
|
|
|
} catch (e) {
|
|
|
|
console.log('Error occured during incremential backup', e);
|
|
|
|
}
|
2020-01-08 00:29:52 +00:00
|
|
|
}
|
|
|
|
|
2020-01-08 01:53:10 +00:00
|
|
|
async function trigger() {
|
2020-01-08 00:29:52 +00:00
|
|
|
const backupDir = getDateFolder();
|
|
|
|
if (!fs.existsSync(backupDir)) {
|
|
|
|
await dailyBackup();
|
2020-01-08 01:53:10 +00:00
|
|
|
} else {
|
|
|
|
await incrementialBackup();
|
2020-01-08 00:29:52 +00:00
|
|
|
}
|
2020-01-08 19:59:24 +00:00
|
|
|
if (CMD) {
|
|
|
|
runCmd(CMD);
|
|
|
|
}
|
2020-01-08 01:53:10 +00:00
|
|
|
if (!INTERVAL) {
|
|
|
|
process.exit(0);
|
|
|
|
}
|
2020-01-08 19:22:52 +00:00
|
|
|
console.log(`Creating next backup in ${INTERVAL} minutes`);
|
2020-01-08 01:53:10 +00:00
|
|
|
setTimeout(trigger, INTERVAL * 60 * 1000);
|
2020-01-08 00:29:52 +00:00
|
|
|
}
|
|
|
|
|
2020-01-08 01:53:10 +00:00
|
|
|
console.log('Starting backup...');
|
2022-04-06 19:50:34 +00:00
|
|
|
canvasRedis.connect()
|
|
|
|
.then(() => backupRedis.connect())
|
|
|
|
.then(() => trigger());
|