355 lines
10 KiB
JavaScript
355 lines
10 KiB
JavaScript
/*
|
|
* Offer functions for Canvas backups
|
|
*
|
|
*/
|
|
|
|
/* eslint-disable no-console */
|
|
|
|
import sharp from 'sharp';
|
|
import fs from 'fs';
|
|
import { commandOptions } from 'redis';
|
|
|
|
import Palette from './Palette';
|
|
import { TILE_SIZE } from './constants';
|
|
|
|
|
|
/*
|
|
* Copy canvases from one redis instance to another
|
|
* @param canvasRedis redis from where to get the data
|
|
* @param backupRedis redis where to write the data to
|
|
* @param canvases Object with all canvas information
|
|
*/
|
|
export async function updateBackupRedis(canvasRedis, backupRedis, canvases) {
|
|
const ids = Object.keys(canvases);
|
|
for (let i = 0; i < ids.length; i += 1) {
|
|
const id = ids[i];
|
|
const canvas = canvases[id];
|
|
if (canvas.v || canvas.hid || canvas.ed) {
|
|
// ignore 3D, hidden and retired canvases
|
|
continue;
|
|
}
|
|
const chunksXY = (canvas.size / TILE_SIZE);
|
|
console.log('Copy Chunks to backup redis...');
|
|
const startTime = Date.now();
|
|
let amount = 0;
|
|
for (let x = 0; x < chunksXY; x++) {
|
|
for (let y = 0; y < chunksXY; y++) {
|
|
const key = `ch:${id}:${x}:${y}`;
|
|
let chunk = null;
|
|
|
|
try {
|
|
// eslint-disable-next-line no-await-in-loop
|
|
chunk = await canvasRedis.get(
|
|
commandOptions({ returnBuffers: true }),
|
|
key,
|
|
);
|
|
} catch (error) {
|
|
console.error(
|
|
// eslint-disable-next-line max-len
|
|
new Error(`Could not get chunk ${key} from redis: ${error.message}`),
|
|
);
|
|
}
|
|
if (chunk) {
|
|
try {
|
|
// eslint-disable-next-line no-await-in-loop
|
|
await backupRedis.set(key, chunk);
|
|
amount += 1;
|
|
} catch (error) {
|
|
console.error(
|
|
// eslint-disable-next-line max-len
|
|
new Error(`Could not create chunk ${key} in backup-redis: ${error.message}`),
|
|
);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
const time = Date.now() - startTime;
|
|
console.log(`Finished Copying ${amount} chunks in ${time}ms.`);
|
|
}
|
|
}
|
|
|
|
|
|
/*
|
|
* Create incremental PNG tile backup between two redis canvases
|
|
* @param canvasRedis redis from where to get the data
|
|
* @param backupRedis redis where to write the data to
|
|
* @param canvases Object with all canvas information
|
|
*/
|
|
export async function incrementalBackupRedis(
|
|
canvasRedis,
|
|
backupRedis,
|
|
canvases,
|
|
backupDir,
|
|
) {
|
|
const ids = Object.keys(canvases);
|
|
for (let ind = 0; ind < ids.length; ind += 1) {
|
|
const id = ids[ind];
|
|
|
|
const canvas = canvases[id];
|
|
if (canvas.v || canvas.hid || canvas.ed) {
|
|
// ignore 3D, hidden and retired canvases
|
|
continue;
|
|
}
|
|
|
|
const canvasBackupDir = `${backupDir}/${id}`;
|
|
if (!fs.existsSync(canvasBackupDir)) {
|
|
fs.mkdirSync(canvasBackupDir);
|
|
}
|
|
const date = new Date();
|
|
let hours = date.getUTCHours();
|
|
let minutes = date.getUTCMinutes();
|
|
if (hours < 10) hours = `0${hours}`;
|
|
if (minutes < 10) minutes = `0${minutes}`;
|
|
const canvasTileBackupDir = `${canvasBackupDir}/${hours}${minutes}`;
|
|
console.log(`Using folder ${canvasTileBackupDir}`);
|
|
if (!fs.existsSync(canvasTileBackupDir)) {
|
|
fs.mkdirSync(canvasTileBackupDir);
|
|
}
|
|
|
|
const palette = new Palette(canvas.colors);
|
|
const chunksXY = (canvas.size / TILE_SIZE);
|
|
console.log('Creating Incremental Backup...');
|
|
const startTime = Date.now();
|
|
let amount = 0;
|
|
for (let x = 0; x < chunksXY; x++) {
|
|
const xBackupDir = `${canvasTileBackupDir}/${x}`;
|
|
let createdDir = false;
|
|
|
|
for (let y = 0; y < chunksXY; y++) {
|
|
const key = `ch:${id}:${x}:${y}`;
|
|
|
|
let curChunk = null;
|
|
try {
|
|
// eslint-disable-next-line no-await-in-loop
|
|
curChunk = await canvasRedis.get(
|
|
commandOptions({ returnBuffers: true }),
|
|
key,
|
|
);
|
|
} catch (error) {
|
|
console.error(
|
|
// eslint-disable-next-line max-len
|
|
new Error(`Could not get chunk ${key} from redis: ${error.message}`),
|
|
);
|
|
}
|
|
|
|
let oldChunk = null;
|
|
try {
|
|
// eslint-disable-next-line no-await-in-loop
|
|
oldChunk = await backupRedis.get(
|
|
commandOptions({ returnBuffers: true }),
|
|
key,
|
|
);
|
|
} catch (error) {
|
|
console.error(
|
|
// eslint-disable-next-line max-len
|
|
new Error(`Could not get chunk ${key} from backup-redis: ${error.message}`),
|
|
);
|
|
}
|
|
|
|
let tileBuffer = null;
|
|
|
|
try {
|
|
if (!oldChunk && !curChunk) {
|
|
continue;
|
|
}
|
|
if (!oldChunk) {
|
|
oldChunk = Buffer.allocUnsafe(0);
|
|
}
|
|
if (!curChunk) {
|
|
curChunk = Buffer.allocUnsafe(0);
|
|
}
|
|
|
|
const { abgr } = palette;
|
|
const oldChunkLength = oldChunk.length;
|
|
const curChunkLength = curChunk.length;
|
|
if (oldChunkLength !== curChunkLength) {
|
|
tileBuffer = new Uint32Array(TILE_SIZE ** 2);
|
|
}
|
|
|
|
const compLength = Math.min(oldChunkLength, curChunkLength);
|
|
for (let i = 0; i < compLength; i += 1) {
|
|
const curPxl = curChunk[i];
|
|
if (oldChunk[i] !== curPxl) {
|
|
if (!tileBuffer) {
|
|
tileBuffer = new Uint32Array(TILE_SIZE ** 2);
|
|
}
|
|
tileBuffer[i] = abgr[curPxl & 0x3F];
|
|
}
|
|
}
|
|
|
|
if (!tileBuffer) {
|
|
continue;
|
|
}
|
|
|
|
for (let i = oldChunkLength; i < curChunkLength; i += 1) {
|
|
// current chunk longer than old chunk
|
|
tileBuffer[i] = abgr[curChunk[i] & 0x3F];
|
|
}
|
|
|
|
if (oldChunkLength > curChunkLength) {
|
|
// old chunk longer than current chunk
|
|
const blank = abgr[0];
|
|
for (let i = curChunkLength; i < oldChunkLength; i += 1) {
|
|
if (oldChunk[i] !== 0) {
|
|
tileBuffer[i] = blank;
|
|
}
|
|
}
|
|
}
|
|
} catch (error) {
|
|
console.error(
|
|
// eslint-disable-next-line max-len
|
|
new Error(`Could not populate incremental backup data of chunk ${key}: ${error.message}`),
|
|
);
|
|
continue;
|
|
}
|
|
|
|
try {
|
|
if (!createdDir && !fs.existsSync(xBackupDir)) {
|
|
createdDir = true;
|
|
fs.mkdirSync(xBackupDir);
|
|
}
|
|
const filename = `${xBackupDir}/${y}.png`;
|
|
// eslint-disable-next-line no-await-in-loop
|
|
await sharp(
|
|
Buffer.from(tileBuffer.buffer), {
|
|
raw: {
|
|
width: TILE_SIZE,
|
|
height: TILE_SIZE,
|
|
channels: 4,
|
|
},
|
|
},
|
|
).toFile(filename);
|
|
amount += 1;
|
|
} catch (error) {
|
|
console.error(
|
|
// eslint-disable-next-line max-len
|
|
new Error(`Could not save incremental backup of chunk ${key}: ${error.message}`),
|
|
);
|
|
}
|
|
}
|
|
}
|
|
const time = Date.now() - startTime;
|
|
console.log(
|
|
`Finished Incremental backup of ${amount} chunks in ${time}ms.`,
|
|
);
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Take a buffer of indexed pixels and output it as RGB Array
|
|
* of full chunk size
|
|
* @param chunkBuffer Buffer of indexed pixels
|
|
* @param palette
|
|
* @return RGB Buffer
|
|
*/
|
|
function buffer2RGB(palette, chunkBuffer) {
|
|
const length = TILE_SIZE ** 2;
|
|
const minLength = Math.min(chunkBuffer.length, length);
|
|
const colors = new Uint8Array(length * 3);
|
|
const { rgb } = palette;
|
|
let color;
|
|
let c = 0;
|
|
for (let i = 0; i < minLength; i++) {
|
|
color = (chunkBuffer[i] & 0x3F) * 3;
|
|
colors[c++] = rgb[color++];
|
|
colors[c++] = rgb[color++];
|
|
colors[c++] = rgb[color];
|
|
}
|
|
|
|
if (minLength < length) {
|
|
const blankR = rgb[0];
|
|
const blankG = rgb[1];
|
|
const blankB = rgb[2];
|
|
for (let i = minLength; i < length; i += 1) {
|
|
colors[c++] = blankR;
|
|
colors[c++] = blankG;
|
|
colors[c++] = blankB;
|
|
}
|
|
}
|
|
return colors;
|
|
}
|
|
|
|
/*
|
|
* Backup all tiles as PNG files into folder
|
|
* @param redisClient RedisClient
|
|
* @param canvases Object with the information to all canvases
|
|
* @param backupDir directory where to save png tiles
|
|
*/
|
|
export async function createPngBackup(
|
|
redisClient,
|
|
canvases,
|
|
backupDir,
|
|
) {
|
|
const ids = Object.keys(canvases);
|
|
for (let ind = 0; ind < ids.length; ind += 1) {
|
|
const id = ids[ind];
|
|
|
|
const canvasBackupDir = `${backupDir}/${id}`;
|
|
if (!fs.existsSync(canvasBackupDir)) {
|
|
fs.mkdirSync(canvasBackupDir);
|
|
}
|
|
const canvasTileBackupDir = `${canvasBackupDir}/tiles`;
|
|
if (!fs.existsSync(canvasTileBackupDir)) {
|
|
fs.mkdirSync(canvasTileBackupDir);
|
|
}
|
|
|
|
const canvas = canvases[id];
|
|
const palette = new Palette(canvas.colors);
|
|
const chunksXY = (canvas.size / TILE_SIZE);
|
|
console.log('Create PNG tiles from backup...');
|
|
const startTime = Date.now();
|
|
let amount = 0;
|
|
for (let x = 0; x < chunksXY; x++) {
|
|
const xBackupDir = `${canvasTileBackupDir}/${x}`;
|
|
if (!fs.existsSync(xBackupDir)) {
|
|
fs.mkdirSync(xBackupDir);
|
|
}
|
|
for (let y = 0; y < chunksXY; y++) {
|
|
const key = `ch:${id}:${x}:${y}`;
|
|
|
|
let chunk = null;
|
|
try {
|
|
// eslint-disable-next-line no-await-in-loop
|
|
chunk = await redisClient.get(
|
|
commandOptions({ returnBuffers: true }),
|
|
key,
|
|
);
|
|
} catch (error) {
|
|
console.error(
|
|
// eslint-disable-next-line max-len
|
|
new Error(`Could not get chunk ${key} from redis: ${error.message}`),
|
|
);
|
|
}
|
|
if (chunk && chunk.length) {
|
|
try {
|
|
const tileBuffer = buffer2RGB(palette, chunk);
|
|
const filename = `${xBackupDir}/${y}.png`;
|
|
|
|
// eslint-disable-next-line no-await-in-loop
|
|
await sharp(
|
|
Buffer.from(tileBuffer.buffer), {
|
|
raw: {
|
|
width: TILE_SIZE,
|
|
height: TILE_SIZE,
|
|
channels: 3,
|
|
},
|
|
},
|
|
).toFile(filename);
|
|
} catch (error) {
|
|
console.error(
|
|
// eslint-disable-next-line max-len
|
|
new Error(`Could not save daily backup of chunk ${key}: ${error.message}`),
|
|
);
|
|
continue;
|
|
}
|
|
amount += 1;
|
|
}
|
|
}
|
|
}
|
|
const time = Date.now() - startTime;
|
|
console.log(
|
|
`Finished creating PNG backup of ${amount} chunks in ${time}ms.`,
|
|
);
|
|
}
|
|
}
|