ppfun/pixelplanet
13
Fork 18
Code Issues 24 Pull Requests Releases 2 Activity

refactor redis to deal with chunks of non-full size

Browse Source
This commit is contained in:
HF 2022-04-10 22:47:10 +02:00
parent 56ff4a0b2c
commit 88e33a4ff6
12 changed files with 360 additions and 264 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
src/core/CanvasCleaner.js
View File

@ -373,13 +373,27 @@ class CanvasCleaner {
) {
chunk = chunks[jAbs - jo + 1][iAbs - io + 1];
} else {
// eslint-disable-next-line no-await-in-loop
chunk = await RedisCanvas.getChunk(canvasId, iAbs, jAbs);
if (!chunk || chunk.length !== TILE_SIZE * TILE_SIZE) {
try {
// eslint-disable-next-line no-await-in-loop
chunk = await RedisCanvas.getChunk(
canvasId,
iAbs,
jAbs,
TILE_SIZE ** 2,
);
} catch (error) {
this.logger(
// eslint-disable-next-line max-len
`Couldn't load chunk ch:${canvasId}:${iAbs}:${jAbs}: ${error.message}}`,
);
}
if (!chunk || !chunk.length) {
chunk = null;
if (chunk) {
// eslint-disable-next-line max-len
this.logger(`Chunk ch:${canvasId}:${iAbs}:${jAbs} has invalid size ${chunk.length}.`);
this.logger(
// eslint-disable-next-line max-len
`Chunk ch:${canvasId}:${iAbs}:${jAbs} has invalid size ${chunk.length}.`,
);
}
}
}

42
src/core/Image.js
View File

@ -39,6 +39,7 @@ export async function imageABGR2Canvas(
logger.info(
`Loading image with dim ${width}/${height} to ${x}/${y}/${canvasId}`,
);
const expectedLength = TILE_SIZE ** 2;
const canvas = canvases[canvasId];
const { colors, cli, size } = canvas;
const palette = new Palette(colors);
@ -50,11 +51,18 @@ export async function imageABGR2Canvas(
let totalPxlCnt = 0;
logger.info(`Loading to chunks from ${ucx} / ${ucy} to ${lcx} / ${lcy} ...`);
let chunk;
for (let cx = ucx; cx <= lcx; cx += 1) {
for (let cy = ucy; cy <= lcy; cy += 1) {
chunk = await RedisCanvas.getChunk(canvasId, cx, cy);
chunk = (chunk && chunk.length === TILE_SIZE * TILE_SIZE)
let chunk = null;
try {
chunk = await RedisCanvas.getChunk(canvasId, cx, cy, expectedLength);
} catch (error) {
logger.error(
// eslint-disable-next-line max-len
`Could not load chunk ch:${canvasId}:${cx}:${cy} for image-load: ${error.message}`,
);
}
chunk = (chunk && chunk.length)
? new Uint8Array(chunk)
: new Uint8Array(TILE_SIZE * TILE_SIZE);
// offset of chunk in image
@ -119,6 +127,7 @@ export async function imagemask2Canvas(
logger.info(
`Loading mask with size ${width} / ${height} to ${x} / ${y} to the canvas`,
);
const expectedLength = TILE_SIZE ** 2;
const canvas = canvases[canvasId];
const palette = new Palette(canvas.colors);
const canvasMinXY = -(canvas.size / 2);
@ -129,11 +138,18 @@ export async function imagemask2Canvas(
const [lcx, lcy] = getChunkOfPixel(canvas.size, x + width, y + height);
logger.info(`Loading to chunks from ${ucx} / ${ucy} to ${lcx} / ${lcy} ...`);
let chunk;
for (let cx = ucx; cx <= lcx; cx += 1) {
for (let cy = ucy; cy <= lcy; cy += 1) {
chunk = await RedisCanvas.getChunk(canvasId, cx, cy);
chunk = (chunk && chunk.length === TILE_SIZE * TILE_SIZE)
let chunk = null;
try {
chunk = await RedisCanvas.getChunk(canvasId, cx, cy, expectedLength);
} catch (error) {
logger.error(
// eslint-disable-next-line max-len
`Could not load chunk ch:${canvasId}:${cx}:${cy} for imagemask-load: ${error.message}`,
);
}
chunk = (chunk && chunk.length)
? new Uint8Array(chunk)
: new Uint8Array(TILE_SIZE * TILE_SIZE);
// offset of chunk in image
@ -191,6 +207,7 @@ export async function protectCanvasArea(
// eslint-disable-next-line max-len
`Setting protection ${protect} with size ${width} / ${height} to ${x} / ${y}`,
);
const expectedLength = TILE_SIZE ** 2;
const canvas = canvases[canvasId];
const canvasMinXY = -(canvas.size / 2);
@ -201,11 +218,18 @@ export async function protectCanvasArea(
);
let totalPxlCnt = 0;
let chunk;
for (let cx = ucx; cx <= lcx; cx += 1) {
for (let cy = ucy; cy <= lcy; cy += 1) {
chunk = await RedisCanvas.getChunk(canvasId, cx, cy);
if (!chunk || chunk.length !== TILE_SIZE * TILE_SIZE) {
let chunk = null;
try {
chunk = await RedisCanvas.getChunk(canvasId, cx, cy, expectedLength);
} catch (error) {
logger.error(
// eslint-disable-next-line max-len
`Could not load chunk ch:${canvasId}:${cx}:${cy} for protection: ${error.message}`,
);
}
if (!chunk || !chunk.length) {
continue;
}
chunk = new Uint8Array(chunk);

3
src/core/Palette.js
View File

@ -110,11 +110,10 @@ class Palette {
const { length } = chunkBuffer;
const colors = new Uint32Array(length);
let value;
const buffer = chunkBuffer;
let pos = 0;
for (let i = 0; i < length; i++) {
value = (buffer[i] & 0x3F);
value = (chunkBuffer[i] & 0x3F);
colors[pos++] = this.abgr[value];
}
return colors;

82
src/core/Tile.js
View File

@ -11,8 +11,8 @@
import sharp from 'sharp';
import fs from 'fs';
import { commandOptions } from 'redis';
import RedisCanvas from '../data/models/RedisCanvas';
import Palette from './Palette';
import { getMaxTiledZoom } from './utils';
import { TILE_SIZE, TILE_ZOOM_LEVEL } from './constants';
@ -29,7 +29,7 @@ function deleteSubtilefromTile(
palette,
subtilesInTile,
cell,
buffer: Uint8Array,
buffer,
) {
const [dx, dy] = cell;
const offset = (dx + dy * TILE_SIZE * subtilesInTile) * TILE_SIZE;
@ -56,8 +56,8 @@ function deleteSubtilefromTile(
function addRGBSubtiletoTile(
subtilesInTile,
cell,
subtile: Buffer,
buffer: Uint8Array,
subtile,
buffer,
) {
const [dx, dy] = cell;
const chunkOffset = (dx + dy * subtilesInTile * TILE_SIZE) * TILE_SIZE;
@ -84,21 +84,32 @@ function addIndexedSubtiletoTile(
palette,
subtilesInTile,
cell,
subtile: Buffer,
buffer: Uint8Array,
subtile,
buffer,
) {
const [dx, dy] = cell;
const chunkOffset = (dx + dy * subtilesInTile * TILE_SIZE) * TILE_SIZE;
const emptyR = palette.rgb[0];
const emptyB = palette.rgb[1];
const emptyG = palette.rgb[1];
let pos = 0;
let clr;
for (let row = 0; row < TILE_SIZE; row += 1) {
let channelOffset = (chunkOffset + row * TILE_SIZE * subtilesInTile) * 3;
const max = channelOffset + TILE_SIZE * 3;
while (channelOffset < max) {
clr = (subtile[pos++] & 0x3F) * 3;
buffer[channelOffset++] = palette.rgb[clr++];
buffer[channelOffset++] = palette.rgb[clr++];
buffer[channelOffset++] = palette.rgb[clr];
if (pos < subtile.length) {
clr = (subtile[pos++] & 0x3F) * 3;
buffer[channelOffset++] = palette.rgb[clr++];
buffer[channelOffset++] = palette.rgb[clr++];
buffer[channelOffset++] = palette.rgb[clr];
} else {
buffer[channelOffset++] = emptyR;
buffer[channelOffset++] = emptyB;
buffer[channelOffset++] = emptyG;
}
}
}
}
@ -115,7 +126,6 @@ function tileFileName(canvasTileFolder, cell) {
}
/*
* @param redisClient redis instance
* @param canvasId id of the canvas
* @param canvas canvas data
* @param canvasTileFolder root folder where to save tiles
@ -123,7 +133,6 @@ function tileFileName(canvasTileFolder, cell) {
* @return true if successfully created tile, false if tile empty
*/
export async function createZoomTileFromChunk(
redisClient,
canvasId,
canvas,
canvasTileFolder,
@ -141,14 +150,22 @@ export async function createZoomTileFromChunk(
const xabs = x * TILE_ZOOM_LEVEL;
const yabs = y * TILE_ZOOM_LEVEL;
const na = [];
let chunk = null;
for (let dy = 0; dy < TILE_ZOOM_LEVEL; dy += 1) {
for (let dx = 0; dx < TILE_ZOOM_LEVEL; dx += 1) {
chunk = await redisClient.get(
commandOptions({ returnBuffers: true }),
`ch:${canvasId}:${xabs + dx}:${yabs + dy}`,
);
if (!chunk || chunk.length !== TILE_SIZE * TILE_SIZE) {
let chunk = null;
try {
chunk = await RedisCanvas.getChunk(
canvasId,
xabs + dx,
yabs + dy,
);
} catch (error) {
console.error(
// eslint-disable-next-line max-len
`Tiling: Failed to get Chunk ch:${canvasId}:${xabs + dx}${yabs + dy} with error ${error.message}`,
);
}
if (!chunk || !chunk.length) {
na.push([dx, dy]);
continue;
}
@ -161,7 +178,6 @@ export async function createZoomTileFromChunk(
);
}
}
chunk = null;
if (na.length !== TILE_ZOOM_LEVEL * TILE_ZOOM_LEVEL) {
na.forEach((element) => {
@ -311,14 +327,12 @@ async function createEmptyTile(
/*
* created 4096x4096 texture of default canvas
* @param redisClient redis instance
* @param canvasId numberical Id of canvas
* @param canvas canvas data
* @param canvasTileFolder root folder where to save texture
*
*/
export async function createTexture(
redisClient,
canvasId,
canvas,
canvasTileFolder,
@ -333,10 +347,10 @@ export async function createTexture(
const startTime = Date.now();
const na = [];
let chunk = null;
if (targetSize !== canvasSize) {
for (let dy = 0; dy < amount; dy += 1) {
for (let dx = 0; dx < amount; dx += 1) {
let chunk = null;
const chunkfile = `${canvasTileFolder}/${zoom}/${dx}/${dy}.png`;
if (!fs.existsSync(chunkfile)) {
na.push([dx, dy]);
@ -356,11 +370,20 @@ export async function createTexture(
} else {
for (let dy = 0; dy < amount; dy += 1) {
for (let dx = 0; dx < amount; dx += 1) {
chunk = await redisClient.get(
commandOptions({ returnBuffers: true }),
`ch:${canvasId}:${dx}:${dy}`,
);
if (!chunk || chunk.length !== TILE_SIZE * TILE_SIZE) {
let chunk = null;
try {
chunk = await RedisCanvas.getChunk(
canvasId,
dx,
dy,
);
} catch (error) {
console.error(
// eslint-disable-next-line max-len
`Tiling: Failed to get Chunk ch:${canvasId}:${dx}${dy} with error ${error.message}`,
);
}
if (!chunk || !chunk.length) {
na.push([dx, dy]);
continue;
}
@ -374,7 +397,6 @@ export async function createTexture(
}
}
}
chunk = null;
na.forEach((element) => {
deleteSubtilefromTile(palette, amount, element, textureBuffer);
@ -404,14 +426,12 @@ export async function createTexture(
/*
* Create all tiles
* @param redisClient redis instance
* @param canvasId id of the canvas
* @param canvas canvas data
* @param canvasTileFolder folder for tiles
* @param force overwrite existing tiles
*/
export async function initializeTiles(
redisClient,
canvasId,
canvas,
canvasTileFolder,
@ -441,7 +461,6 @@ export async function initializeTiles(
const filename = `${canvasTileFolder}/${zoom}/${cx}/${cy}.png`;
if (force || !fs.existsSync(filename)) {
const ret = await createZoomTileFromChunk(
redisClient,
canvasSize,
canvasId,
canvasTileFolder,
@ -487,7 +506,6 @@ export async function initializeTiles(
}
// create snapshot texture
await createTexture(
redisClient,
canvasId,
canvas,
canvasTileFolder,

2
src/core/draw.js
View File

@ -315,7 +315,7 @@ export async function drawByCoords(
}
const isAdmin = (user.userlvl === 1);
const setColor = await RedisCanvas.getPixel(canvasId, x, y, z);
const setColor = await RedisCanvas.getPixel(canvasId, canvas.size, x, y, z);
/*
* bitwise operation to get rid of protection

19
src/core/rollback.js
View File

@ -51,19 +51,26 @@ export default async function rollbackToDate(
let totalPxlCnt = 0;
logger.info(`Loading to chunks from ${ucx} / ${ucy} to ${lcx} / ${lcy} ...`);
let chunk;
let empty = false;
let emptyBackup = false;
let backupChunk;
for (let cx = ucx; cx <= lcx; cx += 1) {
for (let cy = ucy; cy <= lcy; cy += 1) {
chunk = await RedisCanvas.getChunk(canvasId, cx, cy);
if (chunk && chunk.length === TILE_SIZE * TILE_SIZE) {
chunk = new Uint8Array(chunk);
empty = false;
} else {
let chunk = null;
try {
chunk = await RedisCanvas.getChunk(canvasId, cx, cy, TILE_SIZE ** 2);
} catch (error) {
logger.error(
// eslint-disable-next-line max-len
`Chunk ch:${canvasId}:${cx}:${cy} could not be loaded from redis, assuming empty.`,
);
}
if (!chunk || !chunk.length) {
chunk = new Uint8Array(TILE_SIZE * TILE_SIZE);
empty = true;
} else {
chunk = new Uint8Array(chunk);
empty = false;
}
try {
emptyBackup = false;

253
src/core/tilesBackup.js
View File

@ -13,6 +13,22 @@ import Palette from './Palette';
import { TILE_SIZE } from './constants';
/*
* take chunk buffer and pad it to a specific length
* Fill missing pixels with zeros
* @param length target length
*/
function padChunk(chunk, length) {
let retChunk = chunk;
if (!chunk || !chunk.length) {
retChunk = Buffer.alloc(length);
} else if (chunk.length < length) {
const padding = Buffer.alloc(length - chunk.length);
retChunk = Buffer.concat([chunk, padding]);
}
return retChunk;
}
/*
* Copy canvases from one redis instance to another
* @param canvasRedis redis from where to get the data
@ -35,19 +51,31 @@ export async function updateBackupRedis(canvasRedis, backupRedis, canvases) {
for (let x = 0; x < chunksXY; x++) {
for (let y = 0; y < chunksXY; y++) {
const key = `ch:${id}:${x}:${y}`;
/*
* await on every iteration is fine because less resource usage
* in exchange for higher execution time is wanted.
*/
// eslint-disable-next-line no-await-in-loop
const chunk = await canvasRedis.get(
commandOptions({ returnBuffers: true }),
key,
);
if (chunk) {
let chunk = null;
try {
// eslint-disable-next-line no-await-in-loop
await backupRedis.set(key, chunk);
amount += 1;
chunk = await canvasRedis.get(
commandOptions({ returnBuffers: true }),
key,
);
} catch (error) {
console.error(
// eslint-disable-next-line max-len
new Error(`Could not get chunk ${key} from redis: ${error.message}`),
);
}
if (chunk) {
try {
// eslint-disable-next-line no-await-in-loop
await backupRedis.set(key, chunk);
amount += 1;
} catch (error) {
console.error(
// eslint-disable-next-line max-len
new Error(`Could not create chunk ${key} in backup-redis: ${error.message}`),
);
}
}
}
}
@ -105,62 +133,106 @@ export async function incrementialBackupRedis(
for (let y = 0; y < chunksXY; y++) {
const key = `ch:${id}:${x}:${y}`;
/*
* await on every iteration is fine because less resource usage
* in exchange for higher execution time is wanted.
*/
// eslint-disable-next-line no-await-in-loop
const curChunk = await canvasRedis.get(
commandOptions({ returnBuffers: true }),
key,
);
let curChunk = null;
try {
// eslint-disable-next-line no-await-in-loop
curChunk = await canvasRedis.get(
commandOptions({ returnBuffers: true }),
key,
);
} catch (error) {
console.error(
// eslint-disable-next-line max-len
new Error(`Could not get chunk ${key} from redis: ${error.message}`),
);
}
let oldChunk = null;
try {
// eslint-disable-next-line no-await-in-loop
oldChunk = await backupRedis.get(
commandOptions({ returnBuffers: true }),
key,
);
} catch (error) {
console.error(
// eslint-disable-next-line max-len
new Error(`Could not get chunk ${key} from backup-redis: ${error.message}`),
);
}
// is gonna be an Uint32Array
let tileBuffer = null;
if (curChunk) {
if (curChunk.length === TILE_SIZE * TILE_SIZE) {
// eslint-disable-next-line no-await-in-loop
const oldChunk = await backupRedis.get(
commandOptions({ returnBuffers: true }),
key,
);
if (oldChunk && oldChunk.length === TILE_SIZE * TILE_SIZE) {
let pxl = 0;
while (pxl < curChunk.length) {
if (curChunk[pxl] !== oldChunk[pxl]) {
if (!tileBuffer) {
tileBuffer = new Uint32Array(TILE_SIZE * TILE_SIZE);
}
const color = palette.abgr[curChunk[pxl] & 0x3F];
tileBuffer[pxl] = color;
}
pxl += 1;
try {
if (!oldChunk && !curChunk) {
continue;
}
if (oldChunk && !curChunk) {
// one does not exist
curChunk = Buffer.alloc(TILE_SIZE * TILE_SIZE);
tileBuffer = palette.buffer2ABGR(curChunk);
} else if (!oldChunk && curChunk) {
tileBuffer = new Uint32Array(TILE_SIZE * TILE_SIZE);
const pxl = 0;
while (pxl < curChunk.length) {
const clrIndex = curChunk[pxl] & 0x3F;
if (clrIndex > 0) {
const color = palette.abgr[clrIndex];
tileBuffer[pxl] = color;
}
} else {
tileBuffer = palette.buffer2ABGR(curChunk);
}
} else {
console.log(
// eslint-disable-next-line max-len
`Chunk ${key} in backup-redis has invalid length ${curChunk.length}`,
);
if (curChunk.length < oldChunk.length) {
curChunk = padChunk(curChunk, oldChunk.length);
} else if (curChunk.length > oldChunk.length) {
oldChunk = padChunk(oldChunk, curChunk.length);
}
// both exist and are the same length
tileBuffer = new Uint32Array(TILE_SIZE * TILE_SIZE);
let pxl = 0;
while (pxl < curChunk.length) {
if (curChunk[pxl] !== oldChunk[pxl]) {
const color = palette.abgr[curChunk[pxl] & 0x3F];
tileBuffer[pxl] = color;
}
pxl += 1;
}
}
} catch (error) {
console.error(
// eslint-disable-next-line max-len
new Error(`Could not populate incremential backup data of chunk ${key}: ${error.message}`),
);
continue;
}
if (tileBuffer) {
if (!createdDir && !fs.existsSync(xBackupDir)) {
createdDir = true;
fs.mkdirSync(xBackupDir);
}
const filename = `${xBackupDir}/${y}.png`;
// eslint-disable-next-line no-await-in-loop
await sharp(
Buffer.from(tileBuffer.buffer), {
raw: {
width: TILE_SIZE,
height: TILE_SIZE,
channels: 4,
try {
if (!createdDir && !fs.existsSync(xBackupDir)) {
createdDir = true;
fs.mkdirSync(xBackupDir);
}
const filename = `${xBackupDir}/${y}.png`;
// eslint-disable-next-line no-await-in-loop
await sharp(
Buffer.from(tileBuffer.buffer), {
raw: {
width: TILE_SIZE,
height: TILE_SIZE,
channels: 4,
},
},
},
).toFile(filename);
amount += 1;
).toFile(filename);
amount += 1;
} catch (error) {
console.error(
// eslint-disable-next-line max-len
new Error(`Could not save incremential backup of chunk ${key}: ${error.message}`),
);
continue;
}
}
}
}
@ -209,43 +281,44 @@ export async function createPngBackup(
}
for (let y = 0; y < chunksXY; y++) {
const key = `ch:${id}:${x}:${y}`;
let chunk = null;
try {
/*
* await on every iteration is fine because less resource usage
* in exchange for higher execution time is wanted.
*/
// eslint-disable-next-line no-await-in-loop
const chunk = await redisClient.get(
chunk = await redisClient.get(
commandOptions({ returnBuffers: true }),
key,
);
if (chunk) {
if (chunk.length === TILE_SIZE * TILE_SIZE) {
const textureBuffer = palette.buffer2RGB(chunk);
const filename = `${xBackupDir}/${y}.png`;
// eslint-disable-next-line no-await-in-loop
await sharp(
Buffer.from(textureBuffer.buffer), {
raw: {
width: TILE_SIZE,
height: TILE_SIZE,
channels: 3,
},
},
).toFile(filename);
amount += 1;
} else {
console.log(
// eslint-disable-next-line max-len
`Chunk ${key} has invalid length ${chunk.length}`,
);
}
}
} catch {
console.log(
`Couldn't create PNG backup of chunk ${x},${y} of canvas ${id}.`,
} catch (error) {
console.error(
// eslint-disable-next-line max-len
new Error(`Could not get chunk ${key} from redis: ${error.message}`),
);
}
if (chunk && chunk.length) {
chunk = padChunk(chunk, TILE_SIZE * TILE_SIZE);
const textureBuffer = palette.buffer2RGB(chunk);
const filename = `${xBackupDir}/${y}.png`;
try {
// eslint-disable-next-line no-await-in-loop
await sharp(
Buffer.from(textureBuffer.buffer), {
raw: {
width: TILE_SIZE,
height: TILE_SIZE,
channels: 3,
},
},
).toFile(filename);
} catch (error) {
console.error(
// eslint-disable-next-line max-len
new Error(`Could not save daily backup of chunk ${key}: ${error.message}`),
);
continue;
}
amount += 1;
}
}
}
const time = Date.now() - startTime;

71
src/data/models/Event.js
View File

@ -68,32 +68,38 @@ export async function clearOldEvent() {
// 3x3 chunk area centered at i,j
for (let jc = j - 1; jc <= j + 1; jc += 1) {
for (let ic = i - 1; ic <= i + 1; ic += 1) {
const chunkKey = `${EVENT_BACKUP_PREFIX}:${ic}:${jc}`;
const chunk = await redis.get(
commandOptions({ returnBuffers: true }),
chunkKey,
);
if (!chunk) {
logger.warn(
try {
const chunkKey = `${EVENT_BACKUP_PREFIX}:${ic}:${jc}`;
const chunk = await redis.get(
commandOptions({ returnBuffers: true }),
chunkKey,
);
if (!chunk) {
logger.warn(
// eslint-disable-next-line max-len
`Couldn't get chunk event backup for ${ic}/${jc}, which is weird`,
);
continue;
}
if (chunk.length <= 1) {
logger.info(
// eslint-disable-next-line max-len
`Tiny chunk in event backup, not-generated chunk at ${ic}/${jc}`,
);
await RedisCanvas.delChunk(ic, jc, CANVAS_ID);
} else {
logger.info(
`Restoring chunk ${ic}/${jc} from event`,
);
await RedisCanvas.setChunk(ic, jc, chunk, CANVAS_ID);
}
await redis.del(chunkKey);
} catch (error) {
logger.error(
// eslint-disable-next-line max-len
`Couldn't get chunk event backup for ${ic}/${jc}, which is weird`,
`Couldn't restore chunk from RpgEvent ${EVENT_BACKUP_PREFIX}:${ic}:${jc} : ${error.message}`,
);
continue;
}
if (chunk.length <= 256) {
logger.info(
// eslint-disable-next-line max-len
`Tiny chunk in event backup, not-generated chunk at ${ic}/${jc}`,
);
await RedisCanvas.delChunk(ic, jc, CANVAS_ID);
} else {
logger.info(
`Restoring chunk ${ic}/${jc} from event`,
);
const chunkArray = new Uint8Array(chunk);
await RedisCanvas.setChunk(ic, jc, chunkArray, CANVAS_ID);
}
await redis.del(chunkKey);
}
}
await redis.del(EVENT_POSITION_KEY);
@ -109,13 +115,18 @@ export async function setNextEvent(minutes, i, j) {
await clearOldEvent();
for (let jc = j - 1; jc <= j + 1; jc += 1) {
for (let ic = i - 1; ic <= i + 1; ic += 1) {
let chunk = await RedisCanvas.getChunk(CANVAS_ID, ic, jc);
if (!chunk) {
// place a dummy Array inside to mark chunk as none-existent
const buff = new Uint8Array(3);
chunk = Buffer.from(buff);
// place dummy pixel to make RedisCanvas create chunk
await RedisCanvas.setPixelInChunk(ic, jc, 0, 0, CANVAS_ID);
let chunk = null;
try {
chunk = await RedisCanvas.getChunk(CANVAS_ID, ic, jc);
} catch (error) {
logger.error(
// eslint-disable-next-line max-len
`Could not load chunk ch:${CANVAS_ID}:${ic}:${jc} for RpgEvent backup: ${error.message}`,
);
}
if (!chunk || !chunk.length) {
// place a 1-length buffer inside to mark chunk as none-existent
chunk = Buffer.allocUnsafe(1);
}
const chunkKey = `${EVENT_BACKUP_PREFIX}:${ic}:${jc}`;
await redis.set(chunkKey, chunk);

98
src/data/models/RedisCanvas.js
View File

@ -4,30 +4,11 @@
import { commandOptions } from 'redis';
import { getChunkOfPixel, getOffsetOfPixel } from '../../core/utils';
import {
TILE_SIZE,
THREE_TILE_SIZE,
THREE_CANVAS_HEIGHT,
} from '../../core/constants';
// eslint-disable-next-line import/no-unresolved
import canvases from './canvases.json';
import redis from '../redis';
const UINT_SIZE = 'u8';
const EMPTY_CACA = new Uint8Array(TILE_SIZE * TILE_SIZE);
const EMPTY_CHUNK_BUFFER = Buffer.from(EMPTY_CACA.buffer);
const THREE_EMPTY_CACA = new Uint8Array(
THREE_TILE_SIZE * THREE_TILE_SIZE * THREE_CANVAS_HEIGHT,
);
const THREE_EMPTY_CHUNK_BUFFER = Buffer.from(THREE_EMPTY_CACA.buffer);
// cache existence of chunks
const chunks = new Set();
class RedisCanvas {
// array of callback functions that gets informed about chunk changes
static registerChunkChange = [];
@ -41,29 +22,34 @@ class RedisCanvas {
}
}
static getChunk(
/*
* Get chunk from redis
* canvasId integer id of canvas
* i, j chunk coordinates
* [padding] required length of chunk (will be padded with zeros if smaller)
* @return chunk as Buffer
*/
static async getChunk(
canvasId,
i,
j,
padding = null,
) {
// this key is also hardcoded into
// core/tilesBackup.js
// and ./EventData.js
const key = `ch:${canvasId}:${i}:${j}`;
return redis.get(
let chunk = await redis.get(
commandOptions({ returnBuffers: true }),
key,
);
if (padding > 0 && chunk && chunk.length < padding) {
const pad = Buffer.alloc(padding - chunk.length);
chunk = Buffer.concat([chunk, pad]);
}
return chunk;
}
static async setChunk(i, j, chunk, canvasId) {
if (chunk.length !== TILE_SIZE * TILE_SIZE) {
// eslint-disable-next-line no-console
console.error(
new Error(`Tried to set chunk with invalid length ${chunk.length}!`),
);
return false;
}
const key = `ch:${canvasId}:${i}:${j}`;
await redis.set(key, Buffer.from(chunk.buffer));
RedisCanvas.execChunkChangeCallback(canvasId, [i, j]);
@ -73,24 +59,10 @@ class RedisCanvas {
static async delChunk(i, j, canvasId) {
const key = `ch:${canvasId}:${i}:${j}`;
await redis.del(key);
chunks.delete(key);
RedisCanvas.execChunkChangeCallback(canvasId, [i, j]);
return true;
}
static async setPixel(
canvasId,
color,
x,
y,
z = null,
) {
const canvasSize = canvases[canvasId].size;
const [i, j] = getChunkOfPixel(canvasSize, x, y, z);
const offset = getOffsetOfPixel(canvasSize, x, y, z);
RedisCanvas.setPixelInChunk(i, j, offset, color, canvasId);
}
multi = null;
static enqueuePixel(
@ -100,10 +72,20 @@ class RedisCanvas {
j,
offset,
) {
/*
* TODO what if chunk does not exist?
*/
if (!RedisCanvas.multi) {
RedisCanvas.multi = redis.multi();
setTimeout(RedisCanvas.flushPixels, 100);
}
RedisCanvas.multi.addCommand(
/*
* NOTE:
* If chunk doesn't exist or is smaller than the offset,
* redis will pad with zeros
* https://redis.io/commands/bitfield/
*/
[
'BITFIELD',
`ch:${canvasId}:${i}:${j}`,
@ -126,33 +108,6 @@ class RedisCanvas {
return null;
}
static async setPixelInChunk(
i,
j,
offset,
color,
canvasId,
) {
const key = `ch:${canvasId}:${i}:${j}`;
if (!chunks.has(key)) {
if (canvases[canvasId].v) {
await redis.set(key, THREE_EMPTY_CHUNK_BUFFER, {
NX: true,
});
} else {
await redis.set(key, EMPTY_CHUNK_BUFFER, {
NX: true,
});
}
chunks.add(key);
}
const args = ['BITFIELD', key, 'SET', UINT_SIZE, `#${offset}`, color];
await redis.sendCommand(args);
RedisCanvas.execChunkChangeCallback(canvasId, [i, j]);
}
static async getPixelIfExists(
canvasId,
i,
@ -187,11 +142,11 @@ class RedisCanvas {
static async getPixel(
canvasId,
canvasSize,
x,
y,
z = null,
) {
const canvasSize = canvases[canvasId].size;
const [i, j] = getChunkOfPixel(canvasSize, x, y, z);
const offset = getOffsetOfPixel(canvasSize, x, y, z);
@ -200,6 +155,5 @@ class RedisCanvas {
}
}
setInterval(RedisCanvas.flushPixels, 100);
export default RedisCanvas;

12
src/routes/chunks.js
View File

@ -6,11 +6,6 @@
import etag from 'etag';
import RedisCanvas from '../data/models/RedisCanvas';
import {
TILE_SIZE,
THREE_TILE_SIZE,
THREE_CANVAS_HEIGHT,
} from '../core/constants';
import logger from '../core/logger';
const chunkEtags = new Map();
@ -79,13 +74,6 @@ export default async (req, res, next) => {
return;
}
// for temporary logging to see if we have invalid chunks in redis
if (chunk.length !== TILE_SIZE * TILE_SIZE
&& chunk.length !== (THREE_TILE_SIZE ** 2) * THREE_CANVAS_HEIGHT) {
logger.error(`Chunk ${x},${y}/${c} has invalid length ${chunk.length}!`);
}
curEtag = etag(chunk, { weak: true });
res.set({
ETag: curEtag,

10
src/ui/ChunkRGB3D.js
View File

@ -238,7 +238,15 @@ class Chunk {
this.setVoxelByOffset(offset, clr);
}
async fromBuffer(chunkBuffer: Uint8Array) {
async fromBuffer(chunkBufferInpt: Uint8Array) {
let chunkBuffer = chunkBufferInpt;
const neededLength = THREE_TILE_SIZE ** 2 * THREE_CANVAS_HEIGHT;
if (chunkBuffer.byteLength < neededLength) {
// eslint-disable-next-line
console.log(`Padding chunk ${this.key} with ${neededLength - chunkBuffer.byteLength} voxels to full length`);
chunkBuffer = new Uint8Array(neededLength);
chunkBuffer.set(chunkBufferInpt);
}
this.buffer = chunkBuffer;
const [faceCnt, lastPixel, heightMap] = Chunk.calculateMetaData(
chunkBuffer,

8
src/workers/tilewriter.js
View File

@ -6,7 +6,7 @@
import { isMainThread, parentPort } from 'worker_threads';
import redis, { connect as connectRedis } from '../data/redis';
import { connect as connectRedis } from '../data/redis';
import {
createZoomTileFromChunk,
createZoomedTile,
@ -27,16 +27,16 @@ connectRedis()
try {
switch (task) {
case 'createZoomTileFromChunk':
createZoomTileFromChunk(redis, ...args);
createZoomTileFromChunk(...args);
break;
case 'createZoomedTile':
createZoomedTile(...args);
break;
case 'createTexture':
createTexture(redis, ...args);
createTexture(...args);
break;
case 'initializeTiles':
await initializeTiles(redis, ...args);
await initializeTiles(...args);
parentPort.postMessage('Done!');
break;
default: