diff --git a/src/core/CanvasCleaner.js b/src/core/CanvasCleaner.js index e9be10b..2b41e7e 100644 --- a/src/core/CanvasCleaner.js +++ b/src/core/CanvasCleaner.js @@ -373,13 +373,27 @@ class CanvasCleaner { ) { chunk = chunks[jAbs - jo + 1][iAbs - io + 1]; } else { - // eslint-disable-next-line no-await-in-loop - chunk = await RedisCanvas.getChunk(canvasId, iAbs, jAbs); - if (!chunk || chunk.length !== TILE_SIZE * TILE_SIZE) { + try { + // eslint-disable-next-line no-await-in-loop + chunk = await RedisCanvas.getChunk( + canvasId, + iAbs, + jAbs, + TILE_SIZE ** 2, + ); + } catch (error) { + this.logger( + // eslint-disable-next-line max-len + `Couldn't load chunk ch:${canvasId}:${iAbs}:${jAbs}: ${error.message}}`, + ); + } + if (!chunk || !chunk.length) { chunk = null; if (chunk) { - // eslint-disable-next-line max-len - this.logger(`Chunk ch:${canvasId}:${iAbs}:${jAbs} has invalid size ${chunk.length}.`); + this.logger( + // eslint-disable-next-line max-len + `Chunk ch:${canvasId}:${iAbs}:${jAbs} has invalid size ${chunk.length}.`, + ); } } } diff --git a/src/core/Image.js b/src/core/Image.js index e2b4e94..8d4eef9 100644 --- a/src/core/Image.js +++ b/src/core/Image.js @@ -39,6 +39,7 @@ export async function imageABGR2Canvas( logger.info( `Loading image with dim ${width}/${height} to ${x}/${y}/${canvasId}`, ); + const expectedLength = TILE_SIZE ** 2; const canvas = canvases[canvasId]; const { colors, cli, size } = canvas; const palette = new Palette(colors); @@ -50,11 +51,18 @@ export async function imageABGR2Canvas( let totalPxlCnt = 0; logger.info(`Loading to chunks from ${ucx} / ${ucy} to ${lcx} / ${lcy} ...`); - let chunk; for (let cx = ucx; cx <= lcx; cx += 1) { for (let cy = ucy; cy <= lcy; cy += 1) { - chunk = await RedisCanvas.getChunk(canvasId, cx, cy); - chunk = (chunk && chunk.length === TILE_SIZE * TILE_SIZE) + let chunk = null; + try { + chunk = await RedisCanvas.getChunk(canvasId, cx, cy, expectedLength); + } catch (error) { + logger.error( + // eslint-disable-next-line max-len + `Could not load chunk ch:${canvasId}:${cx}:${cy} for image-load: ${error.message}`, + ); + } + chunk = (chunk && chunk.length) ? new Uint8Array(chunk) : new Uint8Array(TILE_SIZE * TILE_SIZE); // offset of chunk in image @@ -119,6 +127,7 @@ export async function imagemask2Canvas( logger.info( `Loading mask with size ${width} / ${height} to ${x} / ${y} to the canvas`, ); + const expectedLength = TILE_SIZE ** 2; const canvas = canvases[canvasId]; const palette = new Palette(canvas.colors); const canvasMinXY = -(canvas.size / 2); @@ -129,11 +138,18 @@ export async function imagemask2Canvas( const [lcx, lcy] = getChunkOfPixel(canvas.size, x + width, y + height); logger.info(`Loading to chunks from ${ucx} / ${ucy} to ${lcx} / ${lcy} ...`); - let chunk; for (let cx = ucx; cx <= lcx; cx += 1) { for (let cy = ucy; cy <= lcy; cy += 1) { - chunk = await RedisCanvas.getChunk(canvasId, cx, cy); - chunk = (chunk && chunk.length === TILE_SIZE * TILE_SIZE) + let chunk = null; + try { + chunk = await RedisCanvas.getChunk(canvasId, cx, cy, expectedLength); + } catch (error) { + logger.error( + // eslint-disable-next-line max-len + `Could not load chunk ch:${canvasId}:${cx}:${cy} for imagemask-load: ${error.message}`, + ); + } + chunk = (chunk && chunk.length) ? new Uint8Array(chunk) : new Uint8Array(TILE_SIZE * TILE_SIZE); // offset of chunk in image @@ -191,6 +207,7 @@ export async function protectCanvasArea( // eslint-disable-next-line max-len `Setting protection ${protect} with size ${width} / ${height} to ${x} / ${y}`, ); + const expectedLength = TILE_SIZE ** 2; const canvas = canvases[canvasId]; const canvasMinXY = -(canvas.size / 2); @@ -201,11 +218,18 @@ export async function protectCanvasArea( ); let totalPxlCnt = 0; - let chunk; for (let cx = ucx; cx <= lcx; cx += 1) { for (let cy = ucy; cy <= lcy; cy += 1) { - chunk = await RedisCanvas.getChunk(canvasId, cx, cy); - if (!chunk || chunk.length !== TILE_SIZE * TILE_SIZE) { + let chunk = null; + try { + chunk = await RedisCanvas.getChunk(canvasId, cx, cy, expectedLength); + } catch (error) { + logger.error( + // eslint-disable-next-line max-len + `Could not load chunk ch:${canvasId}:${cx}:${cy} for protection: ${error.message}`, + ); + } + if (!chunk || !chunk.length) { continue; } chunk = new Uint8Array(chunk); diff --git a/src/core/Palette.js b/src/core/Palette.js index abf5197..f3c5ada 100644 --- a/src/core/Palette.js +++ b/src/core/Palette.js @@ -110,11 +110,10 @@ class Palette { const { length } = chunkBuffer; const colors = new Uint32Array(length); let value; - const buffer = chunkBuffer; let pos = 0; for (let i = 0; i < length; i++) { - value = (buffer[i] & 0x3F); + value = (chunkBuffer[i] & 0x3F); colors[pos++] = this.abgr[value]; } return colors; diff --git a/src/core/Tile.js b/src/core/Tile.js index 48f15de..2a83650 100644 --- a/src/core/Tile.js +++ b/src/core/Tile.js @@ -11,8 +11,8 @@ import sharp from 'sharp'; import fs from 'fs'; -import { commandOptions } from 'redis'; +import RedisCanvas from '../data/models/RedisCanvas'; import Palette from './Palette'; import { getMaxTiledZoom } from './utils'; import { TILE_SIZE, TILE_ZOOM_LEVEL } from './constants'; @@ -29,7 +29,7 @@ function deleteSubtilefromTile( palette, subtilesInTile, cell, - buffer: Uint8Array, + buffer, ) { const [dx, dy] = cell; const offset = (dx + dy * TILE_SIZE * subtilesInTile) * TILE_SIZE; @@ -56,8 +56,8 @@ function deleteSubtilefromTile( function addRGBSubtiletoTile( subtilesInTile, cell, - subtile: Buffer, - buffer: Uint8Array, + subtile, + buffer, ) { const [dx, dy] = cell; const chunkOffset = (dx + dy * subtilesInTile * TILE_SIZE) * TILE_SIZE; @@ -84,21 +84,32 @@ function addIndexedSubtiletoTile( palette, subtilesInTile, cell, - subtile: Buffer, - buffer: Uint8Array, + subtile, + buffer, ) { const [dx, dy] = cell; const chunkOffset = (dx + dy * subtilesInTile * TILE_SIZE) * TILE_SIZE; + + const emptyR = palette.rgb[0]; + const emptyB = palette.rgb[1]; + const emptyG = palette.rgb[1]; + let pos = 0; let clr; for (let row = 0; row < TILE_SIZE; row += 1) { let channelOffset = (chunkOffset + row * TILE_SIZE * subtilesInTile) * 3; const max = channelOffset + TILE_SIZE * 3; while (channelOffset < max) { - clr = (subtile[pos++] & 0x3F) * 3; - buffer[channelOffset++] = palette.rgb[clr++]; - buffer[channelOffset++] = palette.rgb[clr++]; - buffer[channelOffset++] = palette.rgb[clr]; + if (pos < subtile.length) { + clr = (subtile[pos++] & 0x3F) * 3; + buffer[channelOffset++] = palette.rgb[clr++]; + buffer[channelOffset++] = palette.rgb[clr++]; + buffer[channelOffset++] = palette.rgb[clr]; + } else { + buffer[channelOffset++] = emptyR; + buffer[channelOffset++] = emptyB; + buffer[channelOffset++] = emptyG; + } } } } @@ -115,7 +126,6 @@ function tileFileName(canvasTileFolder, cell) { } /* - * @param redisClient redis instance * @param canvasId id of the canvas * @param canvas canvas data * @param canvasTileFolder root folder where to save tiles @@ -123,7 +133,6 @@ function tileFileName(canvasTileFolder, cell) { * @return true if successfully created tile, false if tile empty */ export async function createZoomTileFromChunk( - redisClient, canvasId, canvas, canvasTileFolder, @@ -141,14 +150,22 @@ export async function createZoomTileFromChunk( const xabs = x * TILE_ZOOM_LEVEL; const yabs = y * TILE_ZOOM_LEVEL; const na = []; - let chunk = null; for (let dy = 0; dy < TILE_ZOOM_LEVEL; dy += 1) { for (let dx = 0; dx < TILE_ZOOM_LEVEL; dx += 1) { - chunk = await redisClient.get( - commandOptions({ returnBuffers: true }), - `ch:${canvasId}:${xabs + dx}:${yabs + dy}`, - ); - if (!chunk || chunk.length !== TILE_SIZE * TILE_SIZE) { + let chunk = null; + try { + chunk = await RedisCanvas.getChunk( + canvasId, + xabs + dx, + yabs + dy, + ); + } catch (error) { + console.error( + // eslint-disable-next-line max-len + `Tiling: Failed to get Chunk ch:${canvasId}:${xabs + dx}${yabs + dy} with error ${error.message}`, + ); + } + if (!chunk || !chunk.length) { na.push([dx, dy]); continue; } @@ -161,7 +178,6 @@ export async function createZoomTileFromChunk( ); } } - chunk = null; if (na.length !== TILE_ZOOM_LEVEL * TILE_ZOOM_LEVEL) { na.forEach((element) => { @@ -311,14 +327,12 @@ async function createEmptyTile( /* * created 4096x4096 texture of default canvas - * @param redisClient redis instance * @param canvasId numberical Id of canvas * @param canvas canvas data * @param canvasTileFolder root folder where to save texture * */ export async function createTexture( - redisClient, canvasId, canvas, canvasTileFolder, @@ -333,10 +347,10 @@ export async function createTexture( const startTime = Date.now(); const na = []; - let chunk = null; if (targetSize !== canvasSize) { for (let dy = 0; dy < amount; dy += 1) { for (let dx = 0; dx < amount; dx += 1) { + let chunk = null; const chunkfile = `${canvasTileFolder}/${zoom}/${dx}/${dy}.png`; if (!fs.existsSync(chunkfile)) { na.push([dx, dy]); @@ -356,11 +370,20 @@ export async function createTexture( } else { for (let dy = 0; dy < amount; dy += 1) { for (let dx = 0; dx < amount; dx += 1) { - chunk = await redisClient.get( - commandOptions({ returnBuffers: true }), - `ch:${canvasId}:${dx}:${dy}`, - ); - if (!chunk || chunk.length !== TILE_SIZE * TILE_SIZE) { + let chunk = null; + try { + chunk = await RedisCanvas.getChunk( + canvasId, + dx, + dy, + ); + } catch (error) { + console.error( + // eslint-disable-next-line max-len + `Tiling: Failed to get Chunk ch:${canvasId}:${dx}${dy} with error ${error.message}`, + ); + } + if (!chunk || !chunk.length) { na.push([dx, dy]); continue; } @@ -374,7 +397,6 @@ export async function createTexture( } } } - chunk = null; na.forEach((element) => { deleteSubtilefromTile(palette, amount, element, textureBuffer); @@ -404,14 +426,12 @@ export async function createTexture( /* * Create all tiles - * @param redisClient redis instance * @param canvasId id of the canvas * @param canvas canvas data * @param canvasTileFolder folder for tiles * @param force overwrite existing tiles */ export async function initializeTiles( - redisClient, canvasId, canvas, canvasTileFolder, @@ -441,7 +461,6 @@ export async function initializeTiles( const filename = `${canvasTileFolder}/${zoom}/${cx}/${cy}.png`; if (force || !fs.existsSync(filename)) { const ret = await createZoomTileFromChunk( - redisClient, canvasSize, canvasId, canvasTileFolder, @@ -487,7 +506,6 @@ export async function initializeTiles( } // create snapshot texture await createTexture( - redisClient, canvasId, canvas, canvasTileFolder, diff --git a/src/core/draw.js b/src/core/draw.js index 321fbf9..f058bc8 100644 --- a/src/core/draw.js +++ b/src/core/draw.js @@ -315,7 +315,7 @@ export async function drawByCoords( } const isAdmin = (user.userlvl === 1); - const setColor = await RedisCanvas.getPixel(canvasId, x, y, z); + const setColor = await RedisCanvas.getPixel(canvasId, canvas.size, x, y, z); /* * bitwise operation to get rid of protection diff --git a/src/core/rollback.js b/src/core/rollback.js index 3736d7d..a27d27a 100644 --- a/src/core/rollback.js +++ b/src/core/rollback.js @@ -51,19 +51,26 @@ export default async function rollbackToDate( let totalPxlCnt = 0; logger.info(`Loading to chunks from ${ucx} / ${ucy} to ${lcx} / ${lcy} ...`); - let chunk; let empty = false; let emptyBackup = false; let backupChunk; for (let cx = ucx; cx <= lcx; cx += 1) { for (let cy = ucy; cy <= lcy; cy += 1) { - chunk = await RedisCanvas.getChunk(canvasId, cx, cy); - if (chunk && chunk.length === TILE_SIZE * TILE_SIZE) { - chunk = new Uint8Array(chunk); - empty = false; - } else { + let chunk = null; + try { + chunk = await RedisCanvas.getChunk(canvasId, cx, cy, TILE_SIZE ** 2); + } catch (error) { + logger.error( + // eslint-disable-next-line max-len + `Chunk ch:${canvasId}:${cx}:${cy} could not be loaded from redis, assuming empty.`, + ); + } + if (!chunk || !chunk.length) { chunk = new Uint8Array(TILE_SIZE * TILE_SIZE); empty = true; + } else { + chunk = new Uint8Array(chunk); + empty = false; } try { emptyBackup = false; diff --git a/src/core/tilesBackup.js b/src/core/tilesBackup.js index 915b298..b94325c 100644 --- a/src/core/tilesBackup.js +++ b/src/core/tilesBackup.js @@ -13,6 +13,22 @@ import Palette from './Palette'; import { TILE_SIZE } from './constants'; +/* + * take chunk buffer and pad it to a specific length + * Fill missing pixels with zeros + * @param length target length + */ +function padChunk(chunk, length) { + let retChunk = chunk; + if (!chunk || !chunk.length) { + retChunk = Buffer.alloc(length); + } else if (chunk.length < length) { + const padding = Buffer.alloc(length - chunk.length); + retChunk = Buffer.concat([chunk, padding]); + } + return retChunk; +} + /* * Copy canvases from one redis instance to another * @param canvasRedis redis from where to get the data @@ -35,19 +51,31 @@ export async function updateBackupRedis(canvasRedis, backupRedis, canvases) { for (let x = 0; x < chunksXY; x++) { for (let y = 0; y < chunksXY; y++) { const key = `ch:${id}:${x}:${y}`; - /* - * await on every iteration is fine because less resource usage - * in exchange for higher execution time is wanted. - */ - // eslint-disable-next-line no-await-in-loop - const chunk = await canvasRedis.get( - commandOptions({ returnBuffers: true }), - key, - ); - if (chunk) { + let chunk = null; + + try { // eslint-disable-next-line no-await-in-loop - await backupRedis.set(key, chunk); - amount += 1; + chunk = await canvasRedis.get( + commandOptions({ returnBuffers: true }), + key, + ); + } catch (error) { + console.error( + // eslint-disable-next-line max-len + new Error(`Could not get chunk ${key} from redis: ${error.message}`), + ); + } + if (chunk) { + try { + // eslint-disable-next-line no-await-in-loop + await backupRedis.set(key, chunk); + amount += 1; + } catch (error) { + console.error( + // eslint-disable-next-line max-len + new Error(`Could not create chunk ${key} in backup-redis: ${error.message}`), + ); + } } } } @@ -105,62 +133,106 @@ export async function incrementialBackupRedis( for (let y = 0; y < chunksXY; y++) { const key = `ch:${id}:${x}:${y}`; - /* - * await on every iteration is fine because less resource usage - * in exchange for higher execution time is wanted. - */ - // eslint-disable-next-line no-await-in-loop - const curChunk = await canvasRedis.get( - commandOptions({ returnBuffers: true }), - key, - ); + + let curChunk = null; + try { + // eslint-disable-next-line no-await-in-loop + curChunk = await canvasRedis.get( + commandOptions({ returnBuffers: true }), + key, + ); + } catch (error) { + console.error( + // eslint-disable-next-line max-len + new Error(`Could not get chunk ${key} from redis: ${error.message}`), + ); + } + + let oldChunk = null; + try { + // eslint-disable-next-line no-await-in-loop + oldChunk = await backupRedis.get( + commandOptions({ returnBuffers: true }), + key, + ); + } catch (error) { + console.error( + // eslint-disable-next-line max-len + new Error(`Could not get chunk ${key} from backup-redis: ${error.message}`), + ); + } + + // is gonna be an Uint32Array let tileBuffer = null; - if (curChunk) { - if (curChunk.length === TILE_SIZE * TILE_SIZE) { - // eslint-disable-next-line no-await-in-loop - const oldChunk = await backupRedis.get( - commandOptions({ returnBuffers: true }), - key, - ); - if (oldChunk && oldChunk.length === TILE_SIZE * TILE_SIZE) { - let pxl = 0; - while (pxl < curChunk.length) { - if (curChunk[pxl] !== oldChunk[pxl]) { - if (!tileBuffer) { - tileBuffer = new Uint32Array(TILE_SIZE * TILE_SIZE); - } - const color = palette.abgr[curChunk[pxl] & 0x3F]; - tileBuffer[pxl] = color; - } - pxl += 1; + + try { + if (!oldChunk && !curChunk) { + continue; + } + if (oldChunk && !curChunk) { + // one does not exist + curChunk = Buffer.alloc(TILE_SIZE * TILE_SIZE); + tileBuffer = palette.buffer2ABGR(curChunk); + } else if (!oldChunk && curChunk) { + tileBuffer = new Uint32Array(TILE_SIZE * TILE_SIZE); + const pxl = 0; + while (pxl < curChunk.length) { + const clrIndex = curChunk[pxl] & 0x3F; + if (clrIndex > 0) { + const color = palette.abgr[clrIndex]; + tileBuffer[pxl] = color; } - } else { - tileBuffer = palette.buffer2ABGR(curChunk); } } else { - console.log( - // eslint-disable-next-line max-len - `Chunk ${key} in backup-redis has invalid length ${curChunk.length}`, - ); + if (curChunk.length < oldChunk.length) { + curChunk = padChunk(curChunk, oldChunk.length); + } else if (curChunk.length > oldChunk.length) { + oldChunk = padChunk(oldChunk, curChunk.length); + } + // both exist and are the same length + tileBuffer = new Uint32Array(TILE_SIZE * TILE_SIZE); + let pxl = 0; + while (pxl < curChunk.length) { + if (curChunk[pxl] !== oldChunk[pxl]) { + const color = palette.abgr[curChunk[pxl] & 0x3F]; + tileBuffer[pxl] = color; + } + pxl += 1; + } } + } catch (error) { + console.error( + // eslint-disable-next-line max-len + new Error(`Could not populate incremential backup data of chunk ${key}: ${error.message}`), + ); + continue; } + if (tileBuffer) { - if (!createdDir && !fs.existsSync(xBackupDir)) { - createdDir = true; - fs.mkdirSync(xBackupDir); - } - const filename = `${xBackupDir}/${y}.png`; - // eslint-disable-next-line no-await-in-loop - await sharp( - Buffer.from(tileBuffer.buffer), { - raw: { - width: TILE_SIZE, - height: TILE_SIZE, - channels: 4, + try { + if (!createdDir && !fs.existsSync(xBackupDir)) { + createdDir = true; + fs.mkdirSync(xBackupDir); + } + const filename = `${xBackupDir}/${y}.png`; + // eslint-disable-next-line no-await-in-loop + await sharp( + Buffer.from(tileBuffer.buffer), { + raw: { + width: TILE_SIZE, + height: TILE_SIZE, + channels: 4, + }, }, - }, - ).toFile(filename); - amount += 1; + ).toFile(filename); + amount += 1; + } catch (error) { + console.error( + // eslint-disable-next-line max-len + new Error(`Could not save incremential backup of chunk ${key}: ${error.message}`), + ); + continue; + } } } } @@ -209,43 +281,44 @@ export async function createPngBackup( } for (let y = 0; y < chunksXY; y++) { const key = `ch:${id}:${x}:${y}`; + + let chunk = null; try { - /* - * await on every iteration is fine because less resource usage - * in exchange for higher execution time is wanted. - */ // eslint-disable-next-line no-await-in-loop - const chunk = await redisClient.get( + chunk = await redisClient.get( commandOptions({ returnBuffers: true }), key, ); - if (chunk) { - if (chunk.length === TILE_SIZE * TILE_SIZE) { - const textureBuffer = palette.buffer2RGB(chunk); - const filename = `${xBackupDir}/${y}.png`; - // eslint-disable-next-line no-await-in-loop - await sharp( - Buffer.from(textureBuffer.buffer), { - raw: { - width: TILE_SIZE, - height: TILE_SIZE, - channels: 3, - }, - }, - ).toFile(filename); - amount += 1; - } else { - console.log( - // eslint-disable-next-line max-len - `Chunk ${key} has invalid length ${chunk.length}`, - ); - } - } - } catch { - console.log( - `Couldn't create PNG backup of chunk ${x},${y} of canvas ${id}.`, + } catch (error) { + console.error( + // eslint-disable-next-line max-len + new Error(`Could not get chunk ${key} from redis: ${error.message}`), ); } + if (chunk && chunk.length) { + chunk = padChunk(chunk, TILE_SIZE * TILE_SIZE); + const textureBuffer = palette.buffer2RGB(chunk); + const filename = `${xBackupDir}/${y}.png`; + try { + // eslint-disable-next-line no-await-in-loop + await sharp( + Buffer.from(textureBuffer.buffer), { + raw: { + width: TILE_SIZE, + height: TILE_SIZE, + channels: 3, + }, + }, + ).toFile(filename); + } catch (error) { + console.error( + // eslint-disable-next-line max-len + new Error(`Could not save daily backup of chunk ${key}: ${error.message}`), + ); + continue; + } + amount += 1; + } } } const time = Date.now() - startTime; diff --git a/src/data/models/Event.js b/src/data/models/Event.js index 1e8af22..54edfd3 100644 --- a/src/data/models/Event.js +++ b/src/data/models/Event.js @@ -68,32 +68,38 @@ export async function clearOldEvent() { // 3x3 chunk area centered at i,j for (let jc = j - 1; jc <= j + 1; jc += 1) { for (let ic = i - 1; ic <= i + 1; ic += 1) { - const chunkKey = `${EVENT_BACKUP_PREFIX}:${ic}:${jc}`; - const chunk = await redis.get( - commandOptions({ returnBuffers: true }), - chunkKey, - ); - if (!chunk) { - logger.warn( + try { + const chunkKey = `${EVENT_BACKUP_PREFIX}:${ic}:${jc}`; + const chunk = await redis.get( + commandOptions({ returnBuffers: true }), + chunkKey, + ); + if (!chunk) { + logger.warn( + // eslint-disable-next-line max-len + `Couldn't get chunk event backup for ${ic}/${jc}, which is weird`, + ); + continue; + } + if (chunk.length <= 1) { + logger.info( + // eslint-disable-next-line max-len + `Tiny chunk in event backup, not-generated chunk at ${ic}/${jc}`, + ); + await RedisCanvas.delChunk(ic, jc, CANVAS_ID); + } else { + logger.info( + `Restoring chunk ${ic}/${jc} from event`, + ); + await RedisCanvas.setChunk(ic, jc, chunk, CANVAS_ID); + } + await redis.del(chunkKey); + } catch (error) { + logger.error( // eslint-disable-next-line max-len - `Couldn't get chunk event backup for ${ic}/${jc}, which is weird`, + `Couldn't restore chunk from RpgEvent ${EVENT_BACKUP_PREFIX}:${ic}:${jc} : ${error.message}`, ); - continue; } - if (chunk.length <= 256) { - logger.info( - // eslint-disable-next-line max-len - `Tiny chunk in event backup, not-generated chunk at ${ic}/${jc}`, - ); - await RedisCanvas.delChunk(ic, jc, CANVAS_ID); - } else { - logger.info( - `Restoring chunk ${ic}/${jc} from event`, - ); - const chunkArray = new Uint8Array(chunk); - await RedisCanvas.setChunk(ic, jc, chunkArray, CANVAS_ID); - } - await redis.del(chunkKey); } } await redis.del(EVENT_POSITION_KEY); @@ -109,13 +115,18 @@ export async function setNextEvent(minutes, i, j) { await clearOldEvent(); for (let jc = j - 1; jc <= j + 1; jc += 1) { for (let ic = i - 1; ic <= i + 1; ic += 1) { - let chunk = await RedisCanvas.getChunk(CANVAS_ID, ic, jc); - if (!chunk) { - // place a dummy Array inside to mark chunk as none-existent - const buff = new Uint8Array(3); - chunk = Buffer.from(buff); - // place dummy pixel to make RedisCanvas create chunk - await RedisCanvas.setPixelInChunk(ic, jc, 0, 0, CANVAS_ID); + let chunk = null; + try { + chunk = await RedisCanvas.getChunk(CANVAS_ID, ic, jc); + } catch (error) { + logger.error( + // eslint-disable-next-line max-len + `Could not load chunk ch:${CANVAS_ID}:${ic}:${jc} for RpgEvent backup: ${error.message}`, + ); + } + if (!chunk || !chunk.length) { + // place a 1-length buffer inside to mark chunk as none-existent + chunk = Buffer.allocUnsafe(1); } const chunkKey = `${EVENT_BACKUP_PREFIX}:${ic}:${jc}`; await redis.set(chunkKey, chunk); diff --git a/src/data/models/RedisCanvas.js b/src/data/models/RedisCanvas.js index c56cd8f..635270e 100644 --- a/src/data/models/RedisCanvas.js +++ b/src/data/models/RedisCanvas.js @@ -4,30 +4,11 @@ import { commandOptions } from 'redis'; import { getChunkOfPixel, getOffsetOfPixel } from '../../core/utils'; -import { - TILE_SIZE, - THREE_TILE_SIZE, - THREE_CANVAS_HEIGHT, -} from '../../core/constants'; -// eslint-disable-next-line import/no-unresolved -import canvases from './canvases.json'; - import redis from '../redis'; const UINT_SIZE = 'u8'; -const EMPTY_CACA = new Uint8Array(TILE_SIZE * TILE_SIZE); -const EMPTY_CHUNK_BUFFER = Buffer.from(EMPTY_CACA.buffer); -const THREE_EMPTY_CACA = new Uint8Array( - THREE_TILE_SIZE * THREE_TILE_SIZE * THREE_CANVAS_HEIGHT, -); -const THREE_EMPTY_CHUNK_BUFFER = Buffer.from(THREE_EMPTY_CACA.buffer); - -// cache existence of chunks -const chunks = new Set(); - - class RedisCanvas { // array of callback functions that gets informed about chunk changes static registerChunkChange = []; @@ -41,29 +22,34 @@ class RedisCanvas { } } - static getChunk( + /* + * Get chunk from redis + * canvasId integer id of canvas + * i, j chunk coordinates + * [padding] required length of chunk (will be padded with zeros if smaller) + * @return chunk as Buffer + */ + static async getChunk( canvasId, i, j, + padding = null, ) { // this key is also hardcoded into // core/tilesBackup.js - // and ./EventData.js const key = `ch:${canvasId}:${i}:${j}`; - return redis.get( + let chunk = await redis.get( commandOptions({ returnBuffers: true }), key, ); + if (padding > 0 && chunk && chunk.length < padding) { + const pad = Buffer.alloc(padding - chunk.length); + chunk = Buffer.concat([chunk, pad]); + } + return chunk; } static async setChunk(i, j, chunk, canvasId) { - if (chunk.length !== TILE_SIZE * TILE_SIZE) { - // eslint-disable-next-line no-console - console.error( - new Error(`Tried to set chunk with invalid length ${chunk.length}!`), - ); - return false; - } const key = `ch:${canvasId}:${i}:${j}`; await redis.set(key, Buffer.from(chunk.buffer)); RedisCanvas.execChunkChangeCallback(canvasId, [i, j]); @@ -73,24 +59,10 @@ class RedisCanvas { static async delChunk(i, j, canvasId) { const key = `ch:${canvasId}:${i}:${j}`; await redis.del(key); - chunks.delete(key); RedisCanvas.execChunkChangeCallback(canvasId, [i, j]); return true; } - static async setPixel( - canvasId, - color, - x, - y, - z = null, - ) { - const canvasSize = canvases[canvasId].size; - const [i, j] = getChunkOfPixel(canvasSize, x, y, z); - const offset = getOffsetOfPixel(canvasSize, x, y, z); - RedisCanvas.setPixelInChunk(i, j, offset, color, canvasId); - } - multi = null; static enqueuePixel( @@ -100,10 +72,20 @@ class RedisCanvas { j, offset, ) { + /* + * TODO what if chunk does not exist? + */ if (!RedisCanvas.multi) { RedisCanvas.multi = redis.multi(); + setTimeout(RedisCanvas.flushPixels, 100); } RedisCanvas.multi.addCommand( + /* + * NOTE: + * If chunk doesn't exist or is smaller than the offset, + * redis will pad with zeros + * https://redis.io/commands/bitfield/ + */ [ 'BITFIELD', `ch:${canvasId}:${i}:${j}`, @@ -126,33 +108,6 @@ class RedisCanvas { return null; } - static async setPixelInChunk( - i, - j, - offset, - color, - canvasId, - ) { - const key = `ch:${canvasId}:${i}:${j}`; - - if (!chunks.has(key)) { - if (canvases[canvasId].v) { - await redis.set(key, THREE_EMPTY_CHUNK_BUFFER, { - NX: true, - }); - } else { - await redis.set(key, EMPTY_CHUNK_BUFFER, { - NX: true, - }); - } - chunks.add(key); - } - - const args = ['BITFIELD', key, 'SET', UINT_SIZE, `#${offset}`, color]; - await redis.sendCommand(args); - RedisCanvas.execChunkChangeCallback(canvasId, [i, j]); - } - static async getPixelIfExists( canvasId, i, @@ -187,11 +142,11 @@ class RedisCanvas { static async getPixel( canvasId, + canvasSize, x, y, z = null, ) { - const canvasSize = canvases[canvasId].size; const [i, j] = getChunkOfPixel(canvasSize, x, y, z); const offset = getOffsetOfPixel(canvasSize, x, y, z); @@ -200,6 +155,5 @@ class RedisCanvas { } } -setInterval(RedisCanvas.flushPixels, 100); export default RedisCanvas; diff --git a/src/routes/chunks.js b/src/routes/chunks.js index d80c8bf..ad1d02e 100644 --- a/src/routes/chunks.js +++ b/src/routes/chunks.js @@ -6,11 +6,6 @@ import etag from 'etag'; import RedisCanvas from '../data/models/RedisCanvas'; -import { - TILE_SIZE, - THREE_TILE_SIZE, - THREE_CANVAS_HEIGHT, -} from '../core/constants'; import logger from '../core/logger'; const chunkEtags = new Map(); @@ -79,13 +74,6 @@ export default async (req, res, next) => { return; } - // for temporary logging to see if we have invalid chunks in redis - - if (chunk.length !== TILE_SIZE * TILE_SIZE - && chunk.length !== (THREE_TILE_SIZE ** 2) * THREE_CANVAS_HEIGHT) { - logger.error(`Chunk ${x},${y}/${c} has invalid length ${chunk.length}!`); - } - curEtag = etag(chunk, { weak: true }); res.set({ ETag: curEtag, diff --git a/src/ui/ChunkRGB3D.js b/src/ui/ChunkRGB3D.js index 283b4bd..fb78508 100644 --- a/src/ui/ChunkRGB3D.js +++ b/src/ui/ChunkRGB3D.js @@ -238,7 +238,15 @@ class Chunk { this.setVoxelByOffset(offset, clr); } - async fromBuffer(chunkBuffer: Uint8Array) { + async fromBuffer(chunkBufferInpt: Uint8Array) { + let chunkBuffer = chunkBufferInpt; + const neededLength = THREE_TILE_SIZE ** 2 * THREE_CANVAS_HEIGHT; + if (chunkBuffer.byteLength < neededLength) { + // eslint-disable-next-line + console.log(`Padding chunk ${this.key} with ${neededLength - chunkBuffer.byteLength} voxels to full length`); + chunkBuffer = new Uint8Array(neededLength); + chunkBuffer.set(chunkBufferInpt); + } this.buffer = chunkBuffer; const [faceCnt, lastPixel, heightMap] = Chunk.calculateMetaData( chunkBuffer, diff --git a/src/workers/tilewriter.js b/src/workers/tilewriter.js index 20512cf..5b5b7bb 100644 --- a/src/workers/tilewriter.js +++ b/src/workers/tilewriter.js @@ -6,7 +6,7 @@ import { isMainThread, parentPort } from 'worker_threads'; -import redis, { connect as connectRedis } from '../data/redis'; +import { connect as connectRedis } from '../data/redis'; import { createZoomTileFromChunk, createZoomedTile, @@ -27,16 +27,16 @@ connectRedis() try { switch (task) { case 'createZoomTileFromChunk': - createZoomTileFromChunk(redis, ...args); + createZoomTileFromChunk(...args); break; case 'createZoomedTile': createZoomedTile(...args); break; case 'createTexture': - createTexture(redis, ...args); + createTexture(...args); break; case 'initializeTiles': - await initializeTiles(redis, ...args); + await initializeTiles(...args); parentPort.postMessage('Done!'); break; default: