refactor redis to deal with chunks of non-full size

src/core/CanvasCleaner.js

@@ -373,13 +373,27 @@ class CanvasCleaner {
         ) {
           chunk = chunks[jAbs - jo + 1][iAbs - io + 1];
         } else {
-          // eslint-disable-next-line no-await-in-loop
+          try {
-          chunk = await RedisCanvas.getChunk(canvasId, iAbs, jAbs);
+            // eslint-disable-next-line no-await-in-loop
-          if (!chunk || chunk.length !== TILE_SIZE * TILE_SIZE) {
+            chunk = await RedisCanvas.getChunk(
+              canvasId,
+              iAbs,
+              jAbs,
+              TILE_SIZE ** 2,
+            );
+          } catch (error) {
+            this.logger(
+              // eslint-disable-next-line max-len
+              `Couldn't load chunk ch:${canvasId}:${iAbs}:${jAbs}: ${error.message}}`,
+            );
+          }
+          if (!chunk || !chunk.length) {
             chunk = null;
             if (chunk) {
-              // eslint-disable-next-line max-len
+              this.logger(
-              this.logger(`Chunk ch:${canvasId}:${iAbs}:${jAbs} has invalid size ${chunk.length}.`);
+                // eslint-disable-next-line max-len
+                `Chunk ch:${canvasId}:${iAbs}:${jAbs} has invalid size ${chunk.length}.`,
+              );
             }
           }
         }

src/core/Image.js

@@ -39,6 +39,7 @@ export async function imageABGR2Canvas(
   logger.info(
     `Loading image with dim ${width}/${height} to ${x}/${y}/${canvasId}`,
   );
+  const expectedLength = TILE_SIZE ** 2;
   const canvas = canvases[canvasId];
   const { colors, cli, size } = canvas;
   const palette = new Palette(colors);
@@ -50,11 +51,18 @@ export async function imageABGR2Canvas(
 
   let totalPxlCnt = 0;
   logger.info(`Loading to chunks from ${ucx} / ${ucy} to ${lcx} / ${lcy} ...`);
-  let chunk;
   for (let cx = ucx; cx <= lcx; cx += 1) {
     for (let cy = ucy; cy <= lcy; cy += 1) {
-      chunk = await RedisCanvas.getChunk(canvasId, cx, cy);
+      let chunk = null;
-      chunk = (chunk && chunk.length === TILE_SIZE * TILE_SIZE)
+      try {
+        chunk = await RedisCanvas.getChunk(canvasId, cx, cy, expectedLength);
+      } catch (error) {
+        logger.error(
+          // eslint-disable-next-line max-len
+          `Could not load chunk ch:${canvasId}:${cx}:${cy} for image-load: ${error.message}`,
+        );
+      }
+      chunk = (chunk && chunk.length)
         ? new Uint8Array(chunk)
         : new Uint8Array(TILE_SIZE * TILE_SIZE);
       // offset of chunk in image
@@ -119,6 +127,7 @@ export async function imagemask2Canvas(
   logger.info(
     `Loading mask with size ${width} / ${height} to ${x} / ${y} to the canvas`,
   );
+  const expectedLength = TILE_SIZE ** 2;
   const canvas = canvases[canvasId];
   const palette = new Palette(canvas.colors);
   const canvasMinXY = -(canvas.size / 2);
@@ -129,11 +138,18 @@ export async function imagemask2Canvas(
   const [lcx, lcy] = getChunkOfPixel(canvas.size, x + width, y + height);
 
   logger.info(`Loading to chunks from ${ucx} / ${ucy} to ${lcx} / ${lcy} ...`);
-  let chunk;
   for (let cx = ucx; cx <= lcx; cx += 1) {
     for (let cy = ucy; cy <= lcy; cy += 1) {
-      chunk = await RedisCanvas.getChunk(canvasId, cx, cy);
+      let chunk = null;
-      chunk = (chunk && chunk.length === TILE_SIZE * TILE_SIZE)
+      try {
+        chunk = await RedisCanvas.getChunk(canvasId, cx, cy, expectedLength);
+      } catch (error) {
+        logger.error(
+          // eslint-disable-next-line max-len
+          `Could not load chunk ch:${canvasId}:${cx}:${cy} for imagemask-load: ${error.message}`,
+        );
+      }
+      chunk = (chunk && chunk.length)
         ? new Uint8Array(chunk)
         : new Uint8Array(TILE_SIZE * TILE_SIZE);
       // offset of chunk in image
@@ -191,6 +207,7 @@ export async function protectCanvasArea(
     // eslint-disable-next-line max-len
     `Setting protection ${protect} with size ${width} / ${height} to ${x} / ${y}`,
   );
+  const expectedLength = TILE_SIZE ** 2;
   const canvas = canvases[canvasId];
   const canvasMinXY = -(canvas.size / 2);
 
@@ -201,11 +218,18 @@ export async function protectCanvasArea(
   );
 
   let totalPxlCnt = 0;
-  let chunk;
   for (let cx = ucx; cx <= lcx; cx += 1) {
     for (let cy = ucy; cy <= lcy; cy += 1) {
-      chunk = await RedisCanvas.getChunk(canvasId, cx, cy);
+      let chunk = null;
-      if (!chunk || chunk.length !== TILE_SIZE * TILE_SIZE) {
+      try {
+        chunk = await RedisCanvas.getChunk(canvasId, cx, cy, expectedLength);
+      } catch (error) {
+        logger.error(
+          // eslint-disable-next-line max-len
+          `Could not load chunk ch:${canvasId}:${cx}:${cy} for protection: ${error.message}`,
+        );
+      }
+      if (!chunk || !chunk.length) {
         continue;
       }
       chunk = new Uint8Array(chunk);

src/core/Palette.js

@@ -110,11 +110,10 @@ class Palette {
     const { length } = chunkBuffer;
     const colors = new Uint32Array(length);
     let value;
-    const buffer = chunkBuffer;
 
     let pos = 0;
     for (let i = 0; i < length; i++) {
-      value = (buffer[i] & 0x3F);
+      value = (chunkBuffer[i] & 0x3F);
       colors[pos++] = this.abgr[value];
     }
     return colors;

src/core/Tile.js

@@ -11,8 +11,8 @@
 
 import sharp from 'sharp';
 import fs from 'fs';
-import { commandOptions } from 'redis';
 
+import RedisCanvas from '../data/models/RedisCanvas';
 import Palette from './Palette';
 import { getMaxTiledZoom } from './utils';
 import { TILE_SIZE, TILE_ZOOM_LEVEL } from './constants';
@@ -29,7 +29,7 @@ function deleteSubtilefromTile(
   palette,
   subtilesInTile,
   cell,
-  buffer: Uint8Array,
+  buffer,
 ) {
   const [dx, dy] = cell;
   const offset = (dx + dy * TILE_SIZE * subtilesInTile) * TILE_SIZE;
@@ -56,8 +56,8 @@ function deleteSubtilefromTile(
 function addRGBSubtiletoTile(
   subtilesInTile,
   cell,
-  subtile: Buffer,
+  subtile,
-  buffer: Uint8Array,
+  buffer,
 ) {
   const [dx, dy] = cell;
   const chunkOffset = (dx + dy * subtilesInTile * TILE_SIZE) * TILE_SIZE;
@@ -84,21 +84,32 @@ function addIndexedSubtiletoTile(
   palette,
   subtilesInTile,
   cell,
-  subtile: Buffer,
+  subtile,
-  buffer: Uint8Array,
+  buffer,
 ) {
   const [dx, dy] = cell;
   const chunkOffset = (dx + dy * subtilesInTile * TILE_SIZE) * TILE_SIZE;
+
+  const emptyR = palette.rgb[0];
+  const emptyB = palette.rgb[1];
+  const emptyG = palette.rgb[1];
+
   let pos = 0;
   let clr;
   for (let row = 0; row < TILE_SIZE; row += 1) {
     let channelOffset = (chunkOffset + row * TILE_SIZE * subtilesInTile) * 3;
     const max = channelOffset + TILE_SIZE * 3;
     while (channelOffset < max) {
-      clr = (subtile[pos++] & 0x3F) * 3;
+      if (pos < subtile.length) {
-      buffer[channelOffset++] = palette.rgb[clr++];
+        clr = (subtile[pos++] & 0x3F) * 3;
-      buffer[channelOffset++] = palette.rgb[clr++];
+        buffer[channelOffset++] = palette.rgb[clr++];
-      buffer[channelOffset++] = palette.rgb[clr];
+        buffer[channelOffset++] = palette.rgb[clr++];
+        buffer[channelOffset++] = palette.rgb[clr];
+      } else {
+        buffer[channelOffset++] = emptyR;
+        buffer[channelOffset++] = emptyB;
+        buffer[channelOffset++] = emptyG;
+      }
     }
   }
 }
@@ -115,7 +126,6 @@ function tileFileName(canvasTileFolder, cell) {
 }
 
 /*
- * @param redisClient redis instance
  * @param canvasId id of the canvas
  * @param canvas canvas data
  * @param canvasTileFolder root folder where to save tiles
@@ -123,7 +133,6 @@ function tileFileName(canvasTileFolder, cell) {
  * @return true if successfully created tile, false if tile empty
  */
 export async function createZoomTileFromChunk(
-  redisClient,
   canvasId,
   canvas,
   canvasTileFolder,
@@ -141,14 +150,22 @@ export async function createZoomTileFromChunk(
   const xabs = x * TILE_ZOOM_LEVEL;
   const yabs = y * TILE_ZOOM_LEVEL;
   const na = [];
-  let chunk = null;
   for (let dy = 0; dy < TILE_ZOOM_LEVEL; dy += 1) {
     for (let dx = 0; dx < TILE_ZOOM_LEVEL; dx += 1) {
-      chunk = await redisClient.get(
+      let chunk = null;
-        commandOptions({ returnBuffers: true }),
+      try {
-        `ch:${canvasId}:${xabs + dx}:${yabs + dy}`,
+        chunk = await RedisCanvas.getChunk(
-      );
+          canvasId,
-      if (!chunk || chunk.length !== TILE_SIZE * TILE_SIZE) {
+          xabs + dx,
+          yabs + dy,
+        );
+      } catch (error) {
+        console.error(
+          // eslint-disable-next-line max-len
+          `Tiling: Failed to get Chunk ch:${canvasId}:${xabs + dx}${yabs + dy} with error ${error.message}`,
+        );
+      }
+      if (!chunk || !chunk.length) {
         na.push([dx, dy]);
         continue;
       }
@@ -161,7 +178,6 @@ export async function createZoomTileFromChunk(
       );
     }
   }
-  chunk = null;
 
   if (na.length !== TILE_ZOOM_LEVEL * TILE_ZOOM_LEVEL) {
     na.forEach((element) => {
@@ -311,14 +327,12 @@ async function createEmptyTile(
 
 /*
  * created 4096x4096 texture of default canvas
- * @param redisClient redis instance
  * @param canvasId numberical Id of canvas
  * @param canvas canvas data
  * @param canvasTileFolder root folder where to save texture
  *
  */
 export async function createTexture(
-  redisClient,
   canvasId,
   canvas,
   canvasTileFolder,
@@ -333,10 +347,10 @@ export async function createTexture(
   const startTime = Date.now();
 
   const na = [];
-  let chunk = null;
   if (targetSize !== canvasSize) {
     for (let dy = 0; dy < amount; dy += 1) {
       for (let dx = 0; dx < amount; dx += 1) {
+        let chunk = null;
         const chunkfile = `${canvasTileFolder}/${zoom}/${dx}/${dy}.png`;
         if (!fs.existsSync(chunkfile)) {
           na.push([dx, dy]);
@@ -356,11 +370,20 @@ export async function createTexture(
   } else {
     for (let dy = 0; dy < amount; dy += 1) {
       for (let dx = 0; dx < amount; dx += 1) {
-        chunk = await redisClient.get(
+        let chunk = null;
-          commandOptions({ returnBuffers: true }),
+        try {
-          `ch:${canvasId}:${dx}:${dy}`,
+          chunk = await RedisCanvas.getChunk(
-        );
+            canvasId,
-        if (!chunk || chunk.length !== TILE_SIZE * TILE_SIZE) {
+            dx,
+            dy,
+          );
+        } catch (error) {
+          console.error(
+            // eslint-disable-next-line max-len
+            `Tiling: Failed to get Chunk ch:${canvasId}:${dx}${dy} with error ${error.message}`,
+          );
+        }
+        if (!chunk || !chunk.length) {
           na.push([dx, dy]);
           continue;
         }
@@ -374,7 +397,6 @@ export async function createTexture(
       }
     }
   }
-  chunk = null;
 
   na.forEach((element) => {
     deleteSubtilefromTile(palette, amount, element, textureBuffer);
@@ -404,14 +426,12 @@ export async function createTexture(
 
 /*
  * Create all tiles
- * @param redisClient redis instance
  * @param canvasId id of the canvas
  * @param canvas canvas data
  * @param canvasTileFolder folder for tiles
  * @param force overwrite existing tiles
  */
 export async function initializeTiles(
-  redisClient,
   canvasId,
   canvas,
   canvasTileFolder,
@@ -441,7 +461,6 @@ export async function initializeTiles(
       const filename = `${canvasTileFolder}/${zoom}/${cx}/${cy}.png`;
       if (force || !fs.existsSync(filename)) {
         const ret = await createZoomTileFromChunk(
-          redisClient,
           canvasSize,
           canvasId,
           canvasTileFolder,
@@ -487,7 +506,6 @@ export async function initializeTiles(
   }
   // create snapshot texture
   await createTexture(
-    redisClient,
     canvasId,
     canvas,
     canvasTileFolder,

src/core/draw.js

@@ -315,7 +315,7 @@ export async function drawByCoords(
   }
 
   const isAdmin = (user.userlvl === 1);
-  const setColor = await RedisCanvas.getPixel(canvasId, x, y, z);
+  const setColor = await RedisCanvas.getPixel(canvasId, canvas.size, x, y, z);
 
   /*
    * bitwise operation to get rid of protection

src/core/rollback.js

@@ -51,19 +51,26 @@ export default async function rollbackToDate(
 
   let totalPxlCnt = 0;
   logger.info(`Loading to chunks from ${ucx} / ${ucy} to ${lcx} / ${lcy} ...`);
-  let chunk;
   let empty = false;
   let emptyBackup = false;
   let backupChunk;
   for (let cx = ucx; cx <= lcx; cx += 1) {
     for (let cy = ucy; cy <= lcy; cy += 1) {
-      chunk = await RedisCanvas.getChunk(canvasId, cx, cy);
+      let chunk = null;
-      if (chunk && chunk.length === TILE_SIZE * TILE_SIZE) {
+      try {
-        chunk = new Uint8Array(chunk);
+        chunk = await RedisCanvas.getChunk(canvasId, cx, cy, TILE_SIZE ** 2);
-        empty = false;
+      } catch (error) {
-      } else {
+        logger.error(
+          // eslint-disable-next-line max-len
+          `Chunk ch:${canvasId}:${cx}:${cy} could not be loaded from redis, assuming empty.`,
+        );
+      }
+      if (!chunk || !chunk.length) {
         chunk = new Uint8Array(TILE_SIZE * TILE_SIZE);
         empty = true;
+      } else {
+        chunk = new Uint8Array(chunk);
+        empty = false;
       }
       try {
         emptyBackup = false;

src/core/tilesBackup.js

@@ -13,6 +13,22 @@ import Palette from './Palette';
 import { TILE_SIZE } from './constants';
 
 
+/*
+ * take chunk buffer and pad it to a specific length
+ * Fill missing pixels with zeros
+ * @param length target length
+ */
+function padChunk(chunk, length) {
+  let retChunk = chunk;
+  if (!chunk || !chunk.length) {
+    retChunk = Buffer.alloc(length);
+  } else if (chunk.length < length) {
+    const padding = Buffer.alloc(length - chunk.length);
+    retChunk = Buffer.concat([chunk, padding]);
+  }
+  return retChunk;
+}
+
 /*
  * Copy canvases from one redis instance to another
  * @param canvasRedis redis from where to get the data
@@ -35,19 +51,31 @@ export async function updateBackupRedis(canvasRedis, backupRedis, canvases) {
     for (let x = 0; x < chunksXY; x++) {
       for (let y = 0; y < chunksXY; y++) {
         const key = `ch:${id}:${x}:${y}`;
-        /*
+        let chunk = null;
-         * await on every iteration is fine because less resource usage
+
-         * in exchange for higher execution time is wanted.
+        try {
-         */
-        // eslint-disable-next-line no-await-in-loop
-        const chunk = await canvasRedis.get(
-          commandOptions({ returnBuffers: true }),
-          key,
-        );
-        if (chunk) {
           // eslint-disable-next-line no-await-in-loop
-          await backupRedis.set(key, chunk);
+          chunk = await canvasRedis.get(
-          amount += 1;
+            commandOptions({ returnBuffers: true }),
+            key,
+          );
+        } catch (error) {
+          console.error(
+            // eslint-disable-next-line max-len
+            new Error(`Could not get chunk ${key} from redis: ${error.message}`),
+          );
+        }
+        if (chunk) {
+          try {
+            // eslint-disable-next-line no-await-in-loop
+            await backupRedis.set(key, chunk);
+            amount += 1;
+          } catch (error) {
+            console.error(
+              // eslint-disable-next-line max-len
+              new Error(`Could not create chunk ${key} in backup-redis: ${error.message}`),
+            );
+          }
         }
       }
     }
@@ -105,62 +133,106 @@ export async function incrementialBackupRedis(
 
       for (let y = 0; y < chunksXY; y++) {
         const key = `ch:${id}:${x}:${y}`;
-        /*
+
-         * await on every iteration is fine because less resource usage
+        let curChunk = null;
-         * in exchange for higher execution time is wanted.
+        try {
-         */
+          // eslint-disable-next-line no-await-in-loop
-        // eslint-disable-next-line no-await-in-loop
+          curChunk = await canvasRedis.get(
-        const curChunk = await canvasRedis.get(
+            commandOptions({ returnBuffers: true }),
-          commandOptions({ returnBuffers: true }),
+            key,
-          key,
+          );
-        );
+        } catch (error) {
+          console.error(
+            // eslint-disable-next-line max-len
+            new Error(`Could not get chunk ${key} from redis: ${error.message}`),
+          );
+        }
+
+        let oldChunk = null;
+        try {
+          // eslint-disable-next-line no-await-in-loop
+          oldChunk = await backupRedis.get(
+            commandOptions({ returnBuffers: true }),
+            key,
+          );
+        } catch (error) {
+          console.error(
+            // eslint-disable-next-line max-len
+            new Error(`Could not get chunk ${key} from backup-redis: ${error.message}`),
+          );
+        }
+
+        // is gonna be an Uint32Array
         let tileBuffer = null;
-        if (curChunk) {
+
-          if (curChunk.length === TILE_SIZE * TILE_SIZE) {
+        try {
-            // eslint-disable-next-line no-await-in-loop
+          if (!oldChunk && !curChunk) {
-            const oldChunk = await backupRedis.get(
+            continue;
-              commandOptions({ returnBuffers: true }),
+          }
-              key,
+          if (oldChunk && !curChunk) {
-            );
+            // one does not exist
-            if (oldChunk && oldChunk.length === TILE_SIZE * TILE_SIZE) {
+            curChunk = Buffer.alloc(TILE_SIZE * TILE_SIZE);
-              let pxl = 0;
+            tileBuffer = palette.buffer2ABGR(curChunk);
-              while (pxl < curChunk.length) {
+          } else if (!oldChunk && curChunk) {
-                if (curChunk[pxl] !== oldChunk[pxl]) {
+            tileBuffer = new Uint32Array(TILE_SIZE * TILE_SIZE);
-                  if (!tileBuffer) {
+            const pxl = 0;
-                    tileBuffer = new Uint32Array(TILE_SIZE * TILE_SIZE);
+            while (pxl < curChunk.length) {
-                  }
+              const clrIndex = curChunk[pxl] & 0x3F;
-                  const color = palette.abgr[curChunk[pxl] & 0x3F];
+              if (clrIndex > 0) {
-                  tileBuffer[pxl] = color;
+                const color = palette.abgr[clrIndex];
-                }
+                tileBuffer[pxl] = color;
-                pxl += 1;
               }
-            } else {
-              tileBuffer = palette.buffer2ABGR(curChunk);
             }
           } else {
-            console.log(
+            if (curChunk.length < oldChunk.length) {
-              // eslint-disable-next-line max-len
+              curChunk = padChunk(curChunk, oldChunk.length);
-              `Chunk ${key} in backup-redis has invalid length ${curChunk.length}`,
+            } else if (curChunk.length > oldChunk.length) {
-            );
+              oldChunk = padChunk(oldChunk, curChunk.length);
+            }
+            // both exist and are the same length
+            tileBuffer = new Uint32Array(TILE_SIZE * TILE_SIZE);
+            let pxl = 0;
+            while (pxl < curChunk.length) {
+              if (curChunk[pxl] !== oldChunk[pxl]) {
+                const color = palette.abgr[curChunk[pxl] & 0x3F];
+                tileBuffer[pxl] = color;
+              }
+              pxl += 1;
+            }
           }
+        } catch (error) {
+          console.error(
+            // eslint-disable-next-line max-len
+            new Error(`Could not populate incremential backup data of chunk ${key}: ${error.message}`),
+          );
+          continue;
         }
+
         if (tileBuffer) {
-          if (!createdDir && !fs.existsSync(xBackupDir)) {
+          try {
-            createdDir = true;
+            if (!createdDir && !fs.existsSync(xBackupDir)) {
-            fs.mkdirSync(xBackupDir);
+              createdDir = true;
-          }
+              fs.mkdirSync(xBackupDir);
-          const filename = `${xBackupDir}/${y}.png`;
+            }
-          // eslint-disable-next-line no-await-in-loop
+            const filename = `${xBackupDir}/${y}.png`;
-          await sharp(
+            // eslint-disable-next-line no-await-in-loop
-            Buffer.from(tileBuffer.buffer), {
+            await sharp(
-              raw: {
+              Buffer.from(tileBuffer.buffer), {
-                width: TILE_SIZE,
+                raw: {
-                height: TILE_SIZE,
+                  width: TILE_SIZE,
-                channels: 4,
+                  height: TILE_SIZE,
+                  channels: 4,
+                },
               },
-            },
+            ).toFile(filename);
-          ).toFile(filename);
+            amount += 1;
-          amount += 1;
+          } catch (error) {
+            console.error(
+              // eslint-disable-next-line max-len
+              new Error(`Could not save incremential backup of chunk ${key}: ${error.message}`),
+            );
+            continue;
+          }
         }
       }
     }
@@ -209,43 +281,44 @@ export async function createPngBackup(
       }
       for (let y = 0; y < chunksXY; y++) {
         const key = `ch:${id}:${x}:${y}`;
+
+        let chunk = null;
         try {
-          /*
-           * await on every iteration is fine because less resource usage
-           * in exchange for higher execution time is wanted.
-           */
           // eslint-disable-next-line no-await-in-loop
-          const chunk = await redisClient.get(
+          chunk = await redisClient.get(
             commandOptions({ returnBuffers: true }),
             key,
           );
-          if (chunk) {
+        } catch (error) {
-            if (chunk.length === TILE_SIZE * TILE_SIZE) {
+          console.error(
-              const textureBuffer = palette.buffer2RGB(chunk);
+            // eslint-disable-next-line max-len
-              const filename = `${xBackupDir}/${y}.png`;
+            new Error(`Could not get chunk ${key} from redis: ${error.message}`),
-              // eslint-disable-next-line no-await-in-loop
-              await sharp(
-                Buffer.from(textureBuffer.buffer), {
-                  raw: {
-                    width: TILE_SIZE,
-                    height: TILE_SIZE,
-                    channels: 3,
-                  },
-                },
-              ).toFile(filename);
-              amount += 1;
-            } else {
-              console.log(
-                // eslint-disable-next-line max-len
-                `Chunk ${key} has invalid length ${chunk.length}`,
-              );
-            }
-          }
-        } catch {
-          console.log(
-            `Couldn't create PNG backup of chunk ${x},${y} of canvas ${id}.`,
           );
         }
+        if (chunk && chunk.length) {
+          chunk = padChunk(chunk, TILE_SIZE * TILE_SIZE);
+          const textureBuffer = palette.buffer2RGB(chunk);
+          const filename = `${xBackupDir}/${y}.png`;
+          try {
+            // eslint-disable-next-line no-await-in-loop
+            await sharp(
+              Buffer.from(textureBuffer.buffer), {
+                raw: {
+                  width: TILE_SIZE,
+                  height: TILE_SIZE,
+                  channels: 3,
+                },
+              },
+            ).toFile(filename);
+          } catch (error) {
+            console.error(
+              // eslint-disable-next-line max-len
+              new Error(`Could not save daily backup of chunk ${key}: ${error.message}`),
+            );
+            continue;
+          }
+          amount += 1;
+        }
       }
     }
     const time = Date.now() - startTime;

src/data/models/Event.js

@@ -68,32 +68,38 @@ export async function clearOldEvent() {
     // 3x3 chunk area centered at i,j
     for (let jc = j - 1; jc <= j + 1; jc += 1) {
       for (let ic = i - 1; ic <= i + 1; ic += 1) {
-        const chunkKey = `${EVENT_BACKUP_PREFIX}:${ic}:${jc}`;
+        try {
-        const chunk = await redis.get(
+          const chunkKey = `${EVENT_BACKUP_PREFIX}:${ic}:${jc}`;
-          commandOptions({ returnBuffers: true }),
+          const chunk = await redis.get(
-          chunkKey,
+            commandOptions({ returnBuffers: true }),
-        );
+            chunkKey,
-        if (!chunk) {
+          );
-          logger.warn(
+          if (!chunk) {
+            logger.warn(
+              // eslint-disable-next-line max-len
+              `Couldn't get chunk event backup for ${ic}/${jc}, which is weird`,
+            );
+            continue;
+          }
+          if (chunk.length <= 1) {
+            logger.info(
+              // eslint-disable-next-line max-len
+              `Tiny chunk in event backup, not-generated chunk at ${ic}/${jc}`,
+            );
+            await RedisCanvas.delChunk(ic, jc, CANVAS_ID);
+          } else {
+            logger.info(
+              `Restoring chunk ${ic}/${jc} from event`,
+            );
+            await RedisCanvas.setChunk(ic, jc, chunk, CANVAS_ID);
+          }
+          await redis.del(chunkKey);
+        } catch (error) {
+          logger.error(
             // eslint-disable-next-line max-len
-            `Couldn't get chunk event backup for ${ic}/${jc}, which is weird`,
+            `Couldn't restore chunk from RpgEvent ${EVENT_BACKUP_PREFIX}:${ic}:${jc} : ${error.message}`,
           );
-          continue;
         }
-        if (chunk.length <= 256) {
-          logger.info(
-            // eslint-disable-next-line max-len
-            `Tiny chunk in event backup, not-generated chunk at ${ic}/${jc}`,
-          );
-          await RedisCanvas.delChunk(ic, jc, CANVAS_ID);
-        } else {
-          logger.info(
-            `Restoring chunk ${ic}/${jc} from event`,
-          );
-          const chunkArray = new Uint8Array(chunk);
-          await RedisCanvas.setChunk(ic, jc, chunkArray, CANVAS_ID);
-        }
-        await redis.del(chunkKey);
       }
     }
     await redis.del(EVENT_POSITION_KEY);
@@ -109,13 +115,18 @@ export async function setNextEvent(minutes, i, j) {
   await clearOldEvent();
   for (let jc = j - 1; jc <= j + 1; jc += 1) {
     for (let ic = i - 1; ic <= i + 1; ic += 1) {
-      let chunk = await RedisCanvas.getChunk(CANVAS_ID, ic, jc);
+      let chunk = null;
-      if (!chunk) {
+      try {
-        // place a dummy Array inside to mark chunk as none-existent
+        chunk = await RedisCanvas.getChunk(CANVAS_ID, ic, jc);
-        const buff = new Uint8Array(3);
+      } catch (error) {
-        chunk = Buffer.from(buff);
+        logger.error(
-        // place dummy pixel to make RedisCanvas create chunk
+          // eslint-disable-next-line max-len
-        await RedisCanvas.setPixelInChunk(ic, jc, 0, 0, CANVAS_ID);
+          `Could not load chunk ch:${CANVAS_ID}:${ic}:${jc} for RpgEvent backup: ${error.message}`,
+        );
+      }
+      if (!chunk || !chunk.length) {
+        // place a 1-length buffer inside to mark chunk as none-existent
+        chunk = Buffer.allocUnsafe(1);
       }
       const chunkKey = `${EVENT_BACKUP_PREFIX}:${ic}:${jc}`;
       await redis.set(chunkKey, chunk);

src/data/models/RedisCanvas.js

@@ -4,30 +4,11 @@
 import { commandOptions } from 'redis';
 
 import { getChunkOfPixel, getOffsetOfPixel } from '../../core/utils';
-import {
-  TILE_SIZE,
-  THREE_TILE_SIZE,
-  THREE_CANVAS_HEIGHT,
-} from '../../core/constants';
-// eslint-disable-next-line import/no-unresolved
-import canvases from './canvases.json';
-
 import redis from '../redis';
 
 
 const UINT_SIZE = 'u8';
 
-const EMPTY_CACA = new Uint8Array(TILE_SIZE * TILE_SIZE);
-const EMPTY_CHUNK_BUFFER = Buffer.from(EMPTY_CACA.buffer);
-const THREE_EMPTY_CACA = new Uint8Array(
-  THREE_TILE_SIZE * THREE_TILE_SIZE * THREE_CANVAS_HEIGHT,
-);
-const THREE_EMPTY_CHUNK_BUFFER = Buffer.from(THREE_EMPTY_CACA.buffer);
-
-// cache existence of chunks
-const chunks = new Set();
-
-
 class RedisCanvas {
   // array of callback functions that gets informed about chunk changes
   static registerChunkChange = [];
@@ -41,29 +22,34 @@ class RedisCanvas {
     }
   }
 
-  static getChunk(
+  /*
+   * Get chunk from redis
+   * canvasId integer id of canvas
+   * i, j chunk coordinates
+   * [padding] required length of chunk  (will be padded with zeros if smaller)
+   * @return chunk as Buffer
+   */
+  static async getChunk(
     canvasId,
     i,
     j,
+    padding = null,
   ) {
     // this key is also hardcoded into
     // core/tilesBackup.js
-    // and ./EventData.js
     const key = `ch:${canvasId}:${i}:${j}`;
-    return redis.get(
+    let chunk = await redis.get(
       commandOptions({ returnBuffers: true }),
       key,
     );
+    if (padding > 0 && chunk && chunk.length < padding) {
+      const pad = Buffer.alloc(padding - chunk.length);
+      chunk = Buffer.concat([chunk, pad]);
+    }
+    return chunk;
   }
 
   static async setChunk(i, j, chunk, canvasId) {
-    if (chunk.length !== TILE_SIZE * TILE_SIZE) {
-      // eslint-disable-next-line no-console
-      console.error(
-        new Error(`Tried to set chunk with invalid length ${chunk.length}!`),
-      );
-      return false;
-    }
     const key = `ch:${canvasId}:${i}:${j}`;
     await redis.set(key, Buffer.from(chunk.buffer));
     RedisCanvas.execChunkChangeCallback(canvasId, [i, j]);
@@ -73,24 +59,10 @@ class RedisCanvas {
   static async delChunk(i, j, canvasId) {
     const key = `ch:${canvasId}:${i}:${j}`;
     await redis.del(key);
-    chunks.delete(key);
     RedisCanvas.execChunkChangeCallback(canvasId, [i, j]);
     return true;
   }
 
-  static async setPixel(
-    canvasId,
-    color,
-    x,
-    y,
-    z = null,
-  ) {
-    const canvasSize = canvases[canvasId].size;
-    const [i, j] = getChunkOfPixel(canvasSize, x, y, z);
-    const offset = getOffsetOfPixel(canvasSize, x, y, z);
-    RedisCanvas.setPixelInChunk(i, j, offset, color, canvasId);
-  }
-
   multi = null;
 
   static enqueuePixel(
@@ -100,10 +72,20 @@ class RedisCanvas {
     j,
     offset,
   ) {
+    /*
+     * TODO what if chunk does not exist?
+     */
     if (!RedisCanvas.multi) {
       RedisCanvas.multi = redis.multi();
+      setTimeout(RedisCanvas.flushPixels, 100);
     }
     RedisCanvas.multi.addCommand(
+      /*
+       * NOTE:
+       * If chunk doesn't exist or is smaller than the offset,
+       * redis will pad with zeros
+       * https://redis.io/commands/bitfield/
+       */
       [
         'BITFIELD',
         `ch:${canvasId}:${i}:${j}`,
@@ -126,33 +108,6 @@ class RedisCanvas {
     return null;
   }
 
-  static async setPixelInChunk(
-    i,
-    j,
-    offset,
-    color,
-    canvasId,
-  ) {
-    const key = `ch:${canvasId}:${i}:${j}`;
-
-    if (!chunks.has(key)) {
-      if (canvases[canvasId].v) {
-        await redis.set(key, THREE_EMPTY_CHUNK_BUFFER, {
-          NX: true,
-        });
-      } else {
-        await redis.set(key, EMPTY_CHUNK_BUFFER, {
-          NX: true,
-        });
-      }
-      chunks.add(key);
-    }
-
-    const args = ['BITFIELD', key, 'SET', UINT_SIZE, `#${offset}`, color];
-    await redis.sendCommand(args);
-    RedisCanvas.execChunkChangeCallback(canvasId, [i, j]);
-  }
-
   static async getPixelIfExists(
     canvasId,
     i,
@@ -187,11 +142,11 @@ class RedisCanvas {
 
   static async getPixel(
     canvasId,
+    canvasSize,
     x,
     y,
     z = null,
   ) {
-    const canvasSize = canvases[canvasId].size;
     const [i, j] = getChunkOfPixel(canvasSize, x, y, z);
     const offset = getOffsetOfPixel(canvasSize, x, y, z);
 
@@ -200,6 +155,5 @@ class RedisCanvas {
   }
 }
 
-setInterval(RedisCanvas.flushPixels, 100);
 
 export default RedisCanvas;

src/routes/chunks.js

@@ -6,11 +6,6 @@
 
 import etag from 'etag';
 import RedisCanvas from '../data/models/RedisCanvas';
-import {
-  TILE_SIZE,
-  THREE_TILE_SIZE,
-  THREE_CANVAS_HEIGHT,
-} from '../core/constants';
 import logger from '../core/logger';
 
 const chunkEtags = new Map();
@@ -79,13 +74,6 @@ export default async (req, res, next) => {
       return;
     }
 
-    // for temporary logging to see if we have invalid chunks in redis
-
-    if (chunk.length !== TILE_SIZE * TILE_SIZE
-      && chunk.length !== (THREE_TILE_SIZE ** 2) * THREE_CANVAS_HEIGHT) {
-      logger.error(`Chunk ${x},${y}/${c} has invalid length ${chunk.length}!`);
-    }
-
     curEtag = etag(chunk, { weak: true });
     res.set({
       ETag: curEtag,

src/ui/ChunkRGB3D.js

@@ -238,7 +238,15 @@ class Chunk {
     this.setVoxelByOffset(offset, clr);
   }
 
-  async fromBuffer(chunkBuffer: Uint8Array) {
+  async fromBuffer(chunkBufferInpt: Uint8Array) {
+    let chunkBuffer = chunkBufferInpt;
+    const neededLength = THREE_TILE_SIZE ** 2 * THREE_CANVAS_HEIGHT;
+    if (chunkBuffer.byteLength < neededLength) {
+      // eslint-disable-next-line
+      console.log(`Padding chunk ${this.key} with ${neededLength - chunkBuffer.byteLength} voxels to full length`);
+      chunkBuffer = new Uint8Array(neededLength);
+      chunkBuffer.set(chunkBufferInpt);
+    }
     this.buffer = chunkBuffer;
     const [faceCnt, lastPixel, heightMap] = Chunk.calculateMetaData(
       chunkBuffer,

src/workers/tilewriter.js

@@ -6,7 +6,7 @@
 
 import { isMainThread, parentPort } from 'worker_threads';
 
-import redis, { connect as connectRedis } from '../data/redis';
+import { connect as connectRedis } from '../data/redis';
 import {
   createZoomTileFromChunk,
   createZoomedTile,
@@ -27,16 +27,16 @@ connectRedis()
       try {
         switch (task) {
           case 'createZoomTileFromChunk':
-            createZoomTileFromChunk(redis, ...args);
+            createZoomTileFromChunk(...args);
             break;
           case 'createZoomedTile':
             createZoomedTile(...args);
             break;
           case 'createTexture':
-            createTexture(redis, ...args);
+            createTexture(...args);
             break;
           case 'initializeTiles':
-            await initializeTiles(redis, ...args);
+            await initializeTiles(...args);
             parentPort.postMessage('Done!');
             break;
           default: