Horicontal Scaling

src/components/Admintools.jsx

@@ -5,6 +5,8 @@
 import React, { useState, useEffect } from 'react';
 import { t } from 'ttag';
 
+import { shardOrigin } from '../store/actions/fetch';
+
 async function submitIPAction(
   action,
   vallist,
@@ -13,7 +15,7 @@ async function submitIPAction(
   const data = new FormData();
   data.append('ipaction', action);
   data.append('ip', vallist);
-  const resp = await fetch('/api/modtools', {
+  const resp = await fetch(`${shardOrigin}/api/modtools`, {
     credentials: 'include',
     method: 'POST',
     body: data,
@@ -26,7 +28,7 @@ async function getModList(
 ) {
   const data = new FormData();
   data.append('modlist', true);
-  const resp = await fetch('/api/modtools', {
+  const resp = await fetch(`${shardOrigin}/api/modtools`, {
     credentials: 'include',
     method: 'POST',
     body: data,
@@ -44,7 +46,7 @@ async function submitRemMod(
 ) {
   const data = new FormData();
   data.append('remmod', userId);
-  const resp = await fetch('/api/modtools', {
+  const resp = await fetch(`${shardOrigin}/api/modtools`, {
     credentials: 'include',
     method: 'POST',
     body: data,
@@ -58,7 +60,7 @@ async function submitMakeMod(
 ) {
   const data = new FormData();
   data.append('makemod', userName);
-  const resp = await fetch('/api/modtools', {
+  const resp = await fetch(`${shardOrigin}/api/modtools`, {
     credentials: 'include',
     method: 'POST',
     body: data,

src/components/Captcha.jsx

@@ -10,9 +10,10 @@ import React, { useState, useEffect } from 'react';
 import { t } from 'ttag';
 
 import { IoReloadCircleSharp } from 'react-icons/io5';
+import { shardOrigin } from '../store/actions/fetch';
 
 async function getUrlAndId() {
-  const url = './captcha.svg';
+  const url = `${shardOrigin}/captcha.svg`;
   const resp = await fetch(url, {
     cache: 'no-cache',
   });

src/components/LanguageSelect.jsx

@@ -66,7 +66,14 @@ function LanguageSelect() {
           /* set with selected language */
           const d = new Date();
           d.setTime(d.getTime() + 24 * MONTH);
-          document.cookie = `lang=${langSel};expires=${d.toUTCString()};path=/`;
+          let { host } = window.location;
+          if (host.lastIndexOf('.') !== host.indexOf('.')) {
+            host = host.slice(host.indexOf('.'));
+          } else {
+            host = `.${host}`;
+          }
+          // eslint-disable-next-line max-len
+          document.cookie = `lang=${langSel};expires=${d.toUTCString()};path=/;domain=${host}`;
           window.location.reload();
         }}
       >

src/components/ModCanvastools.jsx

@@ -8,6 +8,7 @@ import { t } from 'ttag';
 
 import useInterval from './hooks/interval';
 import { getToday, dateToString } from '../core/utils';
+import { shardOrigin } from '../store/actions/fetch';
 
 const keptState = {
   coords: '',
@@ -33,7 +34,7 @@ async function submitImageAction(
   data.append('image', file);
   data.append('canvasid', canvas);
   data.append('coords', coords);
-  const resp = await fetch('/api/modtools', {
+  const resp = await fetch(`${shardOrigin}/api/modtools`, {
     credentials: 'include',
     method: 'POST',
     body: data,
@@ -53,7 +54,7 @@ async function submitProtAction(
   data.append('canvasid', canvas);
   data.append('ulcoor', tlcoords);
   data.append('brcoor', brcoords);
-  const resp = await fetch('/api/modtools', {
+  const resp = await fetch(`${shardOrigin}/api/modtools`, {
     credentials: 'include',
     method: 'POST',
     body: data,
@@ -74,7 +75,7 @@ async function submitRollback(
   data.append('canvasid', canvas);
   data.append('ulcoor', tlcoords);
   data.append('brcoor', brcoords);
-  const resp = await fetch('/api/modtools', {
+  const resp = await fetch(`${shardOrigin}/api/modtools`, {
     credentials: 'include',
     method: 'POST',
     body: data,
@@ -94,7 +95,7 @@ async function submitCanvasCleaner(
   data.append('canvasid', canvas);
   data.append('ulcoor', tlcoords);
   data.append('brcoor', brcoords);
-  const resp = await fetch('/api/modtools', {
+  const resp = await fetch(`${shardOrigin}/api/modtools`, {
     credentials: 'include',
     method: 'POST',
     body: data,
@@ -107,7 +108,7 @@ async function getCleanerStats(
 ) {
   const data = new FormData();
   data.append('cleanerstat', true);
-  const resp = await fetch('/api/modtools', {
+  const resp = await fetch(`${shardOrigin}/api/modtools`, {
     credentials: 'include',
     method: 'POST',
     body: data,
@@ -125,7 +126,7 @@ async function getCleanerCancel(
 ) {
   const data = new FormData();
   data.append('cleanercancel', true);
-  const resp = await fetch('/api/modtools', {
+  const resp = await fetch(`${shardOrigin}/api/modtools`, {
     credentials: 'include',
     method: 'POST',
     body: data,

src/components/ModIIDtools.jsx

@@ -6,6 +6,7 @@ import React, { useState } from 'react';
 import { t } from 'ttag';
 
 import { parseInterval } from '../core/utils';
+import { shardOrigin } from '../store/actions/fetch';
 
 async function submitIIDAction(
   action,
@@ -31,7 +32,7 @@ async function submitIIDAction(
   data.append('reason', reason);
   data.append('time', time);
   data.append('iid', iid);
-  const resp = await fetch('/api/modtools', {
+  const resp = await fetch(`${shardOrigin}/api/modtools`, {
     credentials: 'include',
     method: 'POST',
     body: data,

src/components/ModWatchtools.jsx

@@ -9,6 +9,7 @@ import { t } from 'ttag';
 
 import copyTextToClipboard from '../utils/clipboard';
 import { parseInterval } from '../core/utils';
+import { shardOrigin } from '../store/actions/fetch';
 
 const keepState = {
   tlcoords: '',
@@ -54,7 +55,7 @@ async function submitWatchAction(
   data.append('time', time);
   data.append('iid', iid);
   try {
-    const resp = await fetch('/api/modtools', {
+    const resp = await fetch(`${shardOrigin}/api/modtools`, {
       credentials: 'include',
       method: 'POST',
       body: data,

src/core/ChatProvider.js

@@ -10,7 +10,7 @@ import {
 } from '../data/sql';
 import { findIdByNameOrId } from '../data/sql/RegUser';
 import ChatMessageBuffer from './ChatMessageBuffer';
-import socketEvents from '../socket/SocketEvents';
+import socketEvents from '../socket/socketEvents';
 import checkIPAllowed from './isAllowed';
 import { DailyCron } from '../utils/cron';
 import { escapeMd } from './utils';
@@ -93,6 +93,9 @@ export class ChatProvider {
   }
 
   async clearOldMessages() {
+    if (!socketEvents.amIImportant()) {
+      return;
+    }
     const ids = Object.keys(this.defaultChannels);
     for (let i = 0; i < ids.length; i += 1) {
       const cid = ids[i];

src/core/MailProvider.js

@@ -4,20 +4,17 @@
 
 /* eslint-disable max-len */
 
-import { randomUUID } from 'crypto';
 import nodemailer from 'nodemailer';
 
 import logger from './logger';
-import { HOUR, MINUTE } from './constants';
-import { DailyCron, HourlyCron } from '../utils/cron';
 import { getTTag } from './ttag';
+import { codeExists, checkCode, setCode } from '../data/redis/mailCodes';
+import socketEvents from '../socket/socketEvents';
 import { USE_MAILER, MAIL_ADDRESS } from './config';
 
 import { RegUser } from '../data/sql';
 
-
+export class MailProvider {
-// TODO make code expire
-class MailProvider {
   constructor() {
     this.enabled = !!USE_MAILER;
     if (this.enabled) {
@@ -26,13 +23,23 @@ class MailProvider {
         newline: 'unix',
         path: '/usr/sbin/sendmail',
       });
-
-      this.clearCodes = this.clearCodes.bind(this);
-
-      this.verifyCodes = {};
-      HourlyCron.hook(this.clearCodes);
-      DailyCron.hook(MailProvider.cleanUsers);
     }
+
+    /*
+     * mail requests make it through SocketEvents when sharding
+     */
+    socketEvents.on('mail', (type, args) => {
+      switch (type) {
+        case 'verify':
+          this.postVerifyMail(...args);
+          break;
+        case 'pwreset':
+          this.postPasswdResetMail(...args);
+          break;
+        default:
+          // nothing
+      }
+    });
   }
 
   sendMail(to, subject, html) {
@@ -52,29 +59,10 @@ class MailProvider {
     });
   }
 
-  sendVerifyMail(to, name, host, lang) {
+  postVerifyMail(to, name, host, lang, code) {
-    if (!this.enabled) {
-      return null;
-    }
-
     const { t } = getTTag(lang);
-
-    const pastMail = this.verifyCodes[to];
-    if (pastMail) {
-      const minLeft = Math.floor(
-        pastMail.timestamp / MINUTE + 2 - Date.now() / MINUTE,
-      );
-      if (minLeft > 0) {
-        logger.info(
-          `Verify mail for ${to} - already sent, ${minLeft} minutes left`,
-        );
-        return t`We already sent you a verification mail, you can request another one in ${minLeft} minutes.`;
-      }
-    }
-
     logger.info(`Sending verification mail to ${to} / ${name}`);
-    const code = this.setCode(to);
+    const verifyUrl = `${host}/api/auth/verify?token=${code}&email=${encodeURIComponent(to)}`;
-    const verifyUrl = `${host}/api/auth/verify?token=${code}`;
     const subject = t`Welcome ${name} to PixelPlanet, plese verify your mail`;
     const html = `<em>${t`Hello ${name}`}</em>,<br />
       ${t`welcome to our little community of pixelplacers, to use your account, you have to verify your mail. You can do that here: `} <a href="${verifyUrl}">${t`Click to Verify`}</a>. ${t`Or by copying following url:`}<br />${verifyUrl}\n<br />
@@ -82,27 +70,60 @@ class MailProvider {
       ${t`Thanks`}<br /><br />
       <img alt="" src="https://assets.pixelplanet.fun/tile.png" style="height:64px; width:64px" />`;
     this.sendMail(to, subject, html);
+  }
+
+  async sendVerifyMail(to, name, host, lang) {
+    if (!this.enabled && !socketEvents.isCluster) {
+      return null;
+    }
+    const { t } = getTTag(lang);
+
+    const pastCodeAge = await codeExists(to);
+    if (pastCodeAge && pastCodeAge < 180) {
+      const minLeft = Math.ceil((180 - pastCodeAge) / 60);
+      logger.info(
+        `Verify mail for ${to} - already sent, ${minLeft} minutes left`,
+      );
+      return t`We already sent you a verification mail, you can request another one in ${minLeft} minutes.`;
+    }
+
+    const code = setCode(to);
+    if (this.enabled) {
+      this.postVerifyMail(to, name, host, lang, code);
+    } else {
+      socketEvents.sendMail('verify', [to, name, host, lang, code]);
+    }
     return null;
   }
 
+  postPasswdResetMail(to, ip, host, lang, code) {
+    const { t } = getTTag(lang);
+    logger.info(`Sending Password reset mail to ${to}`);
+    const restoreUrl = `${host}/reset_password?token=${code}`;
+    const subject = t`You forgot your password for PixelPlanet? Get a new one here`;
+    const html = `<em>${t`Hello`}</em>,<br />
+      ${t`You requested to get a new password. You can change your password within the next 30min here: `} <a href="${restoreUrl}">${t`Reset Password`}</a>. ${t`Or by copying following url:`}<br />${restoreUrl}\n<br />
+      ${t`If you did not request this mail, please just ignore it (the ip that requested this mail was ${ip}).`}<br />
+      ${t`Thanks`}<br /><br />\n<img alt="" src="https://assets.pixelplanet.fun/tile.png" style="height:64px; width:64px" />`;
+    this.sendMail(to, subject, html);
+  }
+
   async sendPasswdResetMail(to, ip, host, lang) {
     const { t } = getTTag(lang);
-
+    if (!this.enabled && !socketEvents.isCluster) {
-    if (!this.enabled) {
       return t`Mail is not configured on the server`;
     }
 
-    const pastMail = this.verifyCodes[to];
+    const pastCodeAge = await codeExists(to);
-    if (pastMail) {
+    if (pastCodeAge && pastCodeAge < 180) {
-      if (Date.now() < pastMail.timestamp + 15 * MINUTE) {
+      logger.info(
-        logger.info(
+        `Password reset mail for ${to} requested by ${ip} - already sent`,
-          `Password reset mail for ${to} requested by ${ip} - already sent`,
+      );
-        );
+      return t`We already sent you a mail with instructions. Please wait before requesting another mail.`;
-        return t`We already sent you a mail with instructions. Please wait before requesting another mail.`;
-      }
     }
+
     const reguser = await RegUser.findOne({ where: { email: to } });
-    if (pastMail || !reguser) {
+    if (!reguser) {
       logger.info(
         `Password reset mail for ${to} requested by ${ip} - mail not found`,
       );
@@ -119,68 +140,20 @@ class MailProvider {
     }
     */
 
-    logger.info(`Sending Password reset mail to ${to}`);
+    const code = setCode(to);
-    const code = this.setCode(to);
+    if (this.enabled) {
-    const restoreUrl = `${host}/reset_password?token=${code}`;
+      this.postPasswdResetMail(to, ip, host, lang, code);
-    const subject = t`You forgot your password for PixelPlanet? Get a new one here`;
+    } else {
-    const html = `<em>${t`Hello`}</em>,<br />
+      socketEvents.sendMail('pwreset', [to, ip, host, lang, code]);
-      ${t`You requested to get a new password. You can change your password within the next 30min here: `} <a href="${restoreUrl}">${t`Reset Password`}</a>. ${t`Or by copying following url:`}<br />${restoreUrl}\n<br />
+    }
-      ${t`If you did not request this mail, please just ignore it (the ip that requested this mail was ${ip}).`}<br />
-      ${t`Thanks`}<br /><br />\n<img alt="" src="https://assets.pixelplanet.fun/tile.png" style="height:64px; width:64px" />`;
-    this.sendMail(to, subject, html);
     return null;
   }
 
-  setCode(email) {
+  static async verify(email, code) {
-    const code = MailProvider.createCode();
+    const ret = await checkCode(email, code);
-    this.verifyCodes[email] = {
+    if (!ret) {
-      code,
-      timestamp: Date.now(),
-    };
-    return code;
-  }
-
-  async clearCodes() {
-    const curTime = Date.now();
-    const toDelete = [];
-
-    const mails = Object.keys(this.verifyCodes);
-    for (let i = 0; i < mails.length; i += 1) {
-      const iteremail = mails[i];
-      if (curTime > this.verifyCodes[iteremail].timestamp + HOUR) {
-        toDelete.push(iteremail);
-      }
-    }
-    toDelete.forEach((email) => {
-      logger.info(`Mail Code for ${email} expired`);
-      delete this.verifyCodes[email];
-    });
-  }
-
-  // Note: code gets deleted on check
-  checkCode(code) {
-    let email = null;
-    const mails = Object.keys(this.verifyCodes);
-    for (let i = 0; i < mails.length; i += 1) {
-      const iteremail = mails[i];
-      if (this.verifyCodes[iteremail].code === code) {
-        email = iteremail;
-        break;
-      }
-    }
-    if (!email) {
-      logger.info(`Mail Code ${code} not found.`);
       return false;
     }
-    logger.info(`Got Mail Code from ${email}.`);
-    delete this.verifyCodes[email];
-    return email;
-  }
-
-  async verify(code) {
-    const email = this.checkCode(code);
-    if (!email) return false;
-
     const reguser = await RegUser.findOne({ where: { email } });
     if (!reguser) {
       logger.error(`${email} does not exist in database`);
@@ -193,13 +166,10 @@ class MailProvider {
     return reguser.name;
   }
 
-  static createCode() {
+  /*
-    return randomUUID();
+   * we do not use this right now
-  }
-
   static cleanUsers() {
     // delete users that requier verification for more than 4 days
-    /*
     RegUser.destroy({
       where: {
         verificationReqAt: {
@@ -209,8 +179,8 @@ class MailProvider {
         verified: 0,
       },
     });
-    */
   }
+  */
 }
 
 const mailProvider = new MailProvider();

src/core/PixelCache.js

@@ -3,7 +3,7 @@
  * in bursts per chunk
  */
 
-import socketEvents from '../socket/SocketEvents';
+import socketEvents from '../socket/socketEvents';
 
 class PixelCache {
   PXL_CACHE;

src/core/Tile.js

@@ -242,6 +242,14 @@ function addIndexedSubtiletoTile(
 function tileFileName(canvasTileFolder, cell) {
   const [z, x, y] = cell;
   const filename = `${canvasTileFolder}/${z}/${x}/${y}.webp`;
+  try {
+    const mtime = new Date(fs.statSync(filename).mtime).getTime();
+    if (Date.now() - mtime < 120000) {
+      return null;
+    }
+  } catch {
+    // file doesn't exist
+  }
   return filename;
 }
 
@@ -263,6 +271,10 @@ export async function createZoomTileFromChunk(
   const canvasSize = canvas.size;
   const [x, y] = cell;
   const maxTiledZoom = getMaxTiledZoom(canvasSize);
+
+  const filename = tileFileName(canvasTileFolder, [maxTiledZoom - 1, x, y]);
+  if (!filename) return true;
+
   const tileRGBBuffer = new Uint8Array(
     TILE_SIZE * TILE_SIZE * TILE_ZOOM_LEVEL * TILE_ZOOM_LEVEL * 3,
   );
@@ -318,7 +330,6 @@ export async function createZoomTileFromChunk(
       );
     });
 
-    const filename = tileFileName(canvasTileFolder, [maxTiledZoom - 1, x, y]);
     try {
       await sharp(tileRGBBuffer, {
         raw: {
@@ -363,6 +374,9 @@ export async function createZoomedTile(
   );
   const [z, x, y] = cell;
 
+  const filename = tileFileName(canvasTileFolder, [z, x, y]);
+  if (!filename) return true;
+
   const startTime = Date.now();
   const na = [];
 
@@ -409,7 +423,6 @@ export async function createZoomedTile(
       );
     });
 
-    const filename = tileFileName(canvasTileFolder, [z, x, y]);
     try {
       await sharp(tileRGBBuffer, {
         raw: {

src/core/Void.js

@@ -5,7 +5,7 @@
  * if it reaches the TARGET_RADIUS size, the event is lost
  *
  */
-import socketEvents from '../socket/SocketEvents';
+import socketEvents from '../socket/socketEvents';
 import PixelUpdate from '../socket/packets/PixelUpdateServer';
 import { setPixelByOffset } from './setPixel';
 import { TILE_SIZE } from './constants';

src/core/config.js

@@ -20,7 +20,7 @@ export const MAIL_ADDRESS = process.env.MAIL_ADDRESS
 const TILE_FOLDER_REL = process.env.TILE_FOLDER || 'tiles';
 export const TILE_FOLDER = path.join(__dirname, `./${TILE_FOLDER_REL}`);
 
-export const USE_XREALIP = process.env.USE_XREALIP || false;
+export const USE_XREALIP = !!process.env.USE_XREALIP;
 
 export const BACKUP_URL = process.env.BACKUP_URL || null;
 export const BACKUP_DIR = process.env.BACKUP_DIR || null;
@@ -30,6 +30,7 @@ export const USE_PROXYCHECK = parseInt(process.env.USE_PROXYCHECK, 10) || false;
 export const { PROXYCHECK_KEY } = process.env;
 
 export const REDIS_URL = process.env.REDIS_URL || 'redis://localhost:6379';
+export const SHARD_NAME = process.env.SHARD_NAME || null;
 // Database
 export const MYSQL_HOST = process.env.MYSQL_HOST || 'localhost';
 export const MYSQL_DATABASE = process.env.MYSQL_DATABASE || 'pixelplanet';
@@ -52,6 +53,9 @@ export const APISOCKET_KEY = process.env.APISOCKET_KEY || null;
 export const ADMIN_IDS = (process.env.ADMIN_IDS)
   ? process.env.ADMIN_IDS.split(',').map((z) => parseInt(z, 10)) : [];
 
+export const CORS_HOSTS = (process.env.CORS_HOSTS)
+  ? process.env.CORS_HOSTS.split(',') : [];
+
 export const auth = {
   // https://developers.facebook.com/
   facebook: {

src/core/ranking.js

@@ -6,6 +6,7 @@ import Sequelize from 'sequelize';
 import sequelize from '../data/sql/sequelize';
 import RegUser from '../data/sql/RegUser';
 import { saveDailyTop, loadDailyTop } from '../data/redis/PrevDayTop';
+import socketEvents from '../socket/socketEvents';
 import logger from './logger';
 
 import { MINUTE } from './constants';
@@ -33,15 +34,17 @@ class Ranks {
 
   async updateRanking() {
     logger.info('Update pixel rankings');
-    // recalculate ranking column
+    if (socketEvents.amIImportant()) {
-    await sequelize.query(
+      // recalculate ranking column
-      // eslint-disable-next-line max-len
+      await sequelize.query(
-      'SET @r=0; UPDATE Users SET ranking= @r:= (@r + 1) ORDER BY totalPixels DESC;',
+        // eslint-disable-next-line max-len
-    );
+        'SET @r=0; UPDATE Users SET ranking= @r:= (@r + 1) ORDER BY totalPixels DESC;',
-    await sequelize.query(
+      );
-      // eslint-disable-next-line max-len
+      await sequelize.query(
-      'SET @r=0; UPDATE Users SET dailyRanking= @r:= (@r + 1) ORDER BY dailyTotalPixels DESC;',
+        // eslint-disable-next-line max-len
-    );
+        'SET @r=0; UPDATE Users SET dailyRanking= @r:= (@r + 1) ORDER BY dailyTotalPixels DESC;',
+      );
+    }
     // populate dictionaries
     const ranking = await RegUser.findAll({
       attributes: [
@@ -92,6 +95,9 @@ class Ranks {
   }
 
   async resetDailyRanking() {
+    if (!socketEvents.amIImportant()) {
+      return;
+    }
     this.prevTop = await saveDailyTop(this.ranks.dailyRanking);
     logger.info('Resetting Daily Ranking');
     await RegUser.update({ dailyTotalPixels: 0 }, { where: {} });

src/core/session.js

@@ -5,26 +5,36 @@ import expressSession from 'express-session';
 import RedisStore from '../utils/connectRedis';
 
 import client from '../data/redis/client';
+import { getHostFromRequest } from '../utils/ip';
 import { HOUR, COOKIE_SESSION_NAME } from './constants';
 import { SESSION_SECRET } from './config';
 
 
 export const store = new RedisStore({ client });
 
-const session = expressSession({
+/*
-  name: COOKIE_SESSION_NAME,
+ * we cache created session middlewares per domain
-  store,
+ */
-  secret: SESSION_SECRET,
+const middlewareCache = {};
-  // The best way to know is to check with your store if it implements the touch method. If it does, then you can safely set resave: false
-  resave: false,
-  saveUninitialized: false,
-  cookie: {
-    path: '/',
-    httpOnly: true,
-    secure: false,
-    // not setting maxAge or expire makes it a non-persisting cookies
-    maxAge: 30 * 24 * HOUR,
-  },
-});
 
-export default session;
+export default (req, res, next) => {
+  const domain = getHostFromRequest(req, false, true);
+  let session = middlewareCache[domain];
+  if (!session) {
+    session = expressSession({
+      name: COOKIE_SESSION_NAME,
+      store,
+      secret: SESSION_SECRET,
+      resave: false,
+      saveUninitialized: false,
+      cookie: {
+        domain,
+        httpOnly: true,
+        secure: false,
+        maxAge: 30 * 24 * HOUR,
+      },
+    });
+    middlewareCache[domain] = session;
+  }
+  return session(req, res, next);
+};

src/core/tileserver.js

@@ -8,7 +8,7 @@ import { Worker } from 'worker_threads';
 
 import logger from './logger';
 import canvases from './canvases';
-import RedisCanvas from '../data/redis/RedisCanvas';
+import socketEvents from '../socket/socketEvents';
 
 import { TILE_FOLDER } from './config';
 import {
@@ -181,17 +181,16 @@ class CanvasUpdater {
   }
 }
 
-export function registerChunkChange(canvasId, chunk) {
+socketEvents.on('chunkUpdate', (canvasId, chunk) => {
   if (CanvasUpdaters[canvasId]) {
     CanvasUpdaters[canvasId].registerChunkChange(chunk);
   }
-}
+});
-RedisCanvas.setChunkChangeCallback(registerChunkChange);
 
 /*
  * starting update loops for canvases
  */
-export async function startAllCanvasLoops() {
+export default function startAllCanvasLoops() {
   if (!fs.existsSync(`${TILE_FOLDER}`)) fs.mkdirSync(`${TILE_FOLDER}`);
   const ids = Object.keys(canvases);
   for (let i = 0; i < ids.length; i += 1) {

src/data/redis/RedisCanvas.js

@@ -4,24 +4,13 @@
 import { commandOptions } from 'redis';
 
 import { getChunkOfPixel, getOffsetOfPixel } from '../../core/utils';
+import socketEvents from '../../socket/socketEvents';
 import client from './client';
 
 
 const UINT_SIZE = 'u8';
 
 class RedisCanvas {
-  // array of callback functions that gets informed about chunk changes
-  static registerChunkChange = [];
-  static setChunkChangeCallback(cb) {
-    RedisCanvas.registerChunkChange.push(cb);
-  }
-
-  static execChunkChangeCallback(canvasId, cell) {
-    for (let i = 0; i < RedisCanvas.registerChunkChange.length; i += 1) {
-      RedisCanvas.registerChunkChange[i](canvasId, cell);
-    }
-  }
-
   /*
    * Get chunk from redis
    * canvasId integer id of canvas
@@ -56,14 +45,14 @@ class RedisCanvas {
   static async setChunk(i, j, chunk, canvasId) {
     const key = `ch:${canvasId}:${i}:${j}`;
     await client.set(key, Buffer.from(chunk.buffer));
-    RedisCanvas.execChunkChangeCallback(canvasId, [i, j]);
+    socketEvents.broadcastChunkUpdate(canvasId, [i, j]);
     return true;
   }
 
   static async delChunk(i, j, canvasId) {
     const key = `ch:${canvasId}:${i}:${j}`;
     await client.del(key);
-    RedisCanvas.execChunkChangeCallback(canvasId, [i, j]);
+    socketEvents.broadcastChunkUpdate(canvasId, [i, j]);
     return true;
   }
 
@@ -97,8 +86,6 @@ class RedisCanvas {
         String(color),
       ],
     );
-
-    RedisCanvas.execChunkChangeCallback(canvasId, [i, j]);
   }
 
   static flushPixels() {

src/data/redis/client.js

@@ -4,13 +4,14 @@
  */
 import fs from 'fs';
 import { createClient, defineScript } from 'redis';
+import { isMainThread } from 'worker_threads';
 
-import { REDIS_URL } from '../../core/config';
+import { REDIS_URL, SHARD_NAME } from '../../core/config';
 
 const scripts = {
   placePxl: defineScript({
     NUMBER_OF_KEYS: 5,
-    SCRIPT: fs.readFileSync('./workers/placePixel.lua'),
+    SCRIPT: fs.readFileSync('./workers/lua/placePixel.lua'),
     transformArguments(...args) {
       return args.map((a) => ((typeof a === 'string') ? a : a.toString()));
     },
@@ -30,10 +31,24 @@ const client = createClient(REDIS_URL.startsWith('redis://')
   },
 );
 
+/*
+ * for sending messages via cluster
+ */
+export const pubsub = {
+  subscriber: null,
+  publisher: null,
+};
+
 export const connect = async () => {
   // eslint-disable-next-line no-console
   console.log(`Connecting to redis server at ${REDIS_URL}`);
   await client.connect();
+  if (SHARD_NAME && isMainThread) {
+    const subscriber = client.duplicate();
+    await subscriber.connect();
+    pubsub.publisher = client;
+    pubsub.subscriber = subscriber;
+  }
 };
 
 export default client;

src/data/redis/mailCodes.js

@@ -0,0 +1,54 @@
+/*
+ *
+ * data saving for hourly events
+ *
+ */
+import { randomUUID } from 'crypto';
+
+import client from './client';
+
+export const PREFIX = 'mail';
+const EXPIRE_TIME = 3600;
+
+/*
+ * generate and set mail code
+ * @param email
+ * @return code
+ */
+export function setCode(email) {
+  const code = randomUUID();
+  const key = `${PREFIX}:${email}`;
+  client.set(key, code, {
+    EX: EXPIRE_TIME,
+  });
+  return code;
+}
+
+/*
+ * check if email code is correct
+ * @param email
+ * @param code
+ */
+export async function checkCode(email, code) {
+  const key = `${PREFIX}:${email}`;
+  const storedCode = await client.get(key);
+  if (!storedCode || code !== storedCode) {
+    return false;
+  }
+  client.del(key);
+  return true;
+}
+
+/*
+ * check if code exists
+ * @param email
+ * @return null if doesn't, age in seconds if exists
+ */
+export async function codeExists(email) {
+  const key = `${PREFIX}:${email}`;
+  const ttl = await client.ttl(key);
+  if (!ttl) {
+    return null;
+  }
+  return EXPIRE_TIME - ttl;
+}

src/routes/api/auth/change_mail.js

@@ -2,7 +2,7 @@
  * request password change
  */
 
-import mailProvider from '../../../core/mail';
+import mailProvider from '../../../core/MailProvider';
 
 import { validatePassword, validateEMail } from '../../../utils/validation';
 import { getHostFromRequest } from '../../../utils/ip';

src/routes/api/auth/change_name.js

@@ -3,7 +3,7 @@
  */
 
 
-import socketEvents from '../../../socket/SocketEvents';
+import socketEvents from '../../../socket/socketEvents';
 import { RegUser } from '../../../data/sql';
 import { validateName } from '../../../utils/validation';
 

src/routes/api/auth/delete_account.js

@@ -2,6 +2,7 @@
  * request password change
  */
 
+import socketEvents from '../../../socket/socketEvents';
 import { RegUser } from '../../../data/sql';
 import { validatePassword } from '../../../utils/validation';
 import { compareToHash } from '../../../utils/hash';
@@ -35,7 +36,7 @@ export default async (req, res) => {
     });
     return;
   }
-  const { id } = user;
+  const { id, name } = user;
 
   const currentPassword = user.regUser.password;
   if (!currentPassword || !compareToHash(password, currentPassword)) {
@@ -58,6 +59,8 @@ export default async (req, res) => {
 
     RegUser.destroy({ where: { id } });
 
+    socketEvents.reloadUser(name);
+
     res.status(200);
     res.json({
       success: true,

src/routes/api/auth/register.js

@@ -2,7 +2,7 @@ import Sequelize from 'sequelize';
 
 import logger from '../../../core/logger';
 import { RegUser } from '../../../data/sql';
-import mailProvider from '../../../core/mail';
+import mailProvider from '../../../core/MailProvider';
 import getMe from '../../../core/me';
 import { getIPFromRequest, getHostFromRequest } from '../../../utils/ip';
 import {

src/routes/api/auth/resend_verify.js

@@ -2,7 +2,7 @@
  * request resend of verification mail
  */
 
-import mailProvider from '../../../core/mail';
+import mailProvider from '../../../core/MailProvider';
 import { getHostFromRequest } from '../../../utils/ip';
 
 export default async (req, res) => {
@@ -26,7 +26,7 @@ export default async (req, res) => {
 
   const host = getHostFromRequest(req);
 
-  const error = mailProvider.sendVerifyMail(email, name, host, lang);
+  const error = await mailProvider.sendVerifyMail(email, name, host, lang);
   if (error) {
     res.status(400);
     res.json({

src/routes/api/auth/restore_password.js

@@ -3,7 +3,7 @@
  */
 
 
-import mailProvider from '../../../core/mail';
+import mailProvider from '../../../core/MailProvider';
 import { validateEMail } from '../../../utils/validation';
 import { getHostFromRequest } from '../../../utils/ip';
 

src/routes/api/auth/verify.js

@@ -2,31 +2,36 @@
  * verify mail address
  */
 
-import socketEvents from '../../../socket/SocketEvents';
+import socketEvents from '../../../socket/socketEvents';
 import getHtml from '../../../ssr/RedirectionPage';
 import { getHostFromRequest } from '../../../utils/ip';
-import mailProvider from '../../../core/mail';
+import { MailProvider } from '../../../core/MailProvider';
+import { validateEMail } from '../../../utils/validation';
 
 export default async (req, res) => {
-  const { token } = req.query;
+  const { email, token } = req.query;
   const { lang } = req;
   const { t } = req.ttag;
-  const name = await mailProvider.verify(token);
+
   const host = getHostFromRequest(req);
-  if (name) {
+  const error = validateEMail(email);
-    // notify websoecket to reconnect user
+  if (!error) {
-    // thats a bit counter productive because it directly links to the websocket
+    const name = await MailProvider.verify(email, token);
-    socketEvents.reloadUser(name);
+    if (name) {
-    // ---
+      // notify websoecket to reconnect user
-    const index = getHtml(
+      // thats a bit counter productive because it directly links to the websocket
-      t`Mail verification`,
+      socketEvents.reloadUser(name);
-      t`You are now verified :)`,
+      // ---
-      host, lang,
+      const index = getHtml(
-    );
+        t`Mail verification`,
-    res.status(200).send(index);
+        t`You are now verified :)`,
-  } else {
+        host, lang,
-    // eslint-disable-next-line max-len
+      );
-    const index = getHtml(t`Mail verification`, t`Your mail verification code is invalid or already expired :(, please request a new one.`, host, lang);
+      res.status(200).send(index);
-    res.status(400).send(index);
+      return;
+    }
   }
+  // eslint-disable-next-line max-len
+  const index = getHtml(t`Mail verification`, t`Your mail verification code is invalid or already expired :(, please request a new one.`, host, lang);
+  res.status(400).send(index);
 };

src/routes/api/block.js

@@ -5,7 +5,7 @@
  */
 
 import logger from '../../core/logger';
-import socketEvents from '../../socket/SocketEvents';
+import socketEvents from '../../socket/socketEvents';
 import { RegUser, UserBlock, Channel } from '../../data/sql';
 
 async function block(req, res) {

src/routes/api/blockdm.js

@@ -4,7 +4,7 @@
  *
  */
 import logger from '../../core/logger';
-import socketEvents from '../../socket/SocketEvents';
+import socketEvents from '../../socket/socketEvents';
 
 async function blockdm(req, res) {
   const { block } = req.body;

src/routes/api/chathistory.js

@@ -7,11 +7,6 @@ import chatProvider from '../../core/ChatProvider';
 
 async function chatHistory(req, res) {
   let { cid, limit } = req.query;
-  res.set({
-    'Cache-Control': 'no-cache, no-store, must-revalidate',
-    Pragma: 'no-cache',
-    Expires: '0',
-  });
 
   if (!cid || !limit) {
     res.status(400);
@@ -42,19 +37,10 @@ async function chatHistory(req, res) {
     return;
   }
 
-  // try {
   const history = await chatProvider.getHistory(cid, limit);
   res.json({
     history,
   });
-  /*
-  } catch {
-    res.status(500);
-    res.json({
-      errors: ['Can not fetch messages'],
-    });
-  }
-  */
 }
 
 export default chatHistory;

src/routes/api/index.js

@@ -17,10 +17,19 @@ import blockdm from './blockdm';
 import modtools from './modtools';
 import baninfo from './baninfo';
 import getiid from './getiid';
-
+import shards from './shards';
 
 const router = express.Router();
 
+// set cache-control
+router.use((req, res, next) => {
+  res.set({
+    'Cache-Control': 'no-cache, no-store, must-revalidate',
+    Expires: '0',
+  });
+  next();
+});
+
 router.use(express.json());
 
 // eslint-disable-next-line no-unused-vars
@@ -35,6 +44,7 @@ router.use((err, req, res, next) => {
 router.post('/captcha', captcha);
 router.get('/baninfo', baninfo);
 router.get('/getiid', getiid);
+router.get('/shards', shards);
 
 /*
  * get user session

src/routes/api/leavechan.js

@@ -5,7 +5,7 @@
  */
 
 import logger from '../../core/logger';
-import socketEvents from '../../socket/SocketEvents';
+import socketEvents from '../../socket/socketEvents';
 
 async function leaveChan(req, res) {
   const channelId = parseInt(req.body.channelId, 10);

src/routes/api/me.js

@@ -24,10 +24,6 @@ export default async (req, res, next) => {
     }
 
     // https://stackoverflow.com/questions/49547/how-to-control-web-page-caching-across-all-browsers
-    res.set({
-      'Cache-Control': 'no-cache, no-store, must-revalidate',
-      Expires: '0',
-    });
     res.json(userdata);
   } catch (error) {
     next(error);

src/routes/api/shards.js

@@ -0,0 +1,20 @@
+/*
+ * print information for shards
+ */
+import socketEvents from '../../socket/socketEvents';
+
+async function shards(req, res, next) {
+  try {
+    if (!socketEvents.isCluster) {
+      res.status(400).json({
+        errors: ['Not running as cluster'],
+      });
+      return;
+    }
+    res.status(200).json(socketEvents.shardOnlineCounters);
+  } catch (err) {
+    next(err);
+  }
+}
+
+export default shards;

src/routes/chunks.js

@@ -7,11 +7,11 @@
 import etag from 'etag';
 import RedisCanvas from '../data/redis/RedisCanvas';
 import logger from '../core/logger';
+import socketEvents from '../socket/socketEvents';
 
 const chunkEtags = new Map();
-RedisCanvas.setChunkChangeCallback((canvasId, cell) => {
+socketEvents.on('chunkUpdate', (canvasId, [i, j]) => {
-  const [x, y] = cell;
+  chunkEtags.delete(`${canvasId}:${i}:${j}`);
-  chunkEtags.delete(`${canvasId}:${x}:${y}`);
 });
 
 /*
@@ -27,6 +27,9 @@ export default async (req, res, next) => {
   const x = parseInt(paramX, 10);
   const y = parseInt(paramY, 10);
   try {
+    res.set({
+      'Access-Control-allow-origin': '*',
+    });
     // botters where using cachebreakers to update via chunk API
     // lets not allow that for now
     if (Object.keys(req.query).length !== 0) {

src/routes/index.js

@@ -18,6 +18,7 @@ import api from './api';
 
 import { assets } from '../core/assets';
 import { expressTTag } from '../core/ttag';
+import corsMiddleware from '../utils/corsMiddleware';
 import generateGlobePage from '../ssr/Globe';
 import generatePopUpPage from '../ssr/PopUp';
 import generateMainPage from '../ssr/Main';
@@ -29,37 +30,18 @@ import { GUILDED_INVITE } from '../core/config';
 const router = express.Router();
 
 /*
- * void info
+ * Serving Chunks
  */
-router.get('/void', voidl);
+router.get(
-
+  '/chunks/:c([0-9]+)/:x([0-9]+)/:y([0-9]+)(/)?:z([0-9]+)?.bmp',
-/*
+  chunks,
- * ranking of pixels placed
+);
- * daily and total
- */
-router.get('/ranking', ranking);
-
-/*
- * give: date per query
- * returns: array of HHMM backups available
- */
-router.get('/history', history);
 
 /*
  * zoomed tiles
  */
 router.use('/tiles', tiles);
 
-/*
- * adminapi
- */
-router.use('/adminapi', adminapi);
-
-/*
- * serve captcha
- */
-router.get('/captcha.svg', captcha);
-
 /*
  * public folder
  * (this should be served with nginx or other webserver)
@@ -77,12 +59,9 @@ router.use('/guilded', (req, res) => {
 });
 
 /*
- * Serving Chunks
+ * adminapi
  */
-router.get(
+router.use('/adminapi', adminapi);
-  '/chunks/:c([0-9]+)/:x([0-9]+)/:y([0-9]+)(/)?:z([0-9]+)?.bmp',
-  chunks,
-);
 
 /*
  * Following with translations
@@ -90,16 +69,6 @@ router.get(
  */
 router.use(expressTTag);
 
-/*
- * API calls
- */
-router.use('/api', api);
-
-/*
- * Password Reset Link
- */
-router.use('/reset_password', resetPassword);
-
 //
 // 3D Globe (react generated)
 // -----------------------------------------------------------------------------
@@ -149,7 +118,7 @@ router.use(
       return;
     }
 
-    res.status(200).send(generatePopUpPage(req.lang));
+    res.status(200).send(generatePopUpPage(req));
   },
 );
 
@@ -173,7 +142,47 @@ router.get('/', (req, res) => {
     return;
   }
 
-  res.status(200).send(generateMainPage(req.lang));
+  res.status(200).send(generateMainPage(req));
 });
 
+
+/*
+ * Password Reset Link
+ */
+router.use('/reset_password', resetPassword);
+
+/*
+ * Following with CORS
+ * ---------------------------------------------------------------------------
+ */
+router.use(corsMiddleware);
+
+/*
+ * API calls
+ */
+router.use('/api', api);
+
+/*
+ * void info
+ */
+router.get('/void', voidl);
+
+/*
+ * ranking of pixels placed
+ * daily and total
+ */
+router.get('/ranking', ranking);
+
+/*
+ * give: date per query
+ * returns: array of HHMM backups available
+ */
+router.get('/history', history);
+
+/*
+ * serve captcha
+ */
+router.get('/captcha.svg', captcha);
+
+
 export default router;

src/routes/reset_password.js

@@ -8,7 +8,7 @@ import express from 'express';
 import logger from '../core/logger';
 import getPasswordResetHtml from '../ssr/PasswordReset';
 
-import mailProvider from '../core/mail';
+import mailProvider from '../core/MailProvider';
 import { RegUser } from '../data/sql';
 
 

src/routes/tiles.js

@@ -19,6 +19,9 @@ const router = express.Router();
  */
 router.use('/:c([0-9]+)/:z([0-9]+)/:x([0-9]+)/:y([0-9]+).webp',
   (req, res, next) => {
+    res.set({
+      'Access-Control-allow-origin': '*',
+    });
     const { c: id } = req.params;
     const canvas = canvases[id];
     if (!canvas) {

src/server.js

@@ -17,14 +17,16 @@ import chatProvider from './core/ChatProvider';
 import rpgEvent from './core/RpgEvent';
 import canvasCleaner from './core/CanvasCleaner';
 
+import socketEvents from './socket/socketEvents';
 import SocketServer from './socket/SocketServer';
 import APISocketServer from './socket/APISocketServer';
 
-
+import {
-import { PORT, HOST, HOURLY_EVENT } from './core/config';
+  PORT, HOST, HOURLY_EVENT, SHARD_NAME,
+} from './core/config';
 import { SECOND } from './core/constants';
 
-import { startAllCanvasLoops } from './core/tileserver';
+import startAllCanvasLoops from './core/tileserver';
 
 const app = express();
 app.disable('x-powered-by');
@@ -78,15 +80,11 @@ app.use(routes);
 sequelize.sync({ alter: { drop: false } })
   // connect to redis
   .then(connectRedis)
-  .then(() => {
+  .then(async () => {
-    rankings.initialize();
     chatProvider.initialize();
     startAllCanvasLoops();
     usersocket.initialize();
     apisocket.initialize();
-    if (HOURLY_EVENT) {
-      rpgEvent.initialize();
-    }
     canvasCleaner.initialize();
     // start http server
     const startServer = () => {
@@ -108,4 +106,22 @@ sequelize.sync({ alter: { drop: false } })
         startServer();
       }, 5000);
     });
+  })
+  .then(async () => {
+    await socketEvents.initialize();
+  })
+  .then(async () => {
+    /*
+     * initializers that rely on the cluster being fully established
+     * i.e. to know if it is the shard that runs the event
+     */
+    if (socketEvents.isCluster && socketEvents.amIImportant()) {
+      logger.info('I am the main shard');
+    }
+    rankings.initialize();
+    if (HOURLY_EVENT && !SHARD_NAME) {
+      // TODO make it wok in a cluster
+      logger.info('Initializing RpgEvent');
+      rpgEvent.initialize();
+    }
   });

src/socket/APISocketServer.js

@@ -9,7 +9,7 @@
 
 import WebSocket from 'ws';
 
-import socketEvents from './SocketEvents';
+import socketEvents from './socketEvents';
 import chatProvider, { ChatProvider } from '../core/ChatProvider';
 import { RegUser } from '../data/sql';
 import { getIPFromRequest } from '../utils/ip';
@@ -61,7 +61,6 @@ class APISocketServer {
     this.ping = this.ping.bind(this);
     this.broadcastChatMessage = this.broadcastChatMessage.bind(this);
 
-    socketEvents.onAsync('broadcast', this.broadcast);
     socketEvents.onAsync('onlineCounter', this.broadcastOnlineCounter);
     socketEvents.onAsync('pixelUpdate', this.broadcastPixelBuffer);
     socketEvents.onAsync('chatMessage', this.broadcastChatMessage);

src/socket/MessageBroker.js

@@ -0,0 +1,254 @@
+/*
+ * sends messages to other ppfun instances
+ * to work as cluster
+ */
+
+/* eslint-disable no-console */
+
+import { SHARD_NAME } from '../core/config';
+import SocketEvents from './SockEvents';
+import OnlineCounter from './packets/OnlineCounter';
+import PixelUpdate from './packets/PixelUpdateServer';
+import PixelUpdateMB from './packets/PixelUpdateMB';
+import ChunkUpdate from './packets/ChunkUpdate';
+import { pubsub } from '../data/redis/client';
+
+
+class MessageBroker extends SocketEvents {
+  constructor() {
+    super();
+    this.isCluster = true;
+    this.thisShard = SHARD_NAME;
+    /*
+     * all other shards
+     */
+    this.shards = {};
+    /*
+     * online counter of all shards including ourself
+     */
+    this.shardOnlineCounters = [];
+    this.publisher = {
+      publish: () => {},
+    };
+    this.subscriber = {
+      subscribe: () => {},
+      unsubscribe: () => {},
+    };
+    this.checkHealth = this.checkHealth.bind(this);
+    setInterval(this.checkHealth, 10000);
+  }
+
+  // TODO imprement shared storage that is run by main shard
+
+  async initialize() {
+    /*
+     * broadcast channel for staus messages between shards
+     */
+    this.publisher = pubsub.publisher;
+    this.subscriber = pubsub.subscriber;
+    await this.subscriber.subscribe('bc', (...args) => {
+      this.onShardBCMessage(...args);
+    });
+    // give other shards 30s to announce themselves
+    await new Promise((resolve) => {
+      setTimeout(resolve, 25000);
+    });
+    console.log('CLUSTER: Initialized message broker');
+  }
+
+  async onShardBCMessage(message) {
+    try {
+      /*
+       * messages from own shard get dropped
+       */
+      if (!message || message.startsWith(this.thisShard)) {
+        return;
+      }
+      const comma = message.indexOf(',');
+      /*
+       * any other package in the form of 'shard:type,JSONArrayData'
+       * straight sent over websocket
+       */
+      if (~comma) {
+        console.log('CLUSTER: Broadcast', message);
+        const key = message.slice(message.indexOf(':') + 1, comma);
+        const val = JSON.parse(message.slice(comma + 1));
+        super.emit(key, ...val);
+        return;
+      }
+      if (!this.shards[message]) {
+        console.log(`CLUSTER: Shard ${message} connected`);
+        await this.subscriber.subscribe(
+          message,
+          (buffer) => this.onShardBinaryMessage(buffer, message),
+          true,
+        );
+        // immediately give new shards informations
+        this.publisher.publish('bc', this.thisShard);
+      }
+      this.shards[message] = Date.now();
+      return;
+    } catch (err) {
+      console.error(`CLUSTER: Error on broadcast message: ${err.message}`);
+    }
+  }
+
+  getLowestActiveShard() {
+    let lowest = 0;
+    let lShard = null;
+    this.shardOnlineCounters.forEach((shardData) => {
+      const [shard, cnt] = shardData;
+      if (cnt.total < lowest || !lShard) {
+        lShard = shard;
+        lowest = cnt.total;
+      }
+    });
+    return lShard || this.thisShard;
+  }
+
+  amIImportant() {
+    /*
+     * important main shard does tasks like running RpgEvent
+     * or updating rankings
+     */
+    return !this.shardOnlineCounters[0]
+      || this.shardOnlineCounters[0][0] === this.thisShard;
+  }
+
+  updateShardOnlineCounter(shard, cnt) {
+    const shardCounter = this.shardOnlineCounters.find(
+      (c) => c[0] === shard,
+    );
+    if (!shardCounter) {
+      this.shardOnlineCounters.push([shard, cnt]);
+      this.shardOnlineCounters.sort((a, b) => a[0].localeCompare(b[0]));
+    } else {
+      shardCounter[1] = cnt;
+    }
+    this.sumOnlineCounters();
+  }
+
+  onShardBinaryMessage(buffer, shard) {
+    if (buffer.byteLength === 0) return;
+    const opcode = buffer[0];
+    try {
+      switch (opcode) {
+        case PixelUpdateMB.OP_CODE: {
+          const puData = PixelUpdateMB.hydrate(buffer);
+          super.emit('pixelUpdate', ...puData);
+          const chunkId = puData[1];
+          const chunk = [chunkId >> 8, chunkId & 0xFF];
+          super.emit('chunkUpdate', puData[0], chunk);
+          break;
+        }
+        case ChunkUpdate.OP_CODE: {
+          super.emit('chunkUpdate', ...ChunkUpdate.hydrate(buffer));
+          break;
+        }
+        case OnlineCounter.OP_CODE: {
+          const data = new DataView(
+            buffer.buffer,
+            buffer.byteOffset,
+            buffer.length,
+          );
+          const cnt = OnlineCounter.hydrate(data);
+          this.updateShardOnlineCounter(shard, cnt);
+          break;
+        }
+        default:
+          // nothing
+      }
+    } catch (err) {
+      // eslint-disable-next-line max-len
+      console.error(`CLUSTER: Error on binery message of shard ${shard}: ${err.message}`);
+    }
+  }
+
+  sumOnlineCounters() {
+    const newCounter = {};
+    this.shardOnlineCounters.forEach((shardData) => {
+      const [, cnt] = shardData;
+      Object.keys(cnt).forEach((canv) => {
+        const num = cnt[canv];
+        if (newCounter[canv]) {
+          newCounter[canv] += num;
+        } else {
+          newCounter[canv] = num;
+        }
+      });
+    });
+    this.onlineCounter = newCounter;
+  }
+
+  emit(key, ...args) {
+    super.emit(key, ...args);
+    const msg = `${this.thisShard}:${key},${JSON.stringify(args)}`;
+    this.publisher.publish('bc', msg);
+  }
+
+  /*
+   * broadcast pixel message via websocket
+   * @param canvasId number ident of canvas
+   * @param chunkid number id consisting of i,j chunk coordinates
+   * @param pxls buffer with offset and color of one or more pixels
+   */
+  broadcastPixels(
+    canvasId,
+    chunkId,
+    pixels,
+  ) {
+    const i = chunkId >> 8;
+    const j = chunkId & 0xFF;
+    this.publisher.publish(
+      this.thisShard,
+      PixelUpdateMB.dehydrate(canvasId, i, j, pixels),
+    );
+    const buffer = PixelUpdate.dehydrate(i, j, pixels);
+    super.emit('pixelUpdate', canvasId, chunkId, buffer);
+    super.emit('chunkUpdate', canvasId, [i, j]);
+  }
+
+  broadcastChunkUpdate(
+    canvasId,
+    chunk,
+  ) {
+    this.publisher.publish(
+      this.thisShard,
+      ChunkUpdate.dehydrate(canvasId, chunk),
+    );
+    super.emit('chunkUpdate', canvasId, chunk);
+  }
+
+  broadcastOnlineCounter(online) {
+    this.updateShardOnlineCounter(this.thisShard, online);
+    let buffer = OnlineCounter.dehydrate(online);
+    // send our online counter to other shards
+    this.publisher.publish(this.thisShard, buffer);
+    // send total counter to our players
+    buffer = OnlineCounter.dehydrate(this.onlineCounter);
+    super.emit('onlineCounter', buffer);
+  }
+
+  checkHealth() {
+    // remove disconnected shards
+    const threshold = Date.now() - 30000;
+    const { shards } = this;
+    Object.keys(shards).forEach((shard) => {
+      if (shards[shard] < threshold) {
+        console.log(`CLUSTER: Shard ${shard} disconnected`);
+        delete shards[shard];
+        const counterIndex = this.shardOnlineCounters.findIndex(
+          (c) => c[0] === shard,
+        );
+        if (~counterIndex) {
+          this.shardOnlineCounters.splice(counterIndex, 1);
+        }
+        this.subscriber.unsubscribe(shard);
+      }
+    });
+    // send keep alive to others
+    this.publisher.publish('bc', this.thisShard);
+  }
+}
+
+export default MessageBroker;

src/socket/SockEvents.js

@@ -22,6 +22,21 @@ class SocketEvents extends EventEmitter {
     };
   }
 
+  // eslint-disable-next-line class-methods-use-this
+  async initialize() {
+    // nothing, only for child classes
+  }
+
+  // eslint-disable-next-line class-methods-use-this
+  getLowestActiveShard() {
+    return null;
+  }
+
+  // eslint-disable-next-line class-methods-use-this
+  amIImportant() {
+    return true;
+  }
+
   /*
    * async event
    */
@@ -33,14 +48,6 @@ class SocketEvents extends EventEmitter {
     });
   }
 
-  /*
-   * broadcast message via websocket
-   * @param message Buffer Message to send
-   */
-  broadcast(message) {
-    this.emit('broadcast', message);
-  }
-
   /*
    * broadcast pixel message via websocket
    * @param canvasId number ident of canvas
@@ -52,8 +59,34 @@ class SocketEvents extends EventEmitter {
     chunkId,
     pixels,
   ) {
-    const buffer = PixelUpdate.dehydrate(chunkId, pixels);
+    const i = chunkId >> 8;
+    const j = chunkId & 0xFF;
+    const buffer = PixelUpdate.dehydrate(i, j, pixels);
     this.emit('pixelUpdate', canvasId, chunkId, buffer);
+    this.emit('chunkUpdate', canvasId, [i, j]);
+  }
+
+  /*
+   * chunk updates from event, image upload, etc.
+   * everything thats not a pixelUpdate and changes chunks
+   * @param canvasId
+   * @param chunk [i,j] chunk coordinates
+   */
+  broadcastChunkUpdate(
+    canvasId,
+    chunk,
+  ) {
+    this.emit('chunkUpdate', canvasId, chunk);
+  }
+
+  /*
+   * ask other shards to send email for us,
+   * only used when USE_MAILER is false
+   * @param type type of mail to send
+   * @param args
+   */
+  sendMail(...args) {
+    this.emit('mail', ...args);
   }
 
   /*
@@ -169,4 +202,4 @@ class SocketEvents extends EventEmitter {
   }
 }
 
-export default new SocketEvents();
+export default SocketEvents;

src/socket/SocketClient.js

@@ -14,6 +14,7 @@ import RegisterMultipleChunks from './packets/RegisterMultipleChunks';
 import DeRegisterChunk from './packets/DeRegisterChunk';
 import ChangedMe from './packets/ChangedMe';
 import Ping from './packets/Ping';
+import { shardHost } from '../store/actions/fetch';
 
 const chunks = [];
 
@@ -43,9 +44,10 @@ class SocketClient extends EventEmitter {
       console.log('WebSocket already open, not starting');
     }
     this.timeLastConnecting = Date.now();
-    const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
+    const url = `${
-    const url = `${protocol}//${window.location.hostname}${
+      window.location.protocol === 'https:' ? 'wss:' : 'ws:'
-      window.location.port ? `:${window.location.port}` : ''
+    }//${
+      shardHost || window.location.host
     }/ws`;
     this.ws = new WebSocket(url);
     this.ws.binaryType = 'arraybuffer';

src/socket/SocketServer.js

@@ -19,7 +19,7 @@ import DeRegisterMultipleChunks from './packets/DeRegisterMultipleChunks';
 import ChangedMe from './packets/ChangedMe';
 import OnlineCounter from './packets/OnlineCounter';
 
-import socketEvents from './SocketEvents';
+import socketEvents from './socketEvents';
 import chatProvider, { ChatProvider } from '../core/ChatProvider';
 import authenticateClient from './authenticateClient';
 import { drawByOffsets } from '../core/draw';
@@ -111,7 +111,6 @@ class SocketServer {
       });
     });
 
-    socketEvents.on('broadcast', this.broadcast);
     socketEvents.on('onlineCounter', this.broadcast);
     socketEvents.on('pixelUpdate', this.broadcastPixelBuffer);
     socketEvents.on('reloadUser', this.reloadUser);
@@ -189,9 +188,12 @@ class SocketServer {
     }
     // CORS
     const { origin } = headers;
-    if (!origin || !origin.endsWith(getHostFromRequest(request, false))) {
+    const host = getHostFromRequest(request, false, true);
+    if (!origin
+      || !`.${origin.slice(origin.indexOf('//') + 2)}`.endsWith(host)
+    ) {
       // eslint-disable-next-line max-len
-      logger.info(`Rejected CORS request on websocket from ${ip} via ${headers.origin}, expected ${getHostFromRequest(request, false)}`);
+      logger.info(`Rejected CORS request on websocket from ${ip} via ${headers.origin}, expected ${getHostFromRequest(request, false, true)}`);
       socket.write('HTTP/1.1 403 Forbidden\r\n\r\n');
       socket.destroy();
       return;
@@ -433,9 +435,6 @@ class SocketServer {
          * if DM channel, make sure that other user has DM open
          * (needed because we allow user to leave one-sided
          *  and auto-join on message)
-         *  TODO: if we scale and have multiple websocket servers at some point
-         *  this might be an issue. We would hve to make a shared list of online
-         *  users and act based on that on 'chatMessage' event
          */
         const dmUserId = chatProvider.checkIfDm(user, channelId);
         if (dmUserId) {
@@ -443,11 +442,17 @@ class SocketServer {
           if (!dmWs
             || !chatProvider.userHasChannelAccess(dmWs.user, channelId)
           ) {
-            await ChatProvider.addUserToChannel(
+            // TODO this is really ugly
-              dmUserId,
+            // DMS have to be rethought
-              channelId,
+            if (!user.addedDM) user.addedDM = [];
-              [ws.name, 1, Date.now(), user.id],
+            if (!user.addedDM.includes(dmUserId)) {
-            );
+              await ChatProvider.addUserToChannel(
+                dmUserId,
+                channelId,
+                [ws.name, 1, Date.now(), user.id],
+              );
+              user.addedDM.push(dmUserId);
+            }
           }
         }
 

src/socket/packets/ChunkUpdate.js

@@ -0,0 +1,30 @@
+/*
+ * notify that chunk changed
+ * (not sent over websocket, server only)
+ */
+
+const OP_CODE = 0xC4;
+
+export default {
+  OP_CODE,
+  /*
+   * @return canvasId, [i, j]
+   */
+  hydrate(data) {
+    const canvasId = data[1];
+    const chunk = [data[2], data[3]];
+    return [canvasId, chunk];
+  },
+  /*
+   * @param canvasId,
+   * chunkid id consisting of chunk coordinates
+   */
+  dehydrate(canvasId, [i, j]) {
+    return Buffer.from({
+      OP_CODE,
+      canvasId,
+      i,
+      j,
+    });
+  },
+};

src/socket/packets/PixelUpdateMB.js

@@ -0,0 +1,44 @@
+/*
+ * Packet for sending and receiving pixels over Message Broker between shards
+ * Multiple pixels can be sent at once
+ *
+ */
+
+const OP_CODE = 0xC1;
+
+export default {
+  OP_CODE,
+  /*
+   * returns info and PixelUpdate package to send to clients
+   */
+  hydrate(data) {
+    const canvasId = data[1];
+    data.writeUInt8(OP_CODE, 1);
+    const chunkId = data.readUInt16BE(2);
+    const pixelUpdate = Buffer.from(
+      data.buffer,
+      data.byteOffset + 1,
+      data.length - 1,
+    );
+    return [
+      canvasId,
+      chunkId,
+      pixelUpdate,
+    ];
+  },
+
+  /*
+   * @param canvasId
+   * @param chunkId id consisting of chunk coordinates
+   * @param pixels Buffer with offset and color of one or more pixels
+   */
+  dehydrate(canvasId, i, j, pixels) {
+    const index = new Uint8Array([
+      OP_CODE,
+      canvasId,
+      i,
+      j,
+    ]);
+    return Buffer.concat([index, pixels]);
+  },
+};

src/socket/packets/PixelUpdateServer.js

@@ -43,8 +43,8 @@ export default {
    * @param chunkId id consisting of chunk coordinates
    * @param pixels Buffer with offset and color of one or more pixels
    */
-  dehydrate(chunkId, pixels) {
+  dehydrate(i, j, pixels) {
-    const index = new Uint8Array([OP_CODE, chunkId >> 8, chunkId & 0xFF]);
+    const index = new Uint8Array([OP_CODE, i, j]);
     return Buffer.concat([index, pixels]);
   },
 };

src/socket/socketEvents.js

@@ -0,0 +1,10 @@
+import SocketEvents from './SockEvents';
+import MessageBroker from './MessageBroker';
+import { SHARD_NAME } from '../core/config';
+
+/*
+ * if we are a shard in a cluster, do messaging to others via redis
+ */
+const socketEvents = (SHARD_NAME) ? new MessageBroker() : new SocketEvents();
+
+export default socketEvents;

src/ssr/Main.jsx

@@ -8,8 +8,9 @@
 import { langCodeToCC } from '../utils/location';
 import ttags, { getTTag } from '../core/ttag';
 import { styleassets, assets } from '../core/assets';
-
+import socketEvents from '../socket/socketEvents';
 import { BACKUP_URL } from '../core/config';
+import { getHostFromRequest } from '../utils/ip';
 
 /*
  * generate language list
@@ -35,14 +36,19 @@ if (BACKUP_URL) {
  * @param lang language code
  * @return html of mainpage
  */
-function generateMainPage(lang) {
+function generateMainPage(req) {
+  const { lang } = req;
+  const host = getHostFromRequest(req, false);
   const ssvR = {
     ...ssv,
+    shard: (host.startsWith(`${socketEvents.thisShard}.`))
+      ? '' : socketEvents.getLowestActiveShard(),
     lang: lang === 'default' ? 'en' : lang,
   };
   const scripts = (assets[`client-${lang}`])
     ? assets[`client-${lang}`].js
     : assets.client.js;
+
   const { t } = getTTag(lang);
 
   const html = `

src/ssr/PopUp.jsx

@@ -7,10 +7,10 @@
 
 import { langCodeToCC } from '../utils/location';
 import ttags, { getTTag } from '../core/ttag';
-
+import socketEvents from '../socket/socketEvents';
-/* this will be set by webpack */
 import { styleassets, assets } from '../core/assets';
 import { BACKUP_URL } from '../core/config';
+import { getHostFromRequest } from '../utils/ip';
 
 /*
  * generate language list
@@ -35,9 +35,13 @@ if (BACKUP_URL) {
  * @param lang language code
  * @return html of mainpage
  */
-function generatePopUpPage(lang) {
+function generatePopUpPage(req) {
+  const { lang } = req;
+  const host = getHostFromRequest(req);
   const ssvR = {
     ...ssv,
+    shard: (host.startsWith(`${socketEvents.thisShard}.`))
+      ? null : socketEvents.getLowestActiveShard(),
     lang: lang === 'default' ? 'en' : lang,
   };
   const script = (assets[`popup-${lang}`])

src/store/actions/fetch.js

@@ -8,17 +8,30 @@ import { t } from 'ttag';
 
 import { dateToString } from '../../core/utils';
 
+export const shardHost = (function getShardHost() {
+  if (!window.ssv || !window.ssv.shard) {
+    return '';
+  }
+  const hostParts = window.location.host.split('.');
+  if (hostParts.length > 2) {
+    hostParts.shift();
+  }
+  return `${window.ssv.shard}.${hostParts.join('.')}`;
+}());
+export const shardOrigin = shardHost
+  && `${window.location.protocol}//${shardHost}`;
+
 /*
  * Adds customizeable timeout to fetch
  * defaults to 8s
  */
-async function fetchWithTimeout(resource, options = {}) {
+async function fetchWithTimeout(url, options = {}) {
-  const { timeout = 8000 } = options;
+  const { timeout = 10000 } = options;
 
   const controller = new AbortController();
   const id = setTimeout(() => controller.abort(), timeout);
 
-  const response = await fetch(resource, {
+  const response = await fetch(url, {
     ...options,
     signal: controller.signal,
   });
@@ -62,11 +75,19 @@ async function parseAPIresponse(response) {
  * @param body Body of request
  * @return Object with response or error Array
  */
-async function makeAPIPOSTRequest(url, body) {
+async function makeAPIPOSTRequest(
+  url,
+  body,
+  credentials = true,
+  addShard = true,
+) {
+  if (addShard) {
+    url = `${shardOrigin}${url}`;
+  }
   try {
     const response = await fetchWithTimeout(url, {
       method: 'POST',
-      credentials: 'include',
+      credentials: (credentials) ? 'include' : 'omit',
       headers: {
         'Content-Type': 'application/json',
       },
@@ -86,10 +107,17 @@ async function makeAPIPOSTRequest(url, body) {
  * @param url URL of get api endpoint
  * @return Object with response or error Array
  */
-async function makeAPIGETRequest(url) {
+async function makeAPIGETRequest(
+  url,
+  credentials = true,
+  addShard = true,
+) {
+  if (addShard) {
+    url = `${shardOrigin}${url}`;
+  }
   try {
     const response = await fetchWithTimeout(url, {
-      credentials: 'include',
+      credentials: (credentials) ? 'include' : 'omit',
     });
 
     return parseAPIresponse(response);
@@ -193,8 +221,10 @@ export async function requestSolveCaptcha(text, captchaid) {
 export async function requestHistoricalTimes(day, canvasId) {
   try {
     const date = dateToString(day);
-    const url = `history?day=${date}&id=${canvasId}`;
+    // Not going over shard url
+    const url = `/history?day=${date}&id=${canvasId}`;
     const response = await fetchWithTimeout(url, {
+      credentials: 'omit',
       timeout: 45000,
     });
     if (response.status !== 200) {
@@ -212,6 +242,19 @@ export async function requestHistoricalTimes(day, canvasId) {
   }
 }
 
+export async function requestChatMessages(cid) {
+  const response = await fetch(
+    `${shardOrigin}/api/chathistory?cid=${cid}&limit=50`,
+    { credentials: 'include' },
+  );
+  // timeout in order to not spam api requests and get rate limited
+  if (response.ok) {
+    const { history } = await response.json();
+    return history;
+  }
+  return null;
+}
+
 export function requestPasswordChange(newPassword, password) {
   return makeAPIPOSTRequest(
     '/api/auth/change_passwd',
@@ -278,7 +321,8 @@ export function requestDeleteAccount(password) {
 
 export function requestRankings() {
   return makeAPIGETRequest(
-    'ranking',
+    '/ranking',
+    false,
   );
 }
 

src/store/actions/thunks.js

@@ -7,6 +7,7 @@ import {
   requestBlockDm,
   requestLeaveChan,
   requestRankings,
+  requestChatMessages,
   requestMe,
 } from './fetch';
 
@@ -89,21 +90,12 @@ export function fetchMe() {
   };
 }
 
-export function fetchChatMessages(
+export function fetchChatMessages(cid) {
-  cid,
-) {
   return async (dispatch) => {
     dispatch(setChatFetching(true));
-    const response = await fetch(`/api/chathistory?cid=${cid}&limit=50`, {
+    const history = await requestChatMessages(cid);
-      credentials: 'include',
+    if (history) {
-    });
-
-    /*
-     * timeout in order to not spam api requests and get rate limited
-     */
-    if (response.ok) {
       setTimeout(() => { dispatch(setChatFetching(false)); }, 500);
-      const { history } = await response.json();
       dispatch(receiveChatHistory(cid, history));
     } else {
       setTimeout(() => { dispatch(setChatFetching(false)); }, 5000);

src/ui/ChunkLoader2D.js

@@ -4,6 +4,7 @@
 
 import ChunkRGB from './ChunkRGB';
 import { TILE_SIZE, TILE_ZOOM_LEVEL } from '../core/constants';
+import { shardOrigin } from '../store/actions/fetch';
 import {
   loadingTiles,
   loadImage,
@@ -279,7 +280,7 @@ class ChunkLoader {
     const center = [zoom, cx, cy];
     this.store.dispatch(requestBigChunk(center));
     try {
-      const url = `chunks/${this.canvasId}/${cx}/${cy}.bmp`;
+      const url = `${shardOrigin}/chunks/${this.canvasId}/${cx}/${cy}.bmp`;
       const response = await fetch(url);
       if (response.ok) {
         const arrayBuffer = await response.arrayBuffer();
@@ -303,7 +304,8 @@ class ChunkLoader {
     const center = [zoom, cx, cy];
     this.store.dispatch(requestBigChunk(center));
     try {
-      const url = `tiles/${this.canvasId}/${zoom}/${cx}/${cy}.webp`;
+    // eslint-disable-next-line max-len
+      const url = `${shardOrigin}/tiles/${this.canvasId}/${zoom}/${cx}/${cy}.webp`;
       const img = await loadImage(url);
       chunkRGB.fromImage(img);
       this.store.dispatch(receiveBigChunk(center));

src/ui/ChunkLoader3D.js

@@ -3,6 +3,7 @@
  *
  */
 
+import Chunk from './ChunkRGB3D';
 import {
   requestBigChunk,
   receiveBigChunk,
@@ -12,9 +13,7 @@ import {
   getChunkOfPixel,
   getOffsetOfPixel,
 } from '../core/utils';
-
+import { shardOrigin } from '../store/actions/fetch';
-import Chunk from './ChunkRGB3D';
-
 
 class ChunkLoader {
   store = null;
@@ -91,7 +90,7 @@ class ChunkLoader {
     const center = [0, cx, cz];
     this.store.dispatch(requestBigChunk(center));
     try {
-      const url = `chunks/${this.canvasId}/${cx}/${cz}.bmp`;
+      const url = `${shardOrigin}/chunks/${this.canvasId}/${cx}/${cz}.bmp`;
       const response = await fetch(url);
       if (response.ok) {
         const arrayBuffer = await response.arrayBuffer();

src/utils/corsMiddleware.js

@@ -0,0 +1,39 @@
+/*
+ * set CORS Headers
+ */
+import { CORS_HOSTS } from '../core/config';
+
+export default (req, res, next) => {
+  if (!CORS_HOSTS || !req.headers.origin) {
+    next();
+    return;
+  }
+  const { origin } = req.headers;
+
+  const host = origin.slice(origin.indexOf('//') + 2);
+  /*
+   * form .domain.tld will accept both domain.tld and x.domain.tld
+   */
+  const isAllowed = CORS_HOSTS.some((c) => c === host
+    || (c.startsWith('.') && (host.endsWith(c) || host === c.slice(1))));
+
+  if (!isAllowed) {
+    next();
+    return;
+  }
+
+  res.set({
+    'Access-Control-Allow-Origin': origin,
+    'Access-Control-Allow-Credentials': 'true',
+  });
+
+  if (req.method === 'OPTIONS') {
+    res.set({
+      'Access-Control-Allow-Headers': 'Content-Type',
+      'Access-Control-Allow-Methods': 'GET,POST',
+    });
+    res.sendStatus(200);
+    return;
+  }
+  next();
+};

src/utils/ip.js

@@ -50,11 +50,18 @@ function ip4NumToStr(ipNum) {
  * @param includeProto if we include protocol (https, http)
  * @return host (like pixelplanet.fun)
  */
-export function getHostFromRequest(req, includeProto = true) {
+export function getHostFromRequest(req, includeProto = true, stripSub = false) {
   const { headers } = req;
-  const host = headers['x-forwarded-host']
+  let host = headers['x-forwarded-host']
     || headers.host
     || headers[':authority'];
+  if (stripSub) {
+    if (host.lastIndexOf('.') !== host.indexOf('.')) {
+      host = host.slice(host.indexOf('.'));
+    } else {
+      host = `.${host}`;
+    }
+  }
   if (!includeProto) {
     return host;
   }

webpack.config.server.js

@@ -160,8 +160,8 @@ module.exports = ({
             to: path.resolve('dist', 'captchaFonts'),
           },
           {
-            from: path.resolve('src', 'data', 'redis', 'lua', 'placePixel.lua'),
+            from: path.resolve('src', 'data', 'redis', 'lua'),
-            to: path.resolve('dist', 'workers', 'placePixel.lua'),
+            to: path.resolve('dist', 'workers', 'lua'),
           },
         ],
       }),