ppfun
/
pixelplanet
13
Fork 18
Code Issues 24 Pull Requests Releases 2 Activity

Horicontal Scaling

This commit is contained in:
HF 2022-09-10 00:35:28 +02:00
parent ae6ebc2441
commit de1729d56b
55 changed files with 936 additions and 341 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/components/Admintools.jsx
View File

@ -5,6 +5,8 @@
import React, { useState, useEffect } from 'react';
import { t } from 'ttag';
import { shardOrigin } from '../store/actions/fetch';
async function submitIPAction(
action,
vallist,
@ -13,7 +15,7 @@ async function submitIPAction(
const data = new FormData();
data.append('ipaction', action);
data.append('ip', vallist);
const resp = await fetch('/api/modtools', {
const resp = await fetch(`${shardOrigin}/api/modtools`, {
credentials: 'include',
method: 'POST',
body: data,
@ -26,7 +28,7 @@ async function getModList(
) {
const data = new FormData();
data.append('modlist', true);
const resp = await fetch('/api/modtools', {
const resp = await fetch(`${shardOrigin}/api/modtools`, {
credentials: 'include',
method: 'POST',
body: data,
@ -44,7 +46,7 @@ async function submitRemMod(
) {
const data = new FormData();
data.append('remmod', userId);
const resp = await fetch('/api/modtools', {
const resp = await fetch(`${shardOrigin}/api/modtools`, {
credentials: 'include',
method: 'POST',
body: data,
@ -58,7 +60,7 @@ async function submitMakeMod(
) {
const data = new FormData();
data.append('makemod', userName);
const resp = await fetch('/api/modtools', {
const resp = await fetch(`${shardOrigin}/api/modtools`, {
credentials: 'include',
method: 'POST',
body: data,

3
src/components/Captcha.jsx
View File

@ -10,9 +10,10 @@ import React, { useState, useEffect } from 'react';
import { t } from 'ttag';
import { IoReloadCircleSharp } from 'react-icons/io5';
import { shardOrigin } from '../store/actions/fetch';
async function getUrlAndId() {
const url = './captcha.svg';
const url = `${shardOrigin}/captcha.svg`;
const resp = await fetch(url, {
cache: 'no-cache',
});

9
src/components/LanguageSelect.jsx
View File

@ -66,7 +66,14 @@ function LanguageSelect() {
/* set with selected language */
const d = new Date();
d.setTime(d.getTime() + 24 * MONTH);
document.cookie = `lang=${langSel};expires=${d.toUTCString()};path=/`;
let { host } = window.location;
if (host.lastIndexOf('.') !== host.indexOf('.')) {
host = host.slice(host.indexOf('.'));
} else {
host = `.${host}`;
}
// eslint-disable-next-line max-len
document.cookie = `lang=${langSel};expires=${d.toUTCString()};path=/;domain=${host}`;
window.location.reload();
}}
>

13
src/components/ModCanvastools.jsx
View File

@ -8,6 +8,7 @@ import { t } from 'ttag';
import useInterval from './hooks/interval';
import { getToday, dateToString } from '../core/utils';
import { shardOrigin } from '../store/actions/fetch';
const keptState = {
coords: '',
@ -33,7 +34,7 @@ async function submitImageAction(
data.append('image', file);
data.append('canvasid', canvas);
data.append('coords', coords);
const resp = await fetch('/api/modtools', {
const resp = await fetch(`${shardOrigin}/api/modtools`, {
credentials: 'include',
method: 'POST',
body: data,
@ -53,7 +54,7 @@ async function submitProtAction(
data.append('canvasid', canvas);
data.append('ulcoor', tlcoords);
data.append('brcoor', brcoords);
const resp = await fetch('/api/modtools', {
const resp = await fetch(`${shardOrigin}/api/modtools`, {
credentials: 'include',
method: 'POST',
body: data,
@ -74,7 +75,7 @@ async function submitRollback(
data.append('canvasid', canvas);
data.append('ulcoor', tlcoords);
data.append('brcoor', brcoords);
const resp = await fetch('/api/modtools', {
const resp = await fetch(`${shardOrigin}/api/modtools`, {
credentials: 'include',
method: 'POST',
body: data,
@ -94,7 +95,7 @@ async function submitCanvasCleaner(
data.append('canvasid', canvas);
data.append('ulcoor', tlcoords);
data.append('brcoor', brcoords);
const resp = await fetch('/api/modtools', {
const resp = await fetch(`${shardOrigin}/api/modtools`, {
credentials: 'include',
method: 'POST',
body: data,
@ -107,7 +108,7 @@ async function getCleanerStats(
) {
const data = new FormData();
data.append('cleanerstat', true);
const resp = await fetch('/api/modtools', {
const resp = await fetch(`${shardOrigin}/api/modtools`, {
credentials: 'include',
method: 'POST',
body: data,
@ -125,7 +126,7 @@ async function getCleanerCancel(
) {
const data = new FormData();
data.append('cleanercancel', true);
const resp = await fetch('/api/modtools', {
const resp = await fetch(`${shardOrigin}/api/modtools`, {
credentials: 'include',
method: 'POST',
body: data,

3
src/components/ModIIDtools.jsx
View File

@ -6,6 +6,7 @@ import React, { useState } from 'react';
import { t } from 'ttag';
import { parseInterval } from '../core/utils';
import { shardOrigin } from '../store/actions/fetch';
async function submitIIDAction(
action,
@ -31,7 +32,7 @@ async function submitIIDAction(
data.append('reason', reason);
data.append('time', time);
data.append('iid', iid);
const resp = await fetch('/api/modtools', {
const resp = await fetch(`${shardOrigin}/api/modtools`, {
credentials: 'include',
method: 'POST',
body: data,

3
src/components/ModWatchtools.jsx
View File

@ -9,6 +9,7 @@ import { t } from 'ttag';
import copyTextToClipboard from '../utils/clipboard';
import { parseInterval } from '../core/utils';
import { shardOrigin } from '../store/actions/fetch';
const keepState = {
tlcoords: '',
@ -54,7 +55,7 @@ async function submitWatchAction(
data.append('time', time);
data.append('iid', iid);
try {
const resp = await fetch('/api/modtools', {
const resp = await fetch(`${shardOrigin}/api/modtools`, {
credentials: 'include',
method: 'POST',
body: data,

5
src/core/ChatProvider.js
View File

@ -10,7 +10,7 @@ import {
} from '../data/sql';
import { findIdByNameOrId } from '../data/sql/RegUser';
import ChatMessageBuffer from './ChatMessageBuffer';
import socketEvents from '../socket/SocketEvents';
import socketEvents from '../socket/socketEvents';
import checkIPAllowed from './isAllowed';
import { DailyCron } from '../utils/cron';
import { escapeMd } from './utils';
@ -93,6 +93,9 @@ export class ChatProvider {
}
async clearOldMessages() {
if (!socketEvents.amIImportant()) {
return;
}
const ids = Object.keys(this.defaultChannels);
for (let i = 0; i < ids.length; i += 1) {
const cid = ids[i];

184
src/core/mail.js → src/core/MailProvider.js
View File

@ -4,20 +4,17 @@
/* eslint-disable max-len */
import { randomUUID } from 'crypto';
import nodemailer from 'nodemailer';
import logger from './logger';
import { HOUR, MINUTE } from './constants';
import { DailyCron, HourlyCron } from '../utils/cron';
import { getTTag } from './ttag';
import { codeExists, checkCode, setCode } from '../data/redis/mailCodes';
import socketEvents from '../socket/socketEvents';
import { USE_MAILER, MAIL_ADDRESS } from './config';
import { RegUser } from '../data/sql';
// TODO make code expire
class MailProvider {
export class MailProvider {
constructor() {
this.enabled = !!USE_MAILER;
if (this.enabled) {
@ -26,13 +23,23 @@ class MailProvider {
newline: 'unix',
path: '/usr/sbin/sendmail',
});
this.clearCodes = this.clearCodes.bind(this);
this.verifyCodes = {};
HourlyCron.hook(this.clearCodes);
DailyCron.hook(MailProvider.cleanUsers);
}
/*
* mail requests make it through SocketEvents when sharding
*/
socketEvents.on('mail', (type, args) => {
switch (type) {
case 'verify':
this.postVerifyMail(...args);
break;
case 'pwreset':
this.postPasswdResetMail(...args);
break;
default:
// nothing
}
});
}
sendMail(to, subject, html) {
@ -52,29 +59,10 @@ class MailProvider {
});
}
sendVerifyMail(to, name, host, lang) {
if (!this.enabled) {
return null;
}
postVerifyMail(to, name, host, lang, code) {
const { t } = getTTag(lang);
const pastMail = this.verifyCodes[to];
if (pastMail) {
const minLeft = Math.floor(
pastMail.timestamp / MINUTE + 2 - Date.now() / MINUTE,
);
if (minLeft > 0) {
logger.info(
`Verify mail for ${to} - already sent, ${minLeft} minutes left`,
);
return t`We already sent you a verification mail, you can request another one in ${minLeft} minutes.`;
}
}
logger.info(`Sending verification mail to ${to} / ${name}`);
const code = this.setCode(to);
const verifyUrl = `${host}/api/auth/verify?token=${code}`;
const verifyUrl = `${host}/api/auth/verify?token=${code}&email=${encodeURIComponent(to)}`;
const subject = t`Welcome ${name} to PixelPlanet, plese verify your mail`;
const html = `<em>${t`Hello ${name}`}</em>,<br />
${t`welcome to our little community of pixelplacers, to use your account, you have to verify your mail. You can do that here: `} <a href="${verifyUrl}">${t`Click to Verify`}</a>. ${t`Or by copying following url:`}<br />${verifyUrl}\n<br />
@ -82,27 +70,60 @@ class MailProvider {
${t`Thanks`}<br /><br />
<img alt="" src="https://assets.pixelplanet.fun/tile.png" style="height:64px; width:64px" />`;
this.sendMail(to, subject, html);
}
async sendVerifyMail(to, name, host, lang) {
if (!this.enabled && !socketEvents.isCluster) {
return null;
}
const { t } = getTTag(lang);
const pastCodeAge = await codeExists(to);
if (pastCodeAge && pastCodeAge < 180) {
const minLeft = Math.ceil((180 - pastCodeAge) / 60);
logger.info(
`Verify mail for ${to} - already sent, ${minLeft} minutes left`,
);
return t`We already sent you a verification mail, you can request another one in ${minLeft} minutes.`;
}
const code = setCode(to);
if (this.enabled) {
this.postVerifyMail(to, name, host, lang, code);
} else {
socketEvents.sendMail('verify', [to, name, host, lang, code]);
}
return null;
}
postPasswdResetMail(to, ip, host, lang, code) {
const { t } = getTTag(lang);
logger.info(`Sending Password reset mail to ${to}`);
const restoreUrl = `${host}/reset_password?token=${code}`;
const subject = t`You forgot your password for PixelPlanet? Get a new one here`;
const html = `<em>${t`Hello`}</em>,<br />
${t`You requested to get a new password. You can change your password within the next 30min here: `} <a href="${restoreUrl}">${t`Reset Password`}</a>. ${t`Or by copying following url:`}<br />${restoreUrl}\n<br />
${t`If you did not request this mail, please just ignore it (the ip that requested this mail was ${ip}).`}<br />
${t`Thanks`}<br /><br />\n<img alt="" src="https://assets.pixelplanet.fun/tile.png" style="height:64px; width:64px" />`;
this.sendMail(to, subject, html);
}
async sendPasswdResetMail(to, ip, host, lang) {
const { t } = getTTag(lang);
if (!this.enabled) {
if (!this.enabled && !socketEvents.isCluster) {
return t`Mail is not configured on the server`;
}
const pastMail = this.verifyCodes[to];
if (pastMail) {
if (Date.now() < pastMail.timestamp + 15 * MINUTE) {
logger.info(
`Password reset mail for ${to} requested by ${ip} - already sent`,
);
return t`We already sent you a mail with instructions. Please wait before requesting another mail.`;
}
const pastCodeAge = await codeExists(to);
if (pastCodeAge && pastCodeAge < 180) {
logger.info(
`Password reset mail for ${to} requested by ${ip} - already sent`,
);
return t`We already sent you a mail with instructions. Please wait before requesting another mail.`;
}
const reguser = await RegUser.findOne({ where: { email: to } });
if (pastMail || !reguser) {
if (!reguser) {
logger.info(
`Password reset mail for ${to} requested by ${ip} - mail not found`,
);
@ -119,68 +140,20 @@ class MailProvider {
}
*/
logger.info(`Sending Password reset mail to ${to}`);
const code = this.setCode(to);
const restoreUrl = `${host}/reset_password?token=${code}`;
const subject = t`You forgot your password for PixelPlanet? Get a new one here`;
const html = `<em>${t`Hello`}</em>,<br />
${t`You requested to get a new password. You can change your password within the next 30min here: `} <a href="${restoreUrl}">${t`Reset Password`}</a>. ${t`Or by copying following url:`}<br />${restoreUrl}\n<br />
${t`If you did not request this mail, please just ignore it (the ip that requested this mail was ${ip}).`}<br />
${t`Thanks`}<br /><br />\n<img alt="" src="https://assets.pixelplanet.fun/tile.png" style="height:64px; width:64px" />`;
this.sendMail(to, subject, html);
const code = setCode(to);
if (this.enabled) {
this.postPasswdResetMail(to, ip, host, lang, code);
} else {
socketEvents.sendMail('pwreset', [to, ip, host, lang, code]);
}
return null;
}
setCode(email) {
const code = MailProvider.createCode();
this.verifyCodes[email] = {
code,
timestamp: Date.now(),
};
return code;
}
async clearCodes() {
const curTime = Date.now();
const toDelete = [];
const mails = Object.keys(this.verifyCodes);
for (let i = 0; i < mails.length; i += 1) {
const iteremail = mails[i];
if (curTime > this.verifyCodes[iteremail].timestamp + HOUR) {
toDelete.push(iteremail);
}
}
toDelete.forEach((email) => {
logger.info(`Mail Code for ${email} expired`);
delete this.verifyCodes[email];
});
}
// Note: code gets deleted on check
checkCode(code) {
let email = null;
const mails = Object.keys(this.verifyCodes);
for (let i = 0; i < mails.length; i += 1) {
const iteremail = mails[i];
if (this.verifyCodes[iteremail].code === code) {
email = iteremail;
break;
}
}
if (!email) {
logger.info(`Mail Code ${code} not found.`);
static async verify(email, code) {
const ret = await checkCode(email, code);
if (!ret) {
return false;
}
logger.info(`Got Mail Code from ${email}.`);
delete this.verifyCodes[email];
return email;
}
async verify(code) {
const email = this.checkCode(code);
if (!email) return false;
const reguser = await RegUser.findOne({ where: { email } });
if (!reguser) {
logger.error(`${email} does not exist in database`);
@ -193,13 +166,10 @@ class MailProvider {
return reguser.name;
}
static createCode() {
return randomUUID();
}
/*
* we do not use this right now
static cleanUsers() {
// delete users that requier verification for more than 4 days
/*
RegUser.destroy({
where: {
verificationReqAt: {
@ -209,8 +179,8 @@ class MailProvider {
verified: 0,
},
});
*/
}
*/
}
const mailProvider = new MailProvider();

2
src/core/PixelCache.js
View File

@ -3,7 +3,7 @@
* in bursts per chunk
*/
import socketEvents from '../socket/SocketEvents';
import socketEvents from '../socket/socketEvents';
class PixelCache {
PXL_CACHE;

17
src/core/Tile.js
View File

@ -242,6 +242,14 @@ function addIndexedSubtiletoTile(
function tileFileName(canvasTileFolder, cell) {
const [z, x, y] = cell;
const filename = `${canvasTileFolder}/${z}/${x}/${y}.webp`;
try {
const mtime = new Date(fs.statSync(filename).mtime).getTime();
if (Date.now() - mtime < 120000) {
return null;
}
} catch {
// file doesn't exist
}
return filename;
}
@ -263,6 +271,10 @@ export async function createZoomTileFromChunk(
const canvasSize = canvas.size;
const [x, y] = cell;
const maxTiledZoom = getMaxTiledZoom(canvasSize);
const filename = tileFileName(canvasTileFolder, [maxTiledZoom - 1, x, y]);
if (!filename) return true;
const tileRGBBuffer = new Uint8Array(
TILE_SIZE * TILE_SIZE * TILE_ZOOM_LEVEL * TILE_ZOOM_LEVEL * 3,
);
@ -318,7 +330,6 @@ export async function createZoomTileFromChunk(
);
});
const filename = tileFileName(canvasTileFolder, [maxTiledZoom - 1, x, y]);
try {
await sharp(tileRGBBuffer, {
raw: {
@ -363,6 +374,9 @@ export async function createZoomedTile(
);
const [z, x, y] = cell;
const filename = tileFileName(canvasTileFolder, [z, x, y]);
if (!filename) return true;
const startTime = Date.now();
const na = [];
@ -409,7 +423,6 @@ export async function createZoomedTile(
);
});
const filename = tileFileName(canvasTileFolder, [z, x, y]);
try {
await sharp(tileRGBBuffer, {
raw: {

2
src/core/Void.js
View File

@ -5,7 +5,7 @@
* if it reaches the TARGET_RADIUS size, the event is lost
*
*/
import socketEvents from '../socket/SocketEvents';
import socketEvents from '../socket/socketEvents';
import PixelUpdate from '../socket/packets/PixelUpdateServer';
import { setPixelByOffset } from './setPixel';
import { TILE_SIZE } from './constants';

6
src/core/config.js
View File

@ -20,7 +20,7 @@ export const MAIL_ADDRESS = process.env.MAIL_ADDRESS
const TILE_FOLDER_REL = process.env.TILE_FOLDER || 'tiles';
export const TILE_FOLDER = path.join(__dirname, `./${TILE_FOLDER_REL}`);
export const USE_XREALIP = process.env.USE_XREALIP || false;
export const USE_XREALIP = !!process.env.USE_XREALIP;
export const BACKUP_URL = process.env.BACKUP_URL || null;
export const BACKUP_DIR = process.env.BACKUP_DIR || null;
@ -30,6 +30,7 @@ export const USE_PROXYCHECK = parseInt(process.env.USE_PROXYCHECK, 10) || false;
export const { PROXYCHECK_KEY } = process.env;
export const REDIS_URL = process.env.REDIS_URL || 'redis://localhost:6379';
export const SHARD_NAME = process.env.SHARD_NAME || null;
// Database
export const MYSQL_HOST = process.env.MYSQL_HOST || 'localhost';
export const MYSQL_DATABASE = process.env.MYSQL_DATABASE || 'pixelplanet';
@ -52,6 +53,9 @@ export const APISOCKET_KEY = process.env.APISOCKET_KEY || null;
export const ADMIN_IDS = (process.env.ADMIN_IDS)
? process.env.ADMIN_IDS.split(',').map((z) => parseInt(z, 10)) : [];
export const CORS_HOSTS = (process.env.CORS_HOSTS)
? process.env.CORS_HOSTS.split(',') : [];
export const auth = {
// https://developers.facebook.com/
facebook: {

24
src/core/ranking.js
View File

@ -6,6 +6,7 @@ import Sequelize from 'sequelize';
import sequelize from '../data/sql/sequelize';
import RegUser from '../data/sql/RegUser';
import { saveDailyTop, loadDailyTop } from '../data/redis/PrevDayTop';
import socketEvents from '../socket/socketEvents';
import logger from './logger';
import { MINUTE } from './constants';
@ -33,15 +34,17 @@ class Ranks {
async updateRanking() {
logger.info('Update pixel rankings');
// recalculate ranking column
await sequelize.query(
// eslint-disable-next-line max-len
'SET @r=0; UPDATE Users SET ranking= @r:= (@r + 1) ORDER BY totalPixels DESC;',
);
await sequelize.query(
// eslint-disable-next-line max-len
'SET @r=0; UPDATE Users SET dailyRanking= @r:= (@r + 1) ORDER BY dailyTotalPixels DESC;',
);
if (socketEvents.amIImportant()) {
// recalculate ranking column
await sequelize.query(
// eslint-disable-next-line max-len
'SET @r=0; UPDATE Users SET ranking= @r:= (@r + 1) ORDER BY totalPixels DESC;',
);
await sequelize.query(
// eslint-disable-next-line max-len
'SET @r=0; UPDATE Users SET dailyRanking= @r:= (@r + 1) ORDER BY dailyTotalPixels DESC;',
);
}
// populate dictionaries
const ranking = await RegUser.findAll({
attributes: [
@ -92,6 +95,9 @@ class Ranks {
}
async resetDailyRanking() {
if (!socketEvents.amIImportant()) {
return;
}
this.prevTop = await saveDailyTop(this.ranks.dailyRanking);
logger.info('Resetting Daily Ranking');
await RegUser.update({ dailyTotalPixels: 0 }, { where: {} });

42
src/core/session.js
View File

@ -5,26 +5,36 @@ import expressSession from 'express-session';
import RedisStore from '../utils/connectRedis';
import client from '../data/redis/client';
import { getHostFromRequest } from '../utils/ip';
import { HOUR, COOKIE_SESSION_NAME } from './constants';
import { SESSION_SECRET } from './config';
export const store = new RedisStore({ client });
const session = expressSession({
name: COOKIE_SESSION_NAME,
store,
secret: SESSION_SECRET,
// The best way to know is to check with your store if it implements the touch method. If it does, then you can safely set resave: false
resave: false,
saveUninitialized: false,
cookie: {
path: '/',
httpOnly: true,
secure: false,
// not setting maxAge or expire makes it a non-persisting cookies
maxAge: 30 * 24 * HOUR,
},
});
/*
* we cache created session middlewares per domain
*/
const middlewareCache = {};
export default session;
export default (req, res, next) => {
const domain = getHostFromRequest(req, false, true);
let session = middlewareCache[domain];
if (!session) {
session = expressSession({
name: COOKIE_SESSION_NAME,
store,
secret: SESSION_SECRET,
resave: false,
saveUninitialized: false,
cookie: {
domain,
httpOnly: true,
secure: false,
maxAge: 30 * 24 * HOUR,
},
});
middlewareCache[domain] = session;
}
return session(req, res, next);
};

9
src/core/tileserver.js
View File

@ -8,7 +8,7 @@ import { Worker } from 'worker_threads';
import logger from './logger';
import canvases from './canvases';
import RedisCanvas from '../data/redis/RedisCanvas';
import socketEvents from '../socket/socketEvents';
import { TILE_FOLDER } from './config';
import {
@ -181,17 +181,16 @@ class CanvasUpdater {
}
}
export function registerChunkChange(canvasId, chunk) {
socketEvents.on('chunkUpdate', (canvasId, chunk) => {
if (CanvasUpdaters[canvasId]) {
CanvasUpdaters[canvasId].registerChunkChange(chunk);
}
}
RedisCanvas.setChunkChangeCallback(registerChunkChange);
});
/*
* starting update loops for canvases
*/
export async function startAllCanvasLoops() {
export default function startAllCanvasLoops() {
if (!fs.existsSync(`${TILE_FOLDER}`)) fs.mkdirSync(`${TILE_FOLDER}`);
const ids = Object.keys(canvases);
for (let i = 0; i < ids.length; i += 1) {

19
src/data/redis/RedisCanvas.js
View File

@ -4,24 +4,13 @@
import { commandOptions } from 'redis';
import { getChunkOfPixel, getOffsetOfPixel } from '../../core/utils';
import socketEvents from '../../socket/socketEvents';
import client from './client';
const UINT_SIZE = 'u8';
class RedisCanvas {
// array of callback functions that gets informed about chunk changes
static registerChunkChange = [];
static setChunkChangeCallback(cb) {
RedisCanvas.registerChunkChange.push(cb);
}
static execChunkChangeCallback(canvasId, cell) {
for (let i = 0; i < RedisCanvas.registerChunkChange.length; i += 1) {
RedisCanvas.registerChunkChange[i](canvasId, cell);
}
}
/*
* Get chunk from redis
* canvasId integer id of canvas
@ -56,14 +45,14 @@ class RedisCanvas {
static async setChunk(i, j, chunk, canvasId) {
const key = `ch:${canvasId}:${i}:${j}`;
await client.set(key, Buffer.from(chunk.buffer));
RedisCanvas.execChunkChangeCallback(canvasId, [i, j]);
socketEvents.broadcastChunkUpdate(canvasId, [i, j]);
return true;
}
static async delChunk(i, j, canvasId) {
const key = `ch:${canvasId}:${i}:${j}`;
await client.del(key);
RedisCanvas.execChunkChangeCallback(canvasId, [i, j]);
socketEvents.broadcastChunkUpdate(canvasId, [i, j]);
return true;
}
@ -97,8 +86,6 @@ class RedisCanvas {
String(color),
],
);
RedisCanvas.execChunkChangeCallback(canvasId, [i, j]);
}
static flushPixels() {

19
src/data/redis/client.js
View File

@ -4,13 +4,14 @@
*/
import fs from 'fs';
import { createClient, defineScript } from 'redis';
import { isMainThread } from 'worker_threads';
import { REDIS_URL } from '../../core/config';
import { REDIS_URL, SHARD_NAME } from '../../core/config';
const scripts = {
placePxl: defineScript({
NUMBER_OF_KEYS: 5,
SCRIPT: fs.readFileSync('./workers/placePixel.lua'),
SCRIPT: fs.readFileSync('./workers/lua/placePixel.lua'),
transformArguments(...args) {
return args.map((a) => ((typeof a === 'string') ? a : a.toString()));
},
@ -30,10 +31,24 @@ const client = createClient(REDIS_URL.startsWith('redis://')
},
);
/*
* for sending messages via cluster
*/
export const pubsub = {
subscriber: null,
publisher: null,
};
export const connect = async () => {
// eslint-disable-next-line no-console
console.log(`Connecting to redis server at ${REDIS_URL}`);
await client.connect();
if (SHARD_NAME && isMainThread) {
const subscriber = client.duplicate();
await subscriber.connect();
pubsub.publisher = client;
pubsub.subscriber = subscriber;
}
};
export default client;

54
src/data/redis/mailCodes.js Normal file
View File

@ -0,0 +1,54 @@
/*
*
* data saving for hourly events
*
*/
import { randomUUID } from 'crypto';
import client from './client';
export const PREFIX = 'mail';
const EXPIRE_TIME = 3600;
/*
* generate and set mail code
* @param email
* @return code
*/
export function setCode(email) {
const code = randomUUID();
const key = `${PREFIX}:${email}`;
client.set(key, code, {
EX: EXPIRE_TIME,
});
return code;
}
/*
* check if email code is correct
* @param email
* @param code
*/
export async function checkCode(email, code) {
const key = `${PREFIX}:${email}`;
const storedCode = await client.get(key);
if (!storedCode || code !== storedCode) {
return false;
}
client.del(key);
return true;
}
/*
* check if code exists
* @param email
* @return null if doesn't, age in seconds if exists
*/
export async function codeExists(email) {
const key = `${PREFIX}:${email}`;
const ttl = await client.ttl(key);
if (!ttl) {
return null;
}
return EXPIRE_TIME - ttl;
}

2
src/routes/api/auth/change_mail.js
View File

@ -2,7 +2,7 @@
* request password change
*/
import mailProvider from '../../../core/mail';
import mailProvider from '../../../core/MailProvider';
import { validatePassword, validateEMail } from '../../../utils/validation';
import { getHostFromRequest } from '../../../utils/ip';

2
src/routes/api/auth/change_name.js
View File

@ -3,7 +3,7 @@
*/
import socketEvents from '../../../socket/SocketEvents';
import socketEvents from '../../../socket/socketEvents';
import { RegUser } from '../../../data/sql';
import { validateName } from '../../../utils/validation';

5
src/routes/api/auth/delete_account.js
View File

@ -2,6 +2,7 @@
* request password change
*/
import socketEvents from '../../../socket/socketEvents';
import { RegUser } from '../../../data/sql';
import { validatePassword } from '../../../utils/validation';
import { compareToHash } from '../../../utils/hash';
@ -35,7 +36,7 @@ export default async (req, res) => {
});
return;
}
const { id } = user;
const { id, name } = user;
const currentPassword = user.regUser.password;
if (!currentPassword || !compareToHash(password, currentPassword)) {
@ -58,6 +59,8 @@ export default async (req, res) => {
RegUser.destroy({ where: { id } });
socketEvents.reloadUser(name);
res.status(200);
res.json({
success: true,

2
src/routes/api/auth/register.js
View File

@ -2,7 +2,7 @@ import Sequelize from 'sequelize';
import logger from '../../../core/logger';
import { RegUser } from '../../../data/sql';
import mailProvider from '../../../core/mail';
import mailProvider from '../../../core/MailProvider';
import getMe from '../../../core/me';
import { getIPFromRequest, getHostFromRequest } from '../../../utils/ip';
import {

4
src/routes/api/auth/resend_verify.js
View File

@ -2,7 +2,7 @@
* request resend of verification mail
*/
import mailProvider from '../../../core/mail';
import mailProvider from '../../../core/MailProvider';
import { getHostFromRequest } from '../../../utils/ip';
export default async (req, res) => {
@ -26,7 +26,7 @@ export default async (req, res) => {
const host = getHostFromRequest(req);
const error = mailProvider.sendVerifyMail(email, name, host, lang);
const error = await mailProvider.sendVerifyMail(email, name, host, lang);
if (error) {
res.status(400);
res.json({

2
src/routes/api/auth/restore_password.js
View File

@ -3,7 +3,7 @@
*/
import mailProvider from '../../../core/mail';
import mailProvider from '../../../core/MailProvider';
import { validateEMail } from '../../../utils/validation';
import { getHostFromRequest } from '../../../utils/ip';

43
src/routes/api/auth/verify.js
View File

@ -2,31 +2,36 @@
* verify mail address
*/
import socketEvents from '../../../socket/SocketEvents';
import socketEvents from '../../../socket/socketEvents';
import getHtml from '../../../ssr/RedirectionPage';
import { getHostFromRequest } from '../../../utils/ip';
import mailProvider from '../../../core/mail';
import { MailProvider } from '../../../core/MailProvider';
import { validateEMail } from '../../../utils/validation';
export default async (req, res) => {
const { token } = req.query;
const { email, token } = req.query;
const { lang } = req;
const { t } = req.ttag;
const name = await mailProvider.verify(token);
const host = getHostFromRequest(req);
if (name) {
// notify websoecket to reconnect user
// thats a bit counter productive because it directly links to the websocket
socketEvents.reloadUser(name);
// ---
const index = getHtml(
t`Mail verification`,
t`You are now verified :)`,
host, lang,
);
res.status(200).send(index);
} else {
// eslint-disable-next-line max-len
const index = getHtml(t`Mail verification`, t`Your mail verification code is invalid or already expired :(, please request a new one.`, host, lang);
res.status(400).send(index);
const error = validateEMail(email);
if (!error) {
const name = await MailProvider.verify(email, token);
if (name) {
// notify websoecket to reconnect user
// thats a bit counter productive because it directly links to the websocket
socketEvents.reloadUser(name);
// ---
const index = getHtml(
t`Mail verification`,
t`You are now verified :)`,
host, lang,
);
res.status(200).send(index);
return;
}
}
// eslint-disable-next-line max-len
const index = getHtml(t`Mail verification`, t`Your mail verification code is invalid or already expired :(, please request a new one.`, host, lang);
res.status(400).send(index);
};

2
src/routes/api/block.js
View File

@ -5,7 +5,7 @@
*/
import logger from '../../core/logger';
import socketEvents from '../../socket/SocketEvents';
import socketEvents from '../../socket/socketEvents';
import { RegUser, UserBlock, Channel } from '../../data/sql';
async function block(req, res) {

2
src/routes/api/blockdm.js
View File

@ -4,7 +4,7 @@
*
*/
import logger from '../../core/logger';
import socketEvents from '../../socket/SocketEvents';
import socketEvents from '../../socket/socketEvents';
async function blockdm(req, res) {
const { block } = req.body;

14
src/routes/api/chathistory.js
View File

@ -7,11 +7,6 @@ import chatProvider from '../../core/ChatProvider';
async function chatHistory(req, res) {
let { cid, limit } = req.query;
res.set({
'Cache-Control': 'no-cache, no-store, must-revalidate',
Pragma: 'no-cache',
Expires: '0',
});
if (!cid || !limit) {
res.status(400);
@ -42,19 +37,10 @@ async function chatHistory(req, res) {
return;
}
// try {
const history = await chatProvider.getHistory(cid, limit);
res.json({
history,
});
/*
} catch {
res.status(500);
res.json({
errors: ['Can not fetch messages'],
});
}
*/
}
export default chatHistory;

12
src/routes/api/index.js
View File

@ -17,10 +17,19 @@ import blockdm from './blockdm';
import modtools from './modtools';
import baninfo from './baninfo';
import getiid from './getiid';
import shards from './shards';
const router = express.Router();
// set cache-control
router.use((req, res, next) => {
res.set({
'Cache-Control': 'no-cache, no-store, must-revalidate',
Expires: '0',
});
next();
});
router.use(express.json());
// eslint-disable-next-line no-unused-vars
@ -35,6 +44,7 @@ router.use((err, req, res, next) => {
router.post('/captcha', captcha);
router.get('/baninfo', baninfo);
router.get('/getiid', getiid);
router.get('/shards', shards);
/*
* get user session

2
src/routes/api/leavechan.js
View File

@ -5,7 +5,7 @@
*/
import logger from '../../core/logger';
import socketEvents from '../../socket/SocketEvents';
import socketEvents from '../../socket/socketEvents';
async function leaveChan(req, res) {
const channelId = parseInt(req.body.channelId, 10);

4
src/routes/api/me.js
View File

@ -24,10 +24,6 @@ export default async (req, res, next) => {
}
// https://stackoverflow.com/questions/49547/how-to-control-web-page-caching-across-all-browsers
res.set({
'Cache-Control': 'no-cache, no-store, must-revalidate',
Expires: '0',
});
res.json(userdata);
} catch (error) {
next(error);

20
src/routes/api/shards.js Normal file
View File

@ -0,0 +1,20 @@
/*
* print information for shards
*/
import socketEvents from '../../socket/socketEvents';
async function shards(req, res, next) {
try {
if (!socketEvents.isCluster) {
res.status(400).json({
errors: ['Not running as cluster'],
});
return;
}
res.status(200).json(socketEvents.shardOnlineCounters);
} catch (err) {
next(err);
}
}
export default shards;

9
src/routes/chunks.js
View File

@ -7,11 +7,11 @@
import etag from 'etag';
import RedisCanvas from '../data/redis/RedisCanvas';
import logger from '../core/logger';
import socketEvents from '../socket/socketEvents';
const chunkEtags = new Map();
RedisCanvas.setChunkChangeCallback((canvasId, cell) => {
const [x, y] = cell;
chunkEtags.delete(`${canvasId}:${x}:${y}`);
socketEvents.on('chunkUpdate', (canvasId, [i, j]) => {
chunkEtags.delete(`${canvasId}:${i}:${j}`);
});
/*
@ -27,6 +27,9 @@ export default async (req, res, next) => {
const x = parseInt(paramX, 10);
const y = parseInt(paramY, 10);
try {
res.set({
'Access-Control-allow-origin': '*',
});
// botters where using cachebreakers to update via chunk API
// lets not allow that for now
if (Object.keys(req.query).length !== 0) {

91
src/routes/index.js
View File

@ -18,6 +18,7 @@ import api from './api';
import { assets } from '../core/assets';
import { expressTTag } from '../core/ttag';
import corsMiddleware from '../utils/corsMiddleware';
import generateGlobePage from '../ssr/Globe';
import generatePopUpPage from '../ssr/PopUp';
import generateMainPage from '../ssr/Main';
@ -29,37 +30,18 @@ import { GUILDED_INVITE } from '../core/config';
const router = express.Router();
/*
* void info
* Serving Chunks
*/
router.get('/void', voidl);
/*
* ranking of pixels placed
* daily and total
*/
router.get('/ranking', ranking);
/*
* give: date per query
* returns: array of HHMM backups available
*/
router.get('/history', history);
router.get(
'/chunks/:c([0-9]+)/:x([0-9]+)/:y([0-9]+)(/)?:z([0-9]+)?.bmp',
chunks,
);
/*
* zoomed tiles
*/
router.use('/tiles', tiles);
/*
* adminapi
*/
router.use('/adminapi', adminapi);
/*
* serve captcha
*/
router.get('/captcha.svg', captcha);
/*
* public folder
* (this should be served with nginx or other webserver)
@ -77,12 +59,9 @@ router.use('/guilded', (req, res) => {
});
/*
* Serving Chunks
* adminapi
*/
router.get(
'/chunks/:c([0-9]+)/:x([0-9]+)/:y([0-9]+)(/)?:z([0-9]+)?.bmp',
chunks,
);
router.use('/adminapi', adminapi);
/*
* Following with translations
@ -90,16 +69,6 @@ router.get(
*/
router.use(expressTTag);
/*
* API calls
*/
router.use('/api', api);
/*
* Password Reset Link
*/
router.use('/reset_password', resetPassword);
//
// 3D Globe (react generated)
// -----------------------------------------------------------------------------
@ -149,7 +118,7 @@ router.use(
return;
}
res.status(200).send(generatePopUpPage(req.lang));
res.status(200).send(generatePopUpPage(req));
},
);
@ -173,7 +142,47 @@ router.get('/', (req, res) => {
return;
}
res.status(200).send(generateMainPage(req.lang));
res.status(200).send(generateMainPage(req));
});
/*
* Password Reset Link
*/
router.use('/reset_password', resetPassword);
/*
* Following with CORS
* ---------------------------------------------------------------------------
*/
router.use(corsMiddleware);
/*
* API calls
*/
router.use('/api', api);
/*
* void info
*/
router.get('/void', voidl);
/*
* ranking of pixels placed
* daily and total
*/
router.get('/ranking', ranking);
/*
* give: date per query
* returns: array of HHMM backups available
*/
router.get('/history', history);
/*
* serve captcha
*/
router.get('/captcha.svg', captcha);
export default router;

2
src/routes/reset_password.js
View File

@ -8,7 +8,7 @@ import express from 'express';
import logger from '../core/logger';
import getPasswordResetHtml from '../ssr/PasswordReset';
import mailProvider from '../core/mail';
import mailProvider from '../core/MailProvider';
import { RegUser } from '../data/sql';

3
src/routes/tiles.js
View File

@ -19,6 +19,9 @@ const router = express.Router();
*/
router.use('/:c([0-9]+)/:z([0-9]+)/:x([0-9]+)/:y([0-9]+).webp',
(req, res, next) => {
res.set({
'Access-Control-allow-origin': '*',
});
const { c: id } = req.params;
const canvas = canvases[id];
if (!canvas) {

32
src/server.js
View File

@ -17,14 +17,16 @@ import chatProvider from './core/ChatProvider';
import rpgEvent from './core/RpgEvent';
import canvasCleaner from './core/CanvasCleaner';
import socketEvents from './socket/socketEvents';
import SocketServer from './socket/SocketServer';
import APISocketServer from './socket/APISocketServer';
import { PORT, HOST, HOURLY_EVENT } from './core/config';
import {
PORT, HOST, HOURLY_EVENT, SHARD_NAME,
} from './core/config';
import { SECOND } from './core/constants';
import { startAllCanvasLoops } from './core/tileserver';
import startAllCanvasLoops from './core/tileserver';
const app = express();
app.disable('x-powered-by');
@ -78,15 +80,11 @@ app.use(routes);
sequelize.sync({ alter: { drop: false } })
// connect to redis
.then(connectRedis)
.then(() => {
rankings.initialize();
.then(async () => {
chatProvider.initialize();
startAllCanvasLoops();
usersocket.initialize();
apisocket.initialize();
if (HOURLY_EVENT) {
rpgEvent.initialize();
}
canvasCleaner.initialize();
// start http server
const startServer = () => {
@ -108,4 +106,22 @@ sequelize.sync({ alter: { drop: false } })
startServer();
}, 5000);
});
})
.then(async () => {
await socketEvents.initialize();
})
.then(async () => {
/*
* initializers that rely on the cluster being fully established
* i.e. to know if it is the shard that runs the event
*/
if (socketEvents.isCluster && socketEvents.amIImportant()) {
logger.info('I am the main shard');
}
rankings.initialize();
if (HOURLY_EVENT && !SHARD_NAME) {
// TODO make it wok in a cluster
logger.info('Initializing RpgEvent');
rpgEvent.initialize();
}
});

3
src/socket/APISocketServer.js
View File

@ -9,7 +9,7 @@
import WebSocket from 'ws';
import socketEvents from './SocketEvents';
import socketEvents from './socketEvents';
import chatProvider, { ChatProvider } from '../core/ChatProvider';
import { RegUser } from '../data/sql';
import { getIPFromRequest } from '../utils/ip';
@ -61,7 +61,6 @@ class APISocketServer {
this.ping = this.ping.bind(this);
this.broadcastChatMessage = this.broadcastChatMessage.bind(this);
socketEvents.onAsync('broadcast', this.broadcast);
socketEvents.onAsync('onlineCounter', this.broadcastOnlineCounter);
socketEvents.onAsync('pixelUpdate', this.broadcastPixelBuffer);
socketEvents.onAsync('chatMessage', this.broadcastChatMessage);

254
src/socket/MessageBroker.js Normal file
View File

@ -0,0 +1,254 @@
/*
* sends messages to other ppfun instances
* to work as cluster
*/
/* eslint-disable no-console */
import { SHARD_NAME } from '../core/config';
import SocketEvents from './SockEvents';
import OnlineCounter from './packets/OnlineCounter';
import PixelUpdate from './packets/PixelUpdateServer';
import PixelUpdateMB from './packets/PixelUpdateMB';
import ChunkUpdate from './packets/ChunkUpdate';
import { pubsub } from '../data/redis/client';
class MessageBroker extends SocketEvents {
constructor() {
super();
this.isCluster = true;
this.thisShard = SHARD_NAME;
/*
* all other shards
*/
this.shards = {};
/*
* online counter of all shards including ourself
*/
this.shardOnlineCounters = [];
this.publisher = {
publish: () => {},
};
this.subscriber = {
subscribe: () => {},
unsubscribe: () => {},
};
this.checkHealth = this.checkHealth.bind(this);
setInterval(this.checkHealth, 10000);
}
// TODO imprement shared storage that is run by main shard
async initialize() {
/*
* broadcast channel for staus messages between shards
*/
this.publisher = pubsub.publisher;
this.subscriber = pubsub.subscriber;
await this.subscriber.subscribe('bc', (...args) => {
this.onShardBCMessage(...args);
});
// give other shards 30s to announce themselves
await new Promise((resolve) => {
setTimeout(resolve, 25000);
});
console.log('CLUSTER: Initialized message broker');
}
async onShardBCMessage(message) {
try {
/*
* messages from own shard get dropped
*/
if (!message || message.startsWith(this.thisShard)) {
return;
}
const comma = message.indexOf(',');
/*
* any other package in the form of 'shard:type,JSONArrayData'
* straight sent over websocket
*/
if (~comma) {
console.log('CLUSTER: Broadcast', message);
const key = message.slice(message.indexOf(':') + 1, comma);
const val = JSON.parse(message.slice(comma + 1));
super.emit(key, ...val);
return;
}
if (!this.shards[message]) {
console.log(`CLUSTER: Shard ${message} connected`);
await this.subscriber.subscribe(
message,
(buffer) => this.onShardBinaryMessage(buffer, message),
true,
);
// immediately give new shards informations
this.publisher.publish('bc', this.thisShard);
}
this.shards[message] = Date.now();
return;
} catch (err) {
console.error(`CLUSTER: Error on broadcast message: ${err.message}`);
}
}
getLowestActiveShard() {
let lowest = 0;
let lShard = null;
this.shardOnlineCounters.forEach((shardData) => {
const [shard, cnt] = shardData;
if (cnt.total < lowest || !lShard) {
lShard = shard;
lowest = cnt.total;
}
});
return lShard || this.thisShard;
}
amIImportant() {
/*
* important main shard does tasks like running RpgEvent
* or updating rankings
*/
return !this.shardOnlineCounters[0]
|| this.shardOnlineCounters[0][0] === this.thisShard;
}
updateShardOnlineCounter(shard, cnt) {
const shardCounter = this.shardOnlineCounters.find(
(c) => c[0] === shard,
);
if (!shardCounter) {
this.shardOnlineCounters.push([shard, cnt]);
this.shardOnlineCounters.sort((a, b) => a[0].localeCompare(b[0]));
} else {
shardCounter[1] = cnt;
}
this.sumOnlineCounters();
}
onShardBinaryMessage(buffer, shard) {
if (buffer.byteLength === 0) return;
const opcode = buffer[0];
try {
switch (opcode) {
case PixelUpdateMB.OP_CODE: {
const puData = PixelUpdateMB.hydrate(buffer);
super.emit('pixelUpdate', ...puData);
const chunkId = puData[1];
const chunk = [chunkId >> 8, chunkId & 0xFF];
super.emit('chunkUpdate', puData[0], chunk);
break;
}
case ChunkUpdate.OP_CODE: {
super.emit('chunkUpdate', ...ChunkUpdate.hydrate(buffer));
break;
}
case OnlineCounter.OP_CODE: {
const data = new DataView(
buffer.buffer,
buffer.byteOffset,
buffer.length,
);
const cnt = OnlineCounter.hydrate(data);
this.updateShardOnlineCounter(shard, cnt);
break;
}
default:
// nothing
}
} catch (err) {
// eslint-disable-next-line max-len
console.error(`CLUSTER: Error on binery message of shard ${shard}: ${err.message}`);
}
}
sumOnlineCounters() {
const newCounter = {};
this.shardOnlineCounters.forEach((shardData) => {
const [, cnt] = shardData;
Object.keys(cnt).forEach((canv) => {
const num = cnt[canv];
if (newCounter[canv]) {
newCounter[canv] += num;
} else {
newCounter[canv] = num;
}
});
});
this.onlineCounter = newCounter;
}
emit(key, ...args) {
super.emit(key, ...args);
const msg = `${this.thisShard}:${key},${JSON.stringify(args)}`;
this.publisher.publish('bc', msg);
}
/*
* broadcast pixel message via websocket
* @param canvasId number ident of canvas
* @param chunkid number id consisting of i,j chunk coordinates
* @param pxls buffer with offset and color of one or more pixels
*/
broadcastPixels(
canvasId,
chunkId,
pixels,
) {
const i = chunkId >> 8;
const j = chunkId & 0xFF;
this.publisher.publish(
this.thisShard,
PixelUpdateMB.dehydrate(canvasId, i, j, pixels),
);
const buffer = PixelUpdate.dehydrate(i, j, pixels);
super.emit('pixelUpdate', canvasId, chunkId, buffer);
super.emit('chunkUpdate', canvasId, [i, j]);
}
broadcastChunkUpdate(
canvasId,
chunk,
) {
this.publisher.publish(
this.thisShard,
ChunkUpdate.dehydrate(canvasId, chunk),
);
super.emit('chunkUpdate', canvasId, chunk);
}
broadcastOnlineCounter(online) {
this.updateShardOnlineCounter(this.thisShard, online);
let buffer = OnlineCounter.dehydrate(online);
// send our online counter to other shards
this.publisher.publish(this.thisShard, buffer);
// send total counter to our players
buffer = OnlineCounter.dehydrate(this.onlineCounter);
super.emit('onlineCounter', buffer);
}
checkHealth() {
// remove disconnected shards
const threshold = Date.now() - 30000;
const { shards } = this;
Object.keys(shards).forEach((shard) => {
if (shards[shard] < threshold) {
console.log(`CLUSTER: Shard ${shard} disconnected`);
delete shards[shard];
const counterIndex = this.shardOnlineCounters.findIndex(
(c) => c[0] === shard,
);
if (~counterIndex) {
this.shardOnlineCounters.splice(counterIndex, 1);
}
this.subscriber.unsubscribe(shard);
}
});
// send keep alive to others
this.publisher.publish('bc', this.thisShard);
}
}
export default MessageBroker;

53
src/socket/SocketEvents.js → src/socket/SockEvents.js
View File

@ -22,6 +22,21 @@ class SocketEvents extends EventEmitter {
};
}
// eslint-disable-next-line class-methods-use-this
async initialize() {
// nothing, only for child classes
}
// eslint-disable-next-line class-methods-use-this
getLowestActiveShard() {
return null;
}
// eslint-disable-next-line class-methods-use-this
amIImportant() {
return true;
}
/*
* async event
*/
@ -33,14 +48,6 @@ class SocketEvents extends EventEmitter {
});
}
/*
* broadcast message via websocket
* @param message Buffer Message to send
*/
broadcast(message) {
this.emit('broadcast', message);
}
/*
* broadcast pixel message via websocket
* @param canvasId number ident of canvas
@ -52,8 +59,34 @@ class SocketEvents extends EventEmitter {
chunkId,
pixels,
) {
const buffer = PixelUpdate.dehydrate(chunkId, pixels);
const i = chunkId >> 8;
const j = chunkId & 0xFF;
const buffer = PixelUpdate.dehydrate(i, j, pixels);
this.emit('pixelUpdate', canvasId, chunkId, buffer);
this.emit('chunkUpdate', canvasId, [i, j]);
}
/*
* chunk updates from event, image upload, etc.
* everything thats not a pixelUpdate and changes chunks
* @param canvasId
* @param chunk [i,j] chunk coordinates
*/
broadcastChunkUpdate(
canvasId,
chunk,
) {
this.emit('chunkUpdate', canvasId, chunk);
}
/*
* ask other shards to send email for us,
* only used when USE_MAILER is false
* @param type type of mail to send
* @param args
*/
sendMail(...args) {
this.emit('mail', ...args);
}
/*
@ -169,4 +202,4 @@ class SocketEvents extends EventEmitter {
}
}
export default new SocketEvents();
export default SocketEvents;

8
src/socket/SocketClient.js
View File

@ -14,6 +14,7 @@ import RegisterMultipleChunks from './packets/RegisterMultipleChunks';
import DeRegisterChunk from './packets/DeRegisterChunk';
import ChangedMe from './packets/ChangedMe';
import Ping from './packets/Ping';
import { shardHost } from '../store/actions/fetch';
const chunks = [];
@ -43,9 +44,10 @@ class SocketClient extends EventEmitter {
console.log('WebSocket already open, not starting');
}
this.timeLastConnecting = Date.now();
const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
const url = `${protocol}//${window.location.hostname}${
window.location.port ? `:${window.location.port}` : ''
const url = `${
window.location.protocol === 'https:' ? 'wss:' : 'ws:'
}//${
shardHost || window.location.host
}/ws`;
this.ws = new WebSocket(url);
this.ws.binaryType = 'arraybuffer';

29
src/socket/SocketServer.js
View File

@ -19,7 +19,7 @@ import DeRegisterMultipleChunks from './packets/DeRegisterMultipleChunks';
import ChangedMe from './packets/ChangedMe';
import OnlineCounter from './packets/OnlineCounter';
import socketEvents from './SocketEvents';
import socketEvents from './socketEvents';
import chatProvider, { ChatProvider } from '../core/ChatProvider';
import authenticateClient from './authenticateClient';
import { drawByOffsets } from '../core/draw';
@ -111,7 +111,6 @@ class SocketServer {
});
});
socketEvents.on('broadcast', this.broadcast);
socketEvents.on('onlineCounter', this.broadcast);
socketEvents.on('pixelUpdate', this.broadcastPixelBuffer);
socketEvents.on('reloadUser', this.reloadUser);
@ -189,9 +188,12 @@ class SocketServer {
}
// CORS
const { origin } = headers;
if (!origin || !origin.endsWith(getHostFromRequest(request, false))) {
const host = getHostFromRequest(request, false, true);
if (!origin
|| !`.${origin.slice(origin.indexOf('//') + 2)}`.endsWith(host)
) {
// eslint-disable-next-line max-len
logger.info(`Rejected CORS request on websocket from ${ip} via ${headers.origin}, expected ${getHostFromRequest(request, false)}`);
logger.info(`Rejected CORS request on websocket from ${ip} via ${headers.origin}, expected ${getHostFromRequest(request, false, true)}`);
socket.write('HTTP/1.1 403 Forbidden\r\n\r\n');
socket.destroy();
return;
@ -433,9 +435,6 @@ class SocketServer {
* if DM channel, make sure that other user has DM open
* (needed because we allow user to leave one-sided
* and auto-join on message)
* TODO: if we scale and have multiple websocket servers at some point
* this might be an issue. We would hve to make a shared list of online
* users and act based on that on 'chatMessage' event
*/
const dmUserId = chatProvider.checkIfDm(user, channelId);
if (dmUserId) {
@ -443,11 +442,17 @@ class SocketServer {
if (!dmWs
|| !chatProvider.userHasChannelAccess(dmWs.user, channelId)
) {
await ChatProvider.addUserToChannel(
dmUserId,
channelId,
[ws.name, 1, Date.now(), user.id],
);
// TODO this is really ugly
// DMS have to be rethought
if (!user.addedDM) user.addedDM = [];
if (!user.addedDM.includes(dmUserId)) {
await ChatProvider.addUserToChannel(
dmUserId,
channelId,
[ws.name, 1, Date.now(), user.id],
);
user.addedDM.push(dmUserId);
}
}
}

30
src/socket/packets/ChunkUpdate.js Normal file
View File

@ -0,0 +1,30 @@
/*
* notify that chunk changed
* (not sent over websocket, server only)
*/
const OP_CODE = 0xC4;
export default {
OP_CODE,
/*
* @return canvasId, [i, j]
*/
hydrate(data) {
const canvasId = data[1];
const chunk = [data[2], data[3]];
return [canvasId, chunk];
},
/*
* @param canvasId,
* chunkid id consisting of chunk coordinates
*/
dehydrate(canvasId, [i, j]) {
return Buffer.from({
OP_CODE,
canvasId,
i,
j,
});
},
};

44
src/socket/packets/PixelUpdateMB.js Normal file
View File

@ -0,0 +1,44 @@
/*
* Packet for sending and receiving pixels over Message Broker between shards
* Multiple pixels can be sent at once
*
*/
const OP_CODE = 0xC1;
export default {
OP_CODE,
/*
* returns info and PixelUpdate package to send to clients
*/
hydrate(data) {
const canvasId = data[1];
data.writeUInt8(OP_CODE, 1);
const chunkId = data.readUInt16BE(2);
const pixelUpdate = Buffer.from(
data.buffer,
data.byteOffset + 1,
data.length - 1,
);
return [
canvasId,
chunkId,
pixelUpdate,
];
},
/*
* @param canvasId
* @param chunkId id consisting of chunk coordinates
* @param pixels Buffer with offset and color of one or more pixels
*/
dehydrate(canvasId, i, j, pixels) {
const index = new Uint8Array([
OP_CODE,
canvasId,
i,
j,
]);
return Buffer.concat([index, pixels]);
},
};

4
src/socket/packets/PixelUpdateServer.js
View File

@ -43,8 +43,8 @@ export default {
* @param chunkId id consisting of chunk coordinates
* @param pixels Buffer with offset and color of one or more pixels
*/
dehydrate(chunkId, pixels) {
const index = new Uint8Array([OP_CODE, chunkId >> 8, chunkId & 0xFF]);
dehydrate(i, j, pixels) {
const index = new Uint8Array([OP_CODE, i, j]);
return Buffer.concat([index, pixels]);
},
};

10
src/socket/socketEvents.js Normal file
View File

@ -0,0 +1,10 @@
import SocketEvents from './SockEvents';
import MessageBroker from './MessageBroker';
import { SHARD_NAME } from '../core/config';
/*
* if we are a shard in a cluster, do messaging to others via redis
*/
const socketEvents = (SHARD_NAME) ? new MessageBroker() : new SocketEvents();
export default socketEvents;

10
src/ssr/Main.jsx
View File

@ -8,8 +8,9 @@
import { langCodeToCC } from '../utils/location';
import ttags, { getTTag } from '../core/ttag';
import { styleassets, assets } from '../core/assets';
import socketEvents from '../socket/socketEvents';
import { BACKUP_URL } from '../core/config';
import { getHostFromRequest } from '../utils/ip';
/*
* generate language list
@ -35,14 +36,19 @@ if (BACKUP_URL) {
* @param lang language code
* @return html of mainpage
*/
function generateMainPage(lang) {
function generateMainPage(req) {
const { lang } = req;
const host = getHostFromRequest(req, false);
const ssvR = {
...ssv,
shard: (host.startsWith(`${socketEvents.thisShard}.`))
? '' : socketEvents.getLowestActiveShard(),
lang: lang === 'default' ? 'en' : lang,
};
const scripts = (assets[`client-${lang}`])
? assets[`client-${lang}`].js
: assets.client.js;
const { t } = getTTag(lang);
const html = `

10
src/ssr/PopUp.jsx
View File

@ -7,10 +7,10 @@
import { langCodeToCC } from '../utils/location';
import ttags, { getTTag } from '../core/ttag';
/* this will be set by webpack */
import socketEvents from '../socket/socketEvents';
import { styleassets, assets } from '../core/assets';
import { BACKUP_URL } from '../core/config';
import { getHostFromRequest } from '../utils/ip';
/*
* generate language list
@ -35,9 +35,13 @@ if (BACKUP_URL) {
* @param lang language code
* @return html of mainpage
*/
function generatePopUpPage(lang) {
function generatePopUpPage(req) {
const { lang } = req;
const host = getHostFromRequest(req);
const ssvR = {
...ssv,
shard: (host.startsWith(`${socketEvents.thisShard}.`))
? null : socketEvents.getLowestActiveShard(),
lang: lang === 'default' ? 'en' : lang,
};
const script = (assets[`popup-${lang}`])

62
src/store/actions/fetch.js
View File

@ -8,17 +8,30 @@ import { t } from 'ttag';
import { dateToString } from '../../core/utils';
export const shardHost = (function getShardHost() {
if (!window.ssv || !window.ssv.shard) {
return '';
}
const hostParts = window.location.host.split('.');
if (hostParts.length > 2) {
hostParts.shift();
}
return `${window.ssv.shard}.${hostParts.join('.')}`;
}());
export const shardOrigin = shardHost
&& `${window.location.protocol}//${shardHost}`;
/*
* Adds customizeable timeout to fetch
* defaults to 8s
*/
async function fetchWithTimeout(resource, options = {}) {
const { timeout = 8000 } = options;
async function fetchWithTimeout(url, options = {}) {
const { timeout = 10000 } = options;
const controller = new AbortController();
const id = setTimeout(() => controller.abort(), timeout);
const response = await fetch(resource, {
const response = await fetch(url, {
...options,
signal: controller.signal,
});
@ -62,11 +75,19 @@ async function parseAPIresponse(response) {
* @param body Body of request
* @return Object with response or error Array
*/
async function makeAPIPOSTRequest(url, body) {
async function makeAPIPOSTRequest(
url,
body,
credentials = true,
addShard = true,
) {
if (addShard) {
url = `${shardOrigin}${url}`;
}
try {
const response = await fetchWithTimeout(url, {
method: 'POST',
credentials: 'include',
credentials: (credentials) ? 'include' : 'omit',
headers: {
'Content-Type': 'application/json',
},
@ -86,10 +107,17 @@ async function makeAPIPOSTRequest(url, body) {
* @param url URL of get api endpoint
* @return Object with response or error Array
*/
async function makeAPIGETRequest(url) {
async function makeAPIGETRequest(
url,
credentials = true,
addShard = true,
) {
if (addShard) {
url = `${shardOrigin}${url}`;
}
try {
const response = await fetchWithTimeout(url, {
credentials: 'include',
credentials: (credentials) ? 'include' : 'omit',
});
return parseAPIresponse(response);
@ -193,8 +221,10 @@ export async function requestSolveCaptcha(text, captchaid) {
export async function requestHistoricalTimes(day, canvasId) {
try {
const date = dateToString(day);
const url = `history?day=${date}&id=${canvasId}`;
// Not going over shard url
const url = `/history?day=${date}&id=${canvasId}`;
const response = await fetchWithTimeout(url, {
credentials: 'omit',
timeout: 45000,
});
if (response.status !== 200) {
@ -212,6 +242,19 @@ export async function requestHistoricalTimes(day, canvasId) {
}
}
export async function requestChatMessages(cid) {
const response = await fetch(
`${shardOrigin}/api/chathistory?cid=${cid}&limit=50`,
{ credentials: 'include' },
);
// timeout in order to not spam api requests and get rate limited
if (response.ok) {
const { history } = await response.json();
return history;
}
return null;
}
export function requestPasswordChange(newPassword, password) {
return makeAPIPOSTRequest(
'/api/auth/change_passwd',
@ -278,7 +321,8 @@ export function requestDeleteAccount(password) {
export function requestRankings() {
return makeAPIGETRequest(
'ranking',
'/ranking',
false,
);
}

16
src/store/actions/thunks.js
View File

@ -7,6 +7,7 @@ import {
requestBlockDm,
requestLeaveChan,
requestRankings,
requestChatMessages,
requestMe,
} from './fetch';
@ -89,21 +90,12 @@ export function fetchMe() {
};
}
export function fetchChatMessages(
cid,
) {
export function fetchChatMessages(cid) {
return async (dispatch) => {
dispatch(setChatFetching(true));
const response = await fetch(`/api/chathistory?cid=${cid}&limit=50`, {
credentials: 'include',
});
/*
* timeout in order to not spam api requests and get rate limited
*/
if (response.ok) {
const history = await requestChatMessages(cid);
if (history) {
setTimeout(() => { dispatch(setChatFetching(false)); }, 500);
const { history } = await response.json();
dispatch(receiveChatHistory(cid, history));
} else {
setTimeout(() => { dispatch(setChatFetching(false)); }, 5000);

6
src/ui/ChunkLoader2D.js
View File

@ -4,6 +4,7 @@
import ChunkRGB from './ChunkRGB';
import { TILE_SIZE, TILE_ZOOM_LEVEL } from '../core/constants';
import { shardOrigin } from '../store/actions/fetch';
import {
loadingTiles,
loadImage,
@ -279,7 +280,7 @@ class ChunkLoader {
const center = [zoom, cx, cy];
this.store.dispatch(requestBigChunk(center));
try {
const url = `chunks/${this.canvasId}/${cx}/${cy}.bmp`;
const url = `${shardOrigin}/chunks/${this.canvasId}/${cx}/${cy}.bmp`;
const response = await fetch(url);
if (response.ok) {
const arrayBuffer = await response.arrayBuffer();
@ -303,7 +304,8 @@ class ChunkLoader {
const center = [zoom, cx, cy];
this.store.dispatch(requestBigChunk(center));
try {
const url = `tiles/${this.canvasId}/${zoom}/${cx}/${cy}.webp`;
// eslint-disable-next-line max-len
const url = `${shardOrigin}/tiles/${this.canvasId}/${zoom}/${cx}/${cy}.webp`;
const img = await loadImage(url);
chunkRGB.fromImage(img);
this.store.dispatch(receiveBigChunk(center));

7
src/ui/ChunkLoader3D.js
View File

@ -3,6 +3,7 @@
*
*/
import Chunk from './ChunkRGB3D';
import {
requestBigChunk,
receiveBigChunk,
@ -12,9 +13,7 @@ import {
getChunkOfPixel,
getOffsetOfPixel,
} from '../core/utils';
import Chunk from './ChunkRGB3D';
import { shardOrigin } from '../store/actions/fetch';
class ChunkLoader {
store = null;
@ -91,7 +90,7 @@ class ChunkLoader {
const center = [0, cx, cz];
this.store.dispatch(requestBigChunk(center));
try {
const url = `chunks/${this.canvasId}/${cx}/${cz}.bmp`;
const url = `${shardOrigin}/chunks/${this.canvasId}/${cx}/${cz}.bmp`;
const response = await fetch(url);
if (response.ok) {
const arrayBuffer = await response.arrayBuffer();

39
src/utils/corsMiddleware.js Normal file
View File

@ -0,0 +1,39 @@
/*
* set CORS Headers
*/
import { CORS_HOSTS } from '../core/config';
export default (req, res, next) => {
if (!CORS_HOSTS || !req.headers.origin) {
next();
return;
}
const { origin } = req.headers;
const host = origin.slice(origin.indexOf('//') + 2);
/*
* form .domain.tld will accept both domain.tld and x.domain.tld
*/
const isAllowed = CORS_HOSTS.some((c) => c === host
|| (c.startsWith('.') && (host.endsWith(c) || host === c.slice(1))));
if (!isAllowed) {
next();
return;
}
res.set({
'Access-Control-Allow-Origin': origin,
'Access-Control-Allow-Credentials': 'true',
});
if (req.method === 'OPTIONS') {
res.set({
'Access-Control-Allow-Headers': 'Content-Type',
'Access-Control-Allow-Methods': 'GET,POST',
});
res.sendStatus(200);
return;
}
next();
};

11
src/utils/ip.js
View File

@ -50,11 +50,18 @@ function ip4NumToStr(ipNum) {
* @param includeProto if we include protocol (https, http)
* @return host (like pixelplanet.fun)
*/
export function getHostFromRequest(req, includeProto = true) {
export function getHostFromRequest(req, includeProto = true, stripSub = false) {
const { headers } = req;
const host = headers['x-forwarded-host']
let host = headers['x-forwarded-host']
|| headers.host
|| headers[':authority'];
if (stripSub) {
if (host.lastIndexOf('.') !== host.indexOf('.')) {
host = host.slice(host.indexOf('.'));
} else {
host = `.${host}`;
}
}
if (!includeProto) {
return host;
}

4
webpack.config.server.js
View File

@ -160,8 +160,8 @@ module.exports = ({
to: path.resolve('dist', 'captchaFonts'),
},
{
from: path.resolve('src', 'data', 'redis', 'lua', 'placePixel.lua'),
to: path.resolve('dist', 'workers', 'placePixel.lua'),
from: path.resolve('src', 'data', 'redis', 'lua'),
to: path.resolve('dist', 'workers', 'lua'),
},
],
}),