iampeter/pixelplanet
1
Fork 0
forked from ppfun/pixelplanet
Code Pull Requests Activity

Set pixels in redis in batches

Browse Source 
finish node-redis update by making sure that everything that needs redis
runs after redis connected
This commit is contained in:
HF 2022-04-06 21:50:34 +02:00
parent 56bc4352b2
commit e95b6ae8d3
28 changed files with 405 additions and 250 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
src/backup.js
View File

@ -15,7 +15,7 @@ import fs from 'fs';
import os from 'os'; import os from 'os';
import { spawn } from 'child_process'; import { spawn } from 'child_process';
import path from 'path'; import path from 'path';
import redis from 'redis'; import { createClient } from 'redis';
import { import {
@ -53,10 +53,29 @@ if (!CANVAS_REDIS_URL || !BACKUP_REDIS_URL || !BACKUP_DIR) {
process.exit(1); process.exit(1);
} }
const canvasRedis = redis const canvasRedis = createClient(CANVAS_REDIS_URL
.createClient(CANVAS_REDIS_URL); .startsWith('redis://')
const backupRedis = redis ? {
.createClient(BACKUP_REDIS_URL); url: CANVAS_REDIS_URL,
}
: {
socket: {
path: CANVAS_REDIS_URL,
},
},
);
const backupRedis = createClient(BACKUP_REDIS_URL
.startsWith('redis://')
? {
url: BACKUP_REDIS_URL,
}
: {
socket: {
path: BACKUP_REDIS_URL,
},
},
);
//
canvasRedis.on('error', () => { canvasRedis.on('error', () => {
console.error('Could not connect to canvas redis'); console.error('Could not connect to canvas redis');
process.exit(1); process.exit(1);
@ -161,4 +180,6 @@ async function trigger() {
} }
console.log('Starting backup...'); console.log('Starting backup...');
trigger(); canvasRedis.connect()
.then(() => backupRedis.connect())
.then(() => trigger());

83
src/captchaserver.js
View File

@ -11,6 +11,7 @@ import http from 'http';
import url from 'url'; import url from 'url';
import ppfunCaptcha from 'ppfun-captcha'; import ppfunCaptcha from 'ppfun-captcha';
import { connect as connectRedis } from './data/redis';
import { getIPFromRequest } from './utils/ip'; import { getIPFromRequest } from './utils/ip';
import { setCaptchaSolution } from './utils/captcha'; import { setCaptchaSolution } from './utils/captcha';
import { getRandomString } from './core/utils'; import { getRandomString } from './core/utils';
@ -34,32 +35,44 @@ const server = http.createServer((req, res) => {
const urlObject = url.parse(req.url, true); const urlObject = url.parse(req.url, true);
if (req.method === 'GET' && urlObject.pathname.endsWith('.svg')) { if (req.method === 'GET' && urlObject.pathname.endsWith('.svg')) {
const captcha = ppfunCaptcha.create({ try {
width: 500, const captcha = ppfunCaptcha.create({
height: 300, width: 500,
fontSize: 180, height: 300,
stroke: 'black', fontSize: 180,
fill: 'none', stroke: 'black',
nodeDeviation: 2.5, fill: 'none',
connectionPathDeviation: 10.0, nodeDeviation: 2.5,
style: 'stroke-width: 4;', connectionPathDeviation: 10.0,
background: '#EFEFEF', style: 'stroke-width: 4;',
font, background: '#EFEFEF',
}); font,
});
const ip = getIPFromRequest(req); const ip = getIPFromRequest(req);
const captchaid = getRandomString(); const captchaid = getRandomString();
setCaptchaSolution(captcha.text, ip, captchaid); setCaptchaSolution(captcha.text, ip, captchaid);
console.log(`Serving ${captcha.text} to ${ip} / ${captchaid}`); console.log(`Serving ${captcha.text} to ${ip} / ${captchaid}`);
res.writeHead(200, { res.writeHead(200, {
'Content-Type': 'image/svg+xml', 'Content-Type': 'image/svg+xml',
'Cache-Control': 'no-cache', 'Cache-Control': 'no-cache',
'Captcha-Id': captchaid, 'Captcha-Id': captchaid,
}); });
res.write(captcha.data); res.write(captcha.data);
res.end(); res.end();
} catch (error) {
console.error(error);
res.writeHead(503, {
'Content-Type': 'text/html',
'Cache-Control': 'no-cache',
});
res.end(
// eslint-disable-next-line max-len
'<html><body><h1>Captchaserver: 503 Server Error</h1>Captchas are accessible via *.svp paths</body></html>',
);
}
} else { } else {
res.writeHead(404, { res.writeHead(404, {
'Content-Type': 'text/html', 'Content-Type': 'text/html',
@ -72,6 +85,24 @@ const server = http.createServer((req, res) => {
} }
}); });
server.listen(PORT, HOST, () => { // connect to redis
console.log(`Captcha Server listening on port ${PORT}`); connectRedis()
}); .then(() => {
// start http server
const startServer = () => {
server.listen(PORT, HOST, () => {
console.log(`Captcha Server listening on port ${PORT}`);
});
};
startServer();
// catch errors of server
server.on('error', (e) => {
console.error(
`Captcha Server Error ${e.code} occured, trying again in 5s...`,
);
setTimeout(() => {
server.close();
startServer();
}, 5000);
});
});

3
src/core/CanvasCleaner.js
View File

@ -162,7 +162,6 @@ class CanvasCleaner {
}; };
this.cleanChunk = this.cleanChunk.bind(this); this.cleanChunk = this.cleanChunk.bind(this);
this.clearValues(); this.clearValues();
this.loadArgs();
} }
clearValues() { clearValues() {
@ -184,7 +183,7 @@ class CanvasCleaner {
this.tick = null; this.tick = null;
} }
async loadArgs() { async initialize() {
const [cIter, running] = await getStatus(); const [cIter, running] = await getStatus();
if (running) { if (running) {
const [canvasId, x, y, u, v, methodName] = await getData(); const [canvasId, x, y, u, v, methodName] = await getData();

4
src/core/ChatProvider.js
View File

@ -531,7 +531,9 @@ export class ChatProvider {
const key = `mute:${id}`; const key = `mute:${id}`;
if (timeMin) { if (timeMin) {
const ttl = timeMin * 60; const ttl = timeMin * 60;
await redis.set(key, '', 'EX', ttl); await redis.set(key, '', {
EX: ttl,
});
if (printChannel) { if (printChannel) {
this.broadcastChatMessage( this.broadcastChatMessage(
'info', 'info',

21
src/core/PixelCache.js
View File

@ -1,13 +1,12 @@
/* /*
* Caching pixels for a few ms before sending them * Caching pixels for a few ms before sending them
* in bursts per chunk * in bursts per chunk
* @flow
*/ */
import socketEvents from '../socket/SocketEvents'; import socketEvents from '../socket/SocketEvents';
class PixelCache { class PixelCache {
PXL_CACHE: Map<number, Buffer>; PXL_CACHE;
constructor() { constructor() {
this.PXL_CACHE = new Map(); this.PXL_CACHE = new Map();
@ -22,12 +21,12 @@ class PixelCache {
* @param offset Offset of pixel within chunk * @param offset Offset of pixel within chunk
* @param color color index of pixel * @param color color index of pixel
*/ */
append( async append(
canvasId: number, canvasId,
color: number, color,
i: number, i,
j: number, j,
offset: number, offset,
) { ) {
const { PXL_CACHE } = this; const { PXL_CACHE } = this;
const chunkCanvasId = (canvasId << 16) | (i << 8) | j; const chunkCanvasId = (canvasId << 16) | (i << 8) | j;
@ -49,7 +48,7 @@ class PixelCache {
} }
} }
flushCache() { async flushCache() {
const { PXL_CACHE: cache } = this; const { PXL_CACHE: cache } = this;
this.PXL_CACHE = new Map(); this.PXL_CACHE = new Map();
@ -62,7 +61,7 @@ class PixelCache {
} }
const pixelCache = new PixelCache(); const pixelCache = new PixelCache();
// send pixels from cache to websockets every 20ms // send pixels from cache to websockets every 50ms
setInterval(pixelCache.flushCache, 20); setInterval(pixelCache.flushCache, 50);
export default pixelCache; export default pixelCache;

48
src/core/event.js → src/core/RpgEvent.js
View File

@ -15,12 +15,12 @@ import {
clearOldEvent, clearOldEvent,
CANVAS_ID, CANVAS_ID,
} from '../data/models/Event'; } from '../data/models/Event';
import { setCoolDownFactor } from './draw';
import Void from './Void'; import Void from './Void';
import { protectCanvasArea } from './Image'; import { protectCanvasArea } from './Image';
import { setPixelByOffset } from './setPixel'; import { setPixelByOffset } from './setPixel';
import { TILE_SIZE, EVENT_USER_NAME } from './constants'; import { TILE_SIZE, EVENT_USER_NAME } from './constants';
import chatProvider from './ChatProvider'; import chatProvider from './ChatProvider';
import { HOURLY_EVENT } from './config';
// eslint-disable-next-line import/no-unresolved // eslint-disable-next-line import/no-unresolved
import canvases from './canvases.json'; import canvases from './canvases.json';
@ -65,7 +65,7 @@ function drawCross(centerCell, clr, style, radius) {
} }
class Event { class RpgEvent {
eventState: number; eventState: number;
eventTimestamp: number; eventTimestamp: number;
eventCenter: Array; eventCenter: Array;
@ -79,19 +79,35 @@ class Event {
chatTimeout: number; chatTimeout: number;
constructor() { constructor() {
this.enabled = HOURLY_EVENT;
this.eventState = -1; this.eventState = -1;
this.eventCenterC = null; this.eventCenterC = null;
this.void = null; this.void = null;
this.chatTimeout = 0; this.chatTimeout = 0;
this.runEventLoop = this.runEventLoop.bind(this); this.runEventLoop = this.runEventLoop.bind(this);
if (HOURLY_EVENT) {
this.initEventLoop();
}
} }
async initEventLoop() { setSuccess(success, save = true) {
this.success = await getSuccess(); this.success = success;
let fac = 1;
switch (success) {
case 1:
fac /= 2;
break;
case 2:
fac *= 2;
break;
default:
// nothing
}
if (save) {
setSuccess(success);
}
setCoolDownFactor(fac);
}
async initialize() {
const success = await getSuccess();
this.setSuccess(success, false);
let eventTimestamp = await nextEvent(); let eventTimestamp = await nextEvent();
if (!eventTimestamp) { if (!eventTimestamp) {
eventTimestamp = await Event.setNextEvent(); eventTimestamp = await Event.setNextEvent();
@ -271,8 +287,7 @@ class Event {
Event.broadcastChatMessage( Event.broadcastChatMessage(
'Threat couldn\'t be contained, abandon area', 'Threat couldn\'t be contained, abandon area',
); );
this.success = 2; this.setSuccess(2);
setSuccess(2);
this.void = null; this.void = null;
const [x, y, w, h] = this.eventArea; const [x, y, w, h] = this.eventArea;
await protectCanvasArea(CANVAS_ID, x, y, w, h, true); await protectCanvasArea(CANVAS_ID, x, y, w, h, true);
@ -299,8 +314,7 @@ class Event {
Event.broadcastChatMessage( Event.broadcastChatMessage(
'Threat successfully defeated. Good work!', 'Threat successfully defeated. Good work!',
); );
this.success = 1; this.setSuccess(1);
setSuccess(1);
} }
this.void.cancel(); this.void.cancel();
this.void = null; this.void = null;
@ -318,8 +332,7 @@ class Event {
Event.broadcastChatMessage( Event.broadcastChatMessage(
'Void seems to leave again.', 'Void seems to leave again.',
); );
this.success = 0; this.setSuccess(0);
setSuccess(0);
} }
this.eventState = 13; this.eventState = 13;
} }
@ -335,8 +348,7 @@ class Event {
Event.broadcastChatMessage( Event.broadcastChatMessage(
'Celebration time over, get back to work.', 'Celebration time over, get back to work.',
); );
this.success = 0; this.setSuccess(0);
setSuccess(0);
} }
this.eventState = 14; this.eventState = 14;
} }
@ -356,6 +368,4 @@ class Event {
} }
} }
const rpgEvent = new Event(); export default RpgEvent;
export default rpgEvent;

19
src/core/Tile.js
View File

@ -11,6 +11,7 @@
import sharp from 'sharp'; import sharp from 'sharp';
import fs from 'fs'; import fs from 'fs';
import { commandOptions } from 'redis';
import Palette from './Palette'; import Palette from './Palette';
import { getMaxTiledZoom } from './utils'; import { getMaxTiledZoom } from './utils';
@ -144,6 +145,7 @@ export async function createZoomTileFromChunk(
for (let dy = 0; dy < TILE_ZOOM_LEVEL; dy += 1) { for (let dy = 0; dy < TILE_ZOOM_LEVEL; dy += 1) {
for (let dx = 0; dx < TILE_ZOOM_LEVEL; dx += 1) { for (let dx = 0; dx < TILE_ZOOM_LEVEL; dx += 1) {
chunk = await redisClient.get( chunk = await redisClient.get(
commandOptions({ returnBuffers: true }),
`ch:${canvasId}:${xabs + dx}:${yabs + dy}`, `ch:${canvasId}:${xabs + dx}:${yabs + dy}`,
); );
if (!chunk || chunk.length !== TILE_SIZE * TILE_SIZE) { if (!chunk || chunk.length !== TILE_SIZE * TILE_SIZE) {
@ -180,7 +182,7 @@ export async function createZoomTileFromChunk(
.toFile(filename); .toFile(filename);
} catch (error) { } catch (error) {
console.error( console.error(
`Tiling: Error on createZoomTileFromChunk:\n${error.message}`, `Tiling: Error on createZoomTileFromChunk: ${error.message}`,
); );
return false; return false;
} }
@ -226,7 +228,7 @@ export async function createZoomedTile(
} catch (error) { } catch (error) {
console.error( console.error(
// eslint-disable-next-line max-len // eslint-disable-next-line max-len
`Tiling: Error on createZoomedTile on chunk ${chunkfile}:\n${error.message}`, `Tiling: Error on createZoomedTile on chunk ${chunkfile}: ${error.message}`,
); );
} }
} }
@ -252,7 +254,7 @@ export async function createZoomedTile(
).resize(TILE_SIZE).toFile(filename); ).resize(TILE_SIZE).toFile(filename);
} catch (error) { } catch (error) {
console.error( console.error(
`Tiling: Error on createZoomedTile:\n${error.message}`, `Tiling: Error on createZoomedTile: ${error.message}`,
); );
return false; return false;
} }
@ -300,7 +302,7 @@ async function createEmptyTile(
.toFile(filename); .toFile(filename);
} catch (error) { } catch (error) {
console.error( console.error(
`Tiling: Error on createEmptyTile:\n${error.message}`, `Tiling: Error on createEmptyTile: ${error.message}`,
); );
return; return;
} }
@ -346,7 +348,7 @@ export async function createTexture(
} catch (error) { } catch (error) {
console.error( console.error(
// eslint-disable-next-line max-len // eslint-disable-next-line max-len
`Tiling: Error on createTexture in chunk ${chunkfile}:\n${error.message}`, `Tiling: Error on createTexture in chunk ${chunkfile}: ${error.message}`,
); );
} }
} }
@ -354,7 +356,10 @@ export async function createTexture(
} else { } else {
for (let dy = 0; dy < amount; dy += 1) { for (let dy = 0; dy < amount; dy += 1) {
for (let dx = 0; dx < amount; dx += 1) { for (let dx = 0; dx < amount; dx += 1) {
chunk = await redisClient.get(`ch:${canvasId}:${dx}:${dy}`); chunk = await redisClient.get(
commandOptions({ returnBuffers: true }),
`ch:${canvasId}:${dx}:${dy}`,
);
if (!chunk || chunk.length !== TILE_SIZE * TILE_SIZE) { if (!chunk || chunk.length !== TILE_SIZE * TILE_SIZE) {
na.push([dx, dy]); na.push([dx, dy]);
continue; continue;
@ -388,7 +393,7 @@ export async function createTexture(
).toFile(filename); ).toFile(filename);
} catch (error) { } catch (error) {
console.error( console.error(
`Tiling: Error on createTexture:\n${error.message}`, `Tiling: Error on createTexture: ${error.message}`,
); );
return; return;
} }

8
src/core/adminfunctions.js
View File

@ -52,7 +52,9 @@ export async function executeIPAction(action, ips, logger = null) {
await Blacklist.findOrCreate({ await Blacklist.findOrCreate({
where: { ip: ipKey }, where: { ip: ipKey },
}); });
await redis.set(key, 'y', 'EX', 24 * 3600); await redis.set(key, 'y', {
EX: 24 * 3600,
});
break; break;
case 'unban': case 'unban':
await Blacklist.destroy({ await Blacklist.destroy({
@ -64,7 +66,9 @@ export async function executeIPAction(action, ips, logger = null) {
await Whitelist.findOrCreate({ await Whitelist.findOrCreate({
where: { ip: ipKey }, where: { ip: ipKey },
}); });
await redis.set(key, 'n', 'EX', 24 * 3600); await redis.set(key, 'n', {
EX: 24 * 3600,
});
break; break;
case 'unwhitelist': case 'unwhitelist':
await Whitelist.destroy({ await Whitelist.destroy({

26
src/core/draw.js
View File

@ -12,12 +12,15 @@ import {
setPixelByCoords, setPixelByCoords,
} from './setPixel'; } from './setPixel';
import rankings from './ranking'; import rankings from './ranking';
import rpgEvent from './event';
// eslint-disable-next-line import/no-unresolved // eslint-disable-next-line import/no-unresolved
import canvases from './canvases.json'; import canvases from './canvases.json';
import { THREE_CANVAS_HEIGHT, THREE_TILE_SIZE, TILE_SIZE } from './constants'; import { THREE_CANVAS_HEIGHT, THREE_TILE_SIZE, TILE_SIZE } from './constants';
let coolDownFactor = 1;
export function setCoolDownFactor(fac) {
coolDownFactor = fac;
}
/** /**
* *
@ -93,17 +96,6 @@ export async function drawByOffsets(
} }
} }
let coolDownFactor = 1;
if (rpgEvent.success) {
if (rpgEvent.success === 1) {
// if hourly event got won
coolDownFactor = 0.5;
} else {
// if hourly event got lost
coolDownFactor = 2;
}
}
while (pixels.length) { while (pixels.length) {
const [offset, color] = pixels.pop(); const [offset, color] = pixels.pop();
@ -332,14 +324,8 @@ export async function drawByCoords(
? canvas.pcd : canvas.bcd; ? canvas.pcd : canvas.bcd;
if (isAdmin) { if (isAdmin) {
coolDown = 0.0; coolDown = 0.0;
} else if (rpgEvent.success) { } else {
if (rpgEvent.success === 1) { coolDown *= coolDownFactor;
// if HOURLY_EVENT got won
coolDown /= 2;
} else {
// if HOURLY_EVENT got lost
coolDown *= 2;
}
} }
const now = Date.now(); const now = Date.now();

18
src/core/isProxy.js
View File

@ -7,13 +7,10 @@ import fetch from '../utils/proxiedFetch';
import redis from '../data/redis'; import redis from '../data/redis';
import { getIPv6Subnet } from '../utils/ip'; import { getIPv6Subnet } from '../utils/ip';
import { Blacklist, Whitelist } from '../data/models'; import { Blacklist, Whitelist } from '../data/models';
import { proxyLogger } from './logger'; import { proxyLogger as logger } from './logger';
import { USE_PROXYCHECK } from './config'; import { USE_PROXYCHECK } from './config';
const logger = proxyLogger;
/* /*
* check getipintel if IP is proxy * check getipintel if IP is proxy
* Use proxiedFetch with random proxies and random mail for it, to not get blacklisted * Use proxiedFetch with random proxies and random mail for it, to not get blacklisted
@ -170,14 +167,11 @@ async function withCache(f, ip) {
const key = `isprox:${ipKey}`; const key = `isprox:${ipKey}`;
const cache = await redis.get(key); const cache = await redis.get(key);
if (cache) { if (cache) {
const str = cache.toString('utf8'); logger.debug('PROXYCHECK fetch isproxy from cache %s %s %s',
logger.debug('PROXYCHECK fetch isproxy from cache %s %s %s %s %s',
key, key,
cache, cache,
typeof cache, typeof cache);
str, return cache === 'y';
typeof str);
return str === 'y';
} }
logger.debug('PROXYCHECK fetch isproxy not from cache %s', key); logger.debug('PROXYCHECK fetch isproxy not from cache %s', key);
@ -190,7 +184,9 @@ async function withCache(f, ip) {
withoutCache(f, ip) withoutCache(f, ip)
.then((result) => { .then((result) => {
const value = result ? 'y' : 'n'; const value = result ? 'y' : 'n';
redis.set(key, value, 'EX', 3 * 24 * 3600); // cache for three days redis.set(key, value, {
EX: 3 * 24 * 3600,
}); // cache for three days
const pos = checking.indexOf(ipKey); const pos = checking.indexOf(ipKey);
if (~pos) checking.splice(pos, 1); if (~pos) checking.splice(pos, 1);
lock += 1; lock += 1;

8
src/core/ranking.js
View File

@ -23,14 +23,12 @@ class Ranks {
dailyRanking: [], dailyRanking: [],
ranking: [], ranking: [],
}; };
this.loadPrevDayTop();
setInterval(this.updateRanking, 5 * MINUTE);
DailyCron.hook(this.resetDailyRanking);
} }
async loadPrevDayTop() { async initialize() {
this.prevTop = await loadDailyTop(); this.prevTop = await loadDailyTop();
setInterval(this.updateRanking, 5 * MINUTE);
DailyCron.hook(this.resetDailyRanking);
} }
async updateRanking() { async updateRanking() {

2
src/core/setPixel.js
View File

@ -27,7 +27,7 @@ export function setPixelByOffset(
j, j,
offset, offset,
) { ) {
RedisCanvas.setPixelInChunk(i, j, offset, color, canvasId); RedisCanvas.enqueuePixel(canvasId, color, i, j, offset);
pixelCache.append(canvasId, color, i, j, offset); pixelCache.append(canvasId, color, i, j, offset);
} }

35
src/core/tilesBackup.js
View File

@ -1,15 +1,15 @@
/* /*
* Offer functions for Canvas backups * Offer functions for Canvas backups
* *
* @flow
*/ */
/* eslint-disable no-console */ /* eslint-disable no-console */
import sharp from 'sharp'; import sharp from 'sharp';
import fs from 'fs'; import fs from 'fs';
import Palette from './Palette'; import { commandOptions } from 'redis';
import Palette from './Palette';
import { TILE_SIZE } from './constants'; import { TILE_SIZE } from './constants';
@ -40,11 +40,13 @@ export async function updateBackupRedis(canvasRedis, backupRedis, canvases) {
* in exchange for higher execution time is wanted. * in exchange for higher execution time is wanted.
*/ */
// eslint-disable-next-line no-await-in-loop // eslint-disable-next-line no-await-in-loop
const chunk = await canvasRedis.get(key); const chunk = await canvasRedis.get(
commandOptions({ returnBuffers: true }),
key,
);
if (chunk) { if (chunk) {
const setNXArgs = [key, chunk];
// eslint-disable-next-line no-await-in-loop // eslint-disable-next-line no-await-in-loop
await backupRedis.sendCommand('SET', setNXArgs); await backupRedis.set(key, chunk);
amount += 1; amount += 1;
} }
} }
@ -65,7 +67,7 @@ export async function incrementialBackupRedis(
canvasRedis, canvasRedis,
backupRedis, backupRedis,
canvases, canvases,
backupDir: string, backupDir,
) { ) {
const ids = Object.keys(canvases); const ids = Object.keys(canvases);
for (let i = 0; i < ids.length; i += 1) { for (let i = 0; i < ids.length; i += 1) {
@ -108,12 +110,18 @@ export async function incrementialBackupRedis(
* in exchange for higher execution time is wanted. * in exchange for higher execution time is wanted.
*/ */
// eslint-disable-next-line no-await-in-loop // eslint-disable-next-line no-await-in-loop
const curChunk = await canvasRedis.get(key); const curChunk = await canvasRedis.get(
commandOptions({ returnBuffers: true }),
key,
);
let tileBuffer = null; let tileBuffer = null;
if (curChunk) { if (curChunk) {
if (curChunk.length === TILE_SIZE * TILE_SIZE) { if (curChunk.length === TILE_SIZE * TILE_SIZE) {
// eslint-disable-next-line no-await-in-loop // eslint-disable-next-line no-await-in-loop
const oldChunk = await backupRedis.get(key); const oldChunk = await backupRedis.get(
commandOptions({ returnBuffers: true }),
key,
);
if (oldChunk && oldChunk.length === TILE_SIZE * TILE_SIZE) { if (oldChunk && oldChunk.length === TILE_SIZE * TILE_SIZE) {
let pxl = 0; let pxl = 0;
while (pxl < curChunk.length) { while (pxl < curChunk.length) {
@ -171,9 +179,9 @@ export async function incrementialBackupRedis(
* @param backupDir directory where to save png tiles * @param backupDir directory where to save png tiles
*/ */
export async function createPngBackup( export async function createPngBackup(
redisClient: Object, redisClient,
canvases: Object, canvases,
backupDir: string, backupDir,
) { ) {
const ids = Object.keys(canvases); const ids = Object.keys(canvases);
for (let i = 0; i < ids.length; i += 1) { for (let i = 0; i < ids.length; i += 1) {
@ -207,7 +215,10 @@ export async function createPngBackup(
* in exchange for higher execution time is wanted. * in exchange for higher execution time is wanted.
*/ */
// eslint-disable-next-line no-await-in-loop // eslint-disable-next-line no-await-in-loop
const chunk = await redisClient.get(key); const chunk = await redisClient.get(
commandOptions({ returnBuffers: true }),
key,
);
if (chunk) { if (chunk) {
if (chunk.length === TILE_SIZE * TILE_SIZE) { if (chunk.length === TILE_SIZE * TILE_SIZE) {
const textureBuffer = palette.buffer2RGB(chunk); const textureBuffer = palette.buffer2RGB(chunk);

1
src/data/countrycode-coords-array.json
View File

File diff suppressed because one or more lines are too long

127
src/data/models/RedisCanvas.js
View File

@ -1,4 +1,7 @@
/* @flow */ /*
* canvas data on redis
*/
import { commandOptions } from 'redis';
import { getChunkOfPixel, getOffsetOfPixel } from '../../core/utils'; import { getChunkOfPixel, getOffsetOfPixel } from '../../core/utils';
import { import {
@ -9,7 +12,7 @@ import {
// eslint-disable-next-line import/no-unresolved // eslint-disable-next-line import/no-unresolved
import canvases from './canvases.json'; import canvases from './canvases.json';
import redis from '../redis'; import redis, { redisV3 } from '../redis';
const UINT_SIZE = 'u8'; const UINT_SIZE = 'u8';
@ -22,7 +25,7 @@ const THREE_EMPTY_CACA = new Uint8Array(
const THREE_EMPTY_CHUNK_BUFFER = Buffer.from(THREE_EMPTY_CACA.buffer); const THREE_EMPTY_CHUNK_BUFFER = Buffer.from(THREE_EMPTY_CACA.buffer);
// cache existence of chunks // cache existence of chunks
const chunks: Set<string> = new Set(); const chunks = new Set();
class RedisCanvas { class RedisCanvas {
@ -39,19 +42,21 @@ class RedisCanvas {
} }
static getChunk( static getChunk(
canvasId: number, canvasId,
i: number, i,
j: number, j,
): Promise<Buffer> { ) {
// this key is also hardcoded into // this key is also hardcoded into
// core/tilesBackup.js // core/tilesBackup.js
// and ./EventData.js // and ./EventData.js
const key = `ch:${canvasId}:${i}:${j}`; const key = `ch:${canvasId}:${i}:${j}`;
return redis.get(key); return redis.get(
commandOptions({ returnBuffers: true }),
key,
);
} }
static async setChunk(i: number, j: number, chunk: Uint8Array, static async setChunk(i, j, chunk, canvasId) {
canvasId: number) {
if (chunk.length !== TILE_SIZE * TILE_SIZE) { if (chunk.length !== TILE_SIZE * TILE_SIZE) {
// eslint-disable-next-line no-console // eslint-disable-next-line no-console
console.error(`Tried to set chunk with invalid length ${chunk.length}!`); console.error(`Tried to set chunk with invalid length ${chunk.length}!`);
@ -63,7 +68,7 @@ class RedisCanvas {
return true; return true;
} }
static async delChunk(i: number, j: number, canvasId: number) { static async delChunk(i, j, canvasId) {
const key = `ch:${canvasId}:${i}:${j}`; const key = `ch:${canvasId}:${i}:${j}`;
await redis.del(key); await redis.del(key);
chunks.delete(key); chunks.delete(key);
@ -72,11 +77,11 @@ class RedisCanvas {
} }
static async setPixel( static async setPixel(
canvasId: number, canvasId,
color: number, color,
x: number, x,
y: number, y,
z: number = null, z = null,
) { ) {
const canvasSize = canvases[canvasId].size; const canvasSize = canvases[canvasId].size;
const [i, j] = getChunkOfPixel(canvasSize, x, y, z); const [i, j] = getChunkOfPixel(canvasSize, x, y, z);
@ -84,66 +89,104 @@ class RedisCanvas {
RedisCanvas.setPixelInChunk(i, j, offset, color, canvasId); RedisCanvas.setPixelInChunk(i, j, offset, color, canvasId);
} }
multi = null;
static enqueuePixel(
canvasId,
color,
i,
j,
offset,
) {
if (!RedisCanvas.multi) {
RedisCanvas.multi = redisV3.multi();
}
RedisCanvas.multi.v4.addCommand(
[
'BITFIELD',
`ch:${canvasId}:${i}:${j}`,
'SET',
UINT_SIZE,
`#${offset}`,
color,
],
);
RedisCanvas.execChunkChangeCallback(canvasId, [i, j]);
}
static flushPixels() {
if (RedisCanvas.multi) {
const { multi } = RedisCanvas;
RedisCanvas.multi = null;
return multi.execAsPipeline();
}
return null;
}
static async setPixelInChunk( static async setPixelInChunk(
i: number, i,
j: number, j,
offset: number, offset,
color: number, color,
canvasId: number, canvasId,
) { ) {
const key = `ch:${canvasId}:${i}:${j}`; const key = `ch:${canvasId}:${i}:${j}`;
if (!chunks.has(key)) { if (!chunks.has(key)) {
if (canvases[canvasId].v) { if (canvases[canvasId].v) {
await redis.set(key, THREE_EMPTY_CHUNK_BUFFER, 'NX'); await redis.set(key, THREE_EMPTY_CHUNK_BUFFER, {
NX: true,
});
} else { } else {
await redis.set(key, EMPTY_CHUNK_BUFFER, 'NX'); await redis.set(key, EMPTY_CHUNK_BUFFER, {
NX: true,
});
} }
chunks.add(key); chunks.add(key);
} }
const args = [key, 'SET', UINT_SIZE, `#${offset}`, color]; const args = ['BITFIELD', key, 'SET', UINT_SIZE, `#${offset}`, color];
await redis.sendCommand('bitfield', args); await redis.sendCommand(args);
RedisCanvas.execChunkChangeCallback(canvasId, [i, j]); RedisCanvas.execChunkChangeCallback(canvasId, [i, j]);
} }
static async getPixelIfExists( static async getPixelIfExists(
canvasId: number, canvasId,
i: number, i,
j: number, j,
offset: number, offset,
): Promise<number> { ) {
// 1st bit -> protected or not // 1st bit -> protected or not
// 2nd bit -> unused // 2nd bit -> unused
// rest (6 bits) -> index of color // rest (6 bits) -> index of color
const args = [ const args = [
'BITFIELD',
`ch:${canvasId}:${i}:${j}`, `ch:${canvasId}:${i}:${j}`,
'GET', 'GET',
UINT_SIZE, UINT_SIZE,
`#${offset}`, `#${offset}`,
]; ];
const result: ?number = await redis.sendCommand('bitfield', args); const result = await redis.sendCommand(args);
if (!result) return null; if (!result) return null;
const color = result[0]; const color = result[0];
return color; return color;
} }
static async getPixelByOffset( static async getPixelByOffset(
canvasId: number, canvasId,
i: number, i,
j: number, j,
offset: number, offset,
): Promise<number> { ) {
const clr = RedisCanvas.getPixelIfExists(canvasId, i, j, offset); const clr = RedisCanvas.getPixelIfExists(canvasId, i, j, offset);
return (clr == null) ? 0 : clr; return (clr == null) ? 0 : clr;
} }
static async getPixel( static async getPixel(
canvasId: number, canvasId,
x: number, x,
y: number, y,
z: number = null, z = null,
): Promise<number> { ) {
const canvasSize = canvases[canvasId].size; const canvasSize = canvases[canvasId].size;
const [i, j] = getChunkOfPixel(canvasSize, x, y, z); const [i, j] = getChunkOfPixel(canvasSize, x, y, z);
const offset = getOffsetOfPixel(canvasSize, x, y, z); const offset = getOffsetOfPixel(canvasSize, x, y, z);
@ -153,4 +196,6 @@ class RedisCanvas {
} }
} }
setInterval(RedisCanvas.flushPixels, 100);
export default RedisCanvas; export default RedisCanvas;

8
src/data/models/User.js
View File

@ -139,9 +139,13 @@ class User {
async setWait(wait: number, canvasId: number): Promise<boolean> { async setWait(wait: number, canvasId: number): Promise<boolean> {
if (!wait) return false; if (!wait) return false;
// PX is milliseconds expire // PX is milliseconds expire
await redis.set(`cd:${canvasId}:ip:${this.ipSub}`, '', 'PX', wait); await redis.set(`cd:${canvasId}:ip:${this.ipSub}`, '', {
PX: wait,
});
if (this.id != null) { if (this.id != null) {
await redis.set(`cd:${canvasId}:id:${this.id}`, '', 'PX', wait); await redis.set(`cd:${canvasId}:id:${this.id}`, '', {
PX: wait,
});
} }
return true; return true;
} }

25
src/data/redis.js
View File

@ -4,12 +4,27 @@ import { createClient } from 'redis';
import { REDIS_URL } from '../core/config'; import { REDIS_URL } from '../core/config';
const redis = createClient({ const redis = createClient(REDIS_URL.startsWith('redis://')
path: REDIS_URL, ? {
// needed for connect-redis url: REDIS_URL,
legacyMode: true, // needed for connect-redis
}); legacyMode: true,
}
: {
socket: {
path: REDIS_URL,
},
// needed for connect-redis
legacyMode: true,
},
);
export const redisV3 = redis; export const redisV3 = redis;
export const connect = async () => {
// eslint-disable-next-line no-console
console.log(`Connecting to redis server at ${REDIS_URL}`);
await redis.connect();
};
export default redis.v4; export default redis.v4;

3
src/routes/chunks.js
View File

@ -64,7 +64,8 @@ export default async (req, res, next) => {
// eslint-disable-next-line max-len // eslint-disable-next-line max-len
logger.warn(`Long redis response times of ${dur}ms for chunk ${c}:${x},${y}`); logger.warn(`Long redis response times of ${dur}ms for chunk ${c}:${x},${y}`);
} }
} catch { } catch (error) {
logger.error(`Error on routes/chunks: ${error.message}`);
res.status(503).end(); res.status(503).end();
return; return;
} }

44
src/server.js
View File

@ -12,15 +12,17 @@ import forceGC from './core/forceGC';
import logger from './core/logger'; import logger from './core/logger';
import rankings from './core/ranking'; import rankings from './core/ranking';
import models from './data/models'; import models from './data/models';
import { redisV3 } from './data/redis'; import { connect as connectRedis } from './data/redis';
import routes from './routes'; import routes from './routes';
import chatProvider from './core/ChatProvider'; import chatProvider from './core/ChatProvider';
import RpgEvent from './core/RpgEvent';
import canvasCleaner from './core/CanvasCleaner';
import SocketServer from './socket/SocketServer'; import SocketServer from './socket/SocketServer';
import APISocketServer from './socket/APISocketServer'; import APISocketServer from './socket/APISocketServer';
import { PORT, HOST } from './core/config'; import { PORT, HOST, HOURLY_EVENT } from './core/config';
import { SECOND } from './core/constants'; import { SECOND } from './core/constants';
import { startAllCanvasLoops } from './core/tileserver'; import { startAllCanvasLoops } from './core/tileserver';
@ -74,22 +76,46 @@ app.use(compression({
app.use(routes); app.use(routes);
/*
/* Hourly Event
*/
const rpgEvent = new RpgEvent();
// //
// ip config // ip config
// ----------------------------------------------------------------------------- // -----------------------------------------------------------------------------
// sync sql models
models.sync({ alter: { drop: false } }) models.sync({ alter: { drop: false } })
.then(() => redisV3.connect()) // connect to redis
.then(() => connectRedis())
.then(() => { .then(() => {
rankings.updateRanking(); rankings.initialize();
chatProvider.initialize(); chatProvider.initialize();
startAllCanvasLoops(); startAllCanvasLoops();
usersocket.initialize(); usersocket.initialize();
apisocket.initialize(); apisocket.initialize();
server.listen(PORT, HOST, () => { if (HOURLY_EVENT) {
const address = server.address(); rpgEvent.initialize();
logger.log( }
'info', canvasCleaner.initialize();
`web is running at http://${address.host}:${address.port}/`, // start http server
const startServer = () => {
server.listen(PORT, HOST, () => {
logger.log(
'info',
`HTTP Server listening on port ${PORT}`,
);
});
};
startServer();
// catch errors of server
server.on('error', (e) => {
logger.error(
`HTTP Server Error ${e.code} occured, trying again in 5s...`,
); );
setTimeout(() => {
server.close();
startServer();
}, 5000);
}); });
}); });

4
src/socket/SocketServer.js
View File

@ -85,6 +85,10 @@ class SocketServer {
ws.canvasId = null; ws.canvasId = null;
ws.startDate = Date.now(); ws.startDate = Date.now();
const user = await authenticateClient(req); const user = await authenticateClient(req);
if (!user) {
ws.close();
return;
}
ws.user = user; ws.user = user;
ws.name = user.getName(); ws.name = user.getName();
cheapDetector(user.ip); cheapDetector(user.ip);

17
src/utils/captcha.js
View File

@ -73,7 +73,7 @@ function evaluateResult(captchaText, userText) {
* @param ip * @param ip
* @param captchaid * @param captchaid
*/ */
export function setCaptchaSolution( export async function setCaptchaSolution(
text, text,
ip, ip,
captchaid = null, captchaid = null,
@ -82,7 +82,14 @@ export function setCaptchaSolution(
if (captchaid) { if (captchaid) {
key += `:${captchaid}`; key += `:${captchaid}`;
} }
return redis.set(key, text, 'EX', CAPTCHA_TIMEOUT); try {
await redis.set(key, text, {
EX: CAPTCHA_TIMEOUT,
});
} catch (error) {
// eslint-disable-next-line no-console
console.error(error);
}
} }
/* /*
@ -109,10 +116,12 @@ export async function checkCaptchaSolution(
} }
const solution = await redis.get(key); const solution = await redis.get(key);
if (solution) { if (solution) {
if (evaluateResult(solution.toString('utf8'), text)) { if (evaluateResult(solution, text)) {
if (!onetime) { if (!onetime) {
const solvkey = `human:${ipn}`; const solvkey = `human:${ipn}`;
await redis.set(solvkey, '', 'EX', TTL_CACHE); await redis.set(solvkey, '', {
EX: TTL_CACHE,
});
} }
logger.info(`CAPTCHA ${ip} successfully solved captcha`); logger.info(`CAPTCHA ${ip} successfully solved captcha`);
return 0; return 0;

6
src/utils/ip.js
View File

@ -3,8 +3,6 @@
* basic functions to get data fromheaders and parse IPs * basic functions to get data fromheaders and parse IPs
*/ */
import logger from '../core/logger';
import { USE_XREALIP } from '../core/config'; import { USE_XREALIP } from '../core/config';
@ -35,7 +33,8 @@ export function getIPFromRequest(req) {
conip = conip || '0.0.0.1'; conip = conip || '0.0.0.1';
if (!USE_XREALIP) { if (!USE_XREALIP) {
logger.warn( // eslint-disable-next-line no-console
console.warn(
`Connection not going through reverse proxy! IP: ${conip}`, req.headers, `Connection not going through reverse proxy! IP: ${conip}`, req.headers,
); );
} }
@ -47,7 +46,6 @@ export function getIPv6Subnet(ip) {
if (ip.includes(':')) { if (ip.includes(':')) {
// eslint-disable-next-line max-len // eslint-disable-next-line max-len
const ipv6sub = `${ip.split(':').slice(0, 4).join(':')}:0000:0000:0000:0000`; const ipv6sub = `${ip.split(':').slice(0, 4).join(':')}:0000:0000:0000:0000`;
// logger.warn("IPv6 subnet: ", ipv6sub);
return ipv6sub; return ipv6sub;
} }
return ip; return ip;

20
src/utils/location.js
View File

@ -2,24 +2,6 @@
* provide location and lang specific features * provide location and lang specific features
*/ */
import ccCoords from '../data/countrycode-coords-array.json';
/*
* takes country name in two letter ISO style,
* return canvas coords based on pre-made json list
* @param cc Two letter country code
* @return coords X/Y coordinates of the country on the canvas
*/
export function ccToCoords(cc) {
if (!cc) {
return [0, 0];
}
const country = cc.toLowerCase();
const coords = ccCoords[country];
return (coords) || [0, 0];
}
/* /*
* gets prefered language out of localisation string * gets prefered language out of localisation string
* @param location string (like from accept-language header) * @param location string (like from accept-language header)
@ -64,5 +46,3 @@ const lang2CC = {
export function langCodeToCC(lang) { export function langCodeToCC(lang) {
return lang2CC[lang] || lang; return lang2CC[lang] || lang;
} }
export default ccToCoords;

58
src/workers/tilewriter.js
View File

@ -6,7 +6,7 @@
import { isMainThread, parentPort } from 'worker_threads'; import { isMainThread, parentPort } from 'worker_threads';
import redisClient from '../data/redis'; import redis, { connect as connectRedis } from '../data/redis';
import { import {
createZoomTileFromChunk, createZoomTileFromChunk,
createZoomedTile, createZoomedTile,
@ -20,29 +20,33 @@ if (isMainThread) {
); );
} }
parentPort.on('message', async (msg) => { connectRedis()
const { task, args } = msg; .then(() => {
try { parentPort.on('message', async (msg) => {
switch (task) { const { task, args } = msg;
case 'createZoomTileFromChunk': try {
createZoomTileFromChunk(redisClient, ...args); switch (task) {
break; case 'createZoomTileFromChunk':
case 'createZoomedTile': createZoomTileFromChunk(redis, ...args);
createZoomedTile(...args); break;
break; case 'createZoomedTile':
case 'createTexture': createZoomedTile(...args);
createTexture(redisClient, ...args); break;
break; case 'createTexture':
case 'initializeTiles': createTexture(redis, ...args);
await initializeTiles(redisClient, ...args); break;
parentPort.postMessage('Done!'); case 'initializeTiles':
break; await initializeTiles(redis, ...args);
default: parentPort.postMessage('Done!');
console.warn(`Tiling: Main thread requested unknown task ${task}`); break;
} default:
} catch (error) { console.warn(`Tiling: Main thread requested unknown task ${task}`);
console.warn( }
`Tiling: Error on executing task ${task} args ${args}:\n${error.message}`, } catch (error) {
); console.warn(
} // eslint-disable-next-line max-len
}); `Tiling: Error on executing task ${task} args ${args}: ${error.message}`,
);
}
});
});

3
utils/README.md
View File

@ -1,5 +1,8 @@
# Utils for map creation, conversion, 3d models and related stuff # Utils for map creation, conversion, 3d models and related stuff
Note: Note:
- EVERY SCRIPT THAT USES REDIS IS JUST AS REFERENCE (node-redis and keys update and change over time and i am not keeping those up-to-date)
- we use blender 2.8 - we use blender 2.8
- js script are executed with babel-node - js script are executed with babel-node

12
utils/redisConvert.js
View File

@ -1,4 +1,3 @@
/* @flow */
// this scripts converts the old 64x64 chunks that were organiced relative to the center to 256x256 chunks with 0.0 being top-left corner // this scripts converts the old 64x64 chunks that were organiced relative to the center to 256x256 chunks with 0.0 being top-left corner
// it also goes from 2 pixel per byte to 1 pixel per byte // it also goes from 2 pixel per byte to 1 pixel per byte
// old colors are converted to new order // old colors are converted to new order
@ -8,9 +7,12 @@ import { TILE_SIZE, CANVAS_SIZE, CANVAS_MIN_XY, CANVAS_MAX_XY } from '../src/cor
import redis from 'redis'; import redis from 'redis';
//ATTENTION Make suer to set the rdis URLs right!!! //ATTENTION Make suer to set the rdis URLs right!!!
const oldurl = "redis://localhost:6380"; const oldurl = "redis://localhost:6380";
const oldredis = redis.createClient(oldurl, { return_buffers: true }); const oldredis = redis.createClient({ url: oldurl, return_buffers: true });
const newurl = "redis://localhost:6379"; const newurl = "redis://localhost:6379";
const newredis = redis.createClient(newurl, { return_buffers: true }); const newredis = redis.createClient({ url: newurl, return_buffers: true });
oldredis.connect();
newredis.connect();
const CHUNK_SIZE = 64; //old chunk size const CHUNK_SIZE = 64; //old chunk size
const CHUNKS_IN_BASETILE = TILE_SIZE / CHUNK_SIZE; const CHUNKS_IN_BASETILE = TILE_SIZE / CHUNK_SIZE;
@ -161,4 +163,6 @@ async function convert() {
} }
} }
convert(); oldredis.connect()
.then(() => newredis.connect())
.then(() => convert());

13
utils/redisCopy.js
View File

@ -10,9 +10,12 @@ import {
//ATTENTION Make suer to set the rdis URLs right!!! //ATTENTION Make suer to set the rdis URLs right!!!
const oldurl = "redis://localhost:6380"; const oldurl = "redis://localhost:6380";
const oldredis = redis.createClient(oldurl, { return_buffers: true }); const oldredis = redis.createClient({ url: oldurl });
const newurl = "redis://localhost:6379"; const newurl = "redis://localhost:6379";
const newredis = redis.createClient(newurl, { return_buffers: true }); const newredis = redis.createClient({ url: newurl });
oldredis.connect();
newredis.connect();
const CANVAS_SIZE = 1024; const CANVAS_SIZE = 1024;
const OUR_TILE_SIZE = THREE_TILE_SIZE; const OUR_TILE_SIZE = THREE_TILE_SIZE;
@ -25,8 +28,7 @@ async function copyChunks() {
const newkey = `ch:2:${x}:${y}`; const newkey = `ch:2:${x}:${y}`;
const chunk = await oldredis.get(oldkey); const chunk = await oldredis.get(oldkey);
if (chunk) { if (chunk) {
const setNXArgs = [newkey, chunk]; await newredis.set(newkey, chunk);
await newredis.sendCommand('SET', setNXArgs);
console.log("Created Chunk ", newkey); console.log("Created Chunk ", newkey);
} }
} }
@ -48,8 +50,7 @@ async function copyChunksByCoords(xMin, xMax, yMin, yMax) {
const newkey = `ch:2:${x}:${y}`; const newkey = `ch:2:${x}:${y}`;
const chunk = await oldredis.get(oldkey); const chunk = await oldredis.get(oldkey);
if (chunk) { if (chunk) {
const setNXArgs = [newkey, chunk]; await newredis.set(newkey, chunk);
await newredis.sendCommand('SET', setNXArgs);
console.log("Created Chunk ", newkey); console.log("Created Chunk ", newkey);
} else { } else {
await newredis.del(newkey); await newredis.del(newkey);

6
utils/redisMoveCanvas.js
View File

@ -1,11 +1,10 @@
/* @flow */
// this script moves chunks of a canvas, i.e. to center it after changing size // this script moves chunks of a canvas, i.e. to center it after changing size
import redis from 'redis'; import redis from 'redis';
//ATTENTION Make suer to set the rdis URLs right!!! //ATTENTION Make suer to set the rdis URLs right!!!
const url = "redis://localhost:6379"; const url = "redis://localhost:6379";
const redisc = redis.createClient(url, { return_buffers: true }); const redisc = redis.createClient({ url, return_buffers: true });
const CANVAS_SIZE = 4096; const CANVAS_SIZE = 4096;
const TILE_SIZE = 256; const TILE_SIZE = 256;
@ -30,4 +29,5 @@ async function move() {
console.log("done"); console.log("done");
} }
move(); redisc.connect()
.then(() => move());