Set pixels in redis in batches

finish node-redis update by making sure that everything that needs redis
runs after redis connected
This commit is contained in:
HF 2022-04-06 21:50:34 +02:00
parent 56bc4352b2
commit e95b6ae8d3
28 changed files with 405 additions and 250 deletions

View File

@ -15,7 +15,7 @@ import fs from 'fs';
import os from 'os';
import { spawn } from 'child_process';
import path from 'path';
import redis from 'redis';
import { createClient } from 'redis';
import {
@ -53,10 +53,29 @@ if (!CANVAS_REDIS_URL || !BACKUP_REDIS_URL || !BACKUP_DIR) {
process.exit(1);
}
const canvasRedis = redis
.createClient(CANVAS_REDIS_URL);
const backupRedis = redis
.createClient(BACKUP_REDIS_URL);
const canvasRedis = createClient(CANVAS_REDIS_URL
.startsWith('redis://')
? {
url: CANVAS_REDIS_URL,
}
: {
socket: {
path: CANVAS_REDIS_URL,
},
},
);
const backupRedis = createClient(BACKUP_REDIS_URL
.startsWith('redis://')
? {
url: BACKUP_REDIS_URL,
}
: {
socket: {
path: BACKUP_REDIS_URL,
},
},
);
//
canvasRedis.on('error', () => {
console.error('Could not connect to canvas redis');
process.exit(1);
@ -161,4 +180,6 @@ async function trigger() {
}
console.log('Starting backup...');
trigger();
canvasRedis.connect()
.then(() => backupRedis.connect())
.then(() => trigger());

View File

@ -11,6 +11,7 @@ import http from 'http';
import url from 'url';
import ppfunCaptcha from 'ppfun-captcha';
import { connect as connectRedis } from './data/redis';
import { getIPFromRequest } from './utils/ip';
import { setCaptchaSolution } from './utils/captcha';
import { getRandomString } from './core/utils';
@ -34,32 +35,44 @@ const server = http.createServer((req, res) => {
const urlObject = url.parse(req.url, true);
if (req.method === 'GET' && urlObject.pathname.endsWith('.svg')) {
const captcha = ppfunCaptcha.create({
width: 500,
height: 300,
fontSize: 180,
stroke: 'black',
fill: 'none',
nodeDeviation: 2.5,
connectionPathDeviation: 10.0,
style: 'stroke-width: 4;',
background: '#EFEFEF',
font,
});
try {
const captcha = ppfunCaptcha.create({
width: 500,
height: 300,
fontSize: 180,
stroke: 'black',
fill: 'none',
nodeDeviation: 2.5,
connectionPathDeviation: 10.0,
style: 'stroke-width: 4;',
background: '#EFEFEF',
font,
});
const ip = getIPFromRequest(req);
const captchaid = getRandomString();
const ip = getIPFromRequest(req);
const captchaid = getRandomString();
setCaptchaSolution(captcha.text, ip, captchaid);
console.log(`Serving ${captcha.text} to ${ip} / ${captchaid}`);
setCaptchaSolution(captcha.text, ip, captchaid);
console.log(`Serving ${captcha.text} to ${ip} / ${captchaid}`);
res.writeHead(200, {
'Content-Type': 'image/svg+xml',
'Cache-Control': 'no-cache',
'Captcha-Id': captchaid,
});
res.write(captcha.data);
res.end();
res.writeHead(200, {
'Content-Type': 'image/svg+xml',
'Cache-Control': 'no-cache',
'Captcha-Id': captchaid,
});
res.write(captcha.data);
res.end();
} catch (error) {
console.error(error);
res.writeHead(503, {
'Content-Type': 'text/html',
'Cache-Control': 'no-cache',
});
res.end(
// eslint-disable-next-line max-len
'<html><body><h1>Captchaserver: 503 Server Error</h1>Captchas are accessible via *.svp paths</body></html>',
);
}
} else {
res.writeHead(404, {
'Content-Type': 'text/html',
@ -72,6 +85,24 @@ const server = http.createServer((req, res) => {
}
});
server.listen(PORT, HOST, () => {
console.log(`Captcha Server listening on port ${PORT}`);
});
// connect to redis
connectRedis()
.then(() => {
// start http server
const startServer = () => {
server.listen(PORT, HOST, () => {
console.log(`Captcha Server listening on port ${PORT}`);
});
};
startServer();
// catch errors of server
server.on('error', (e) => {
console.error(
`Captcha Server Error ${e.code} occured, trying again in 5s...`,
);
setTimeout(() => {
server.close();
startServer();
}, 5000);
});
});

View File

@ -162,7 +162,6 @@ class CanvasCleaner {
};
this.cleanChunk = this.cleanChunk.bind(this);
this.clearValues();
this.loadArgs();
}
clearValues() {
@ -184,7 +183,7 @@ class CanvasCleaner {
this.tick = null;
}
async loadArgs() {
async initialize() {
const [cIter, running] = await getStatus();
if (running) {
const [canvasId, x, y, u, v, methodName] = await getData();

View File

@ -531,7 +531,9 @@ export class ChatProvider {
const key = `mute:${id}`;
if (timeMin) {
const ttl = timeMin * 60;
await redis.set(key, '', 'EX', ttl);
await redis.set(key, '', {
EX: ttl,
});
if (printChannel) {
this.broadcastChatMessage(
'info',

View File

@ -1,13 +1,12 @@
/*
* Caching pixels for a few ms before sending them
* in bursts per chunk
* @flow
*/
import socketEvents from '../socket/SocketEvents';
class PixelCache {
PXL_CACHE: Map<number, Buffer>;
PXL_CACHE;
constructor() {
this.PXL_CACHE = new Map();
@ -22,12 +21,12 @@ class PixelCache {
* @param offset Offset of pixel within chunk
* @param color color index of pixel
*/
append(
canvasId: number,
color: number,
i: number,
j: number,
offset: number,
async append(
canvasId,
color,
i,
j,
offset,
) {
const { PXL_CACHE } = this;
const chunkCanvasId = (canvasId << 16) | (i << 8) | j;
@ -49,7 +48,7 @@ class PixelCache {
}
}
flushCache() {
async flushCache() {
const { PXL_CACHE: cache } = this;
this.PXL_CACHE = new Map();
@ -62,7 +61,7 @@ class PixelCache {
}
const pixelCache = new PixelCache();
// send pixels from cache to websockets every 20ms
setInterval(pixelCache.flushCache, 20);
// send pixels from cache to websockets every 50ms
setInterval(pixelCache.flushCache, 50);
export default pixelCache;

View File

@ -15,12 +15,12 @@ import {
clearOldEvent,
CANVAS_ID,
} from '../data/models/Event';
import { setCoolDownFactor } from './draw';
import Void from './Void';
import { protectCanvasArea } from './Image';
import { setPixelByOffset } from './setPixel';
import { TILE_SIZE, EVENT_USER_NAME } from './constants';
import chatProvider from './ChatProvider';
import { HOURLY_EVENT } from './config';
// eslint-disable-next-line import/no-unresolved
import canvases from './canvases.json';
@ -65,7 +65,7 @@ function drawCross(centerCell, clr, style, radius) {
}
class Event {
class RpgEvent {
eventState: number;
eventTimestamp: number;
eventCenter: Array;
@ -79,19 +79,35 @@ class Event {
chatTimeout: number;
constructor() {
this.enabled = HOURLY_EVENT;
this.eventState = -1;
this.eventCenterC = null;
this.void = null;
this.chatTimeout = 0;
this.runEventLoop = this.runEventLoop.bind(this);
if (HOURLY_EVENT) {
this.initEventLoop();
}
}
async initEventLoop() {
this.success = await getSuccess();
setSuccess(success, save = true) {
this.success = success;
let fac = 1;
switch (success) {
case 1:
fac /= 2;
break;
case 2:
fac *= 2;
break;
default:
// nothing
}
if (save) {
setSuccess(success);
}
setCoolDownFactor(fac);
}
async initialize() {
const success = await getSuccess();
this.setSuccess(success, false);
let eventTimestamp = await nextEvent();
if (!eventTimestamp) {
eventTimestamp = await Event.setNextEvent();
@ -271,8 +287,7 @@ class Event {
Event.broadcastChatMessage(
'Threat couldn\'t be contained, abandon area',
);
this.success = 2;
setSuccess(2);
this.setSuccess(2);
this.void = null;
const [x, y, w, h] = this.eventArea;
await protectCanvasArea(CANVAS_ID, x, y, w, h, true);
@ -299,8 +314,7 @@ class Event {
Event.broadcastChatMessage(
'Threat successfully defeated. Good work!',
);
this.success = 1;
setSuccess(1);
this.setSuccess(1);
}
this.void.cancel();
this.void = null;
@ -318,8 +332,7 @@ class Event {
Event.broadcastChatMessage(
'Void seems to leave again.',
);
this.success = 0;
setSuccess(0);
this.setSuccess(0);
}
this.eventState = 13;
}
@ -335,8 +348,7 @@ class Event {
Event.broadcastChatMessage(
'Celebration time over, get back to work.',
);
this.success = 0;
setSuccess(0);
this.setSuccess(0);
}
this.eventState = 14;
}
@ -356,6 +368,4 @@ class Event {
}
}
const rpgEvent = new Event();
export default rpgEvent;
export default RpgEvent;

View File

@ -11,6 +11,7 @@
import sharp from 'sharp';
import fs from 'fs';
import { commandOptions } from 'redis';
import Palette from './Palette';
import { getMaxTiledZoom } from './utils';
@ -144,6 +145,7 @@ export async function createZoomTileFromChunk(
for (let dy = 0; dy < TILE_ZOOM_LEVEL; dy += 1) {
for (let dx = 0; dx < TILE_ZOOM_LEVEL; dx += 1) {
chunk = await redisClient.get(
commandOptions({ returnBuffers: true }),
`ch:${canvasId}:${xabs + dx}:${yabs + dy}`,
);
if (!chunk || chunk.length !== TILE_SIZE * TILE_SIZE) {
@ -180,7 +182,7 @@ export async function createZoomTileFromChunk(
.toFile(filename);
} catch (error) {
console.error(
`Tiling: Error on createZoomTileFromChunk:\n${error.message}`,
`Tiling: Error on createZoomTileFromChunk: ${error.message}`,
);
return false;
}
@ -226,7 +228,7 @@ export async function createZoomedTile(
} catch (error) {
console.error(
// eslint-disable-next-line max-len
`Tiling: Error on createZoomedTile on chunk ${chunkfile}:\n${error.message}`,
`Tiling: Error on createZoomedTile on chunk ${chunkfile}: ${error.message}`,
);
}
}
@ -252,7 +254,7 @@ export async function createZoomedTile(
).resize(TILE_SIZE).toFile(filename);
} catch (error) {
console.error(
`Tiling: Error on createZoomedTile:\n${error.message}`,
`Tiling: Error on createZoomedTile: ${error.message}`,
);
return false;
}
@ -300,7 +302,7 @@ async function createEmptyTile(
.toFile(filename);
} catch (error) {
console.error(
`Tiling: Error on createEmptyTile:\n${error.message}`,
`Tiling: Error on createEmptyTile: ${error.message}`,
);
return;
}
@ -346,7 +348,7 @@ export async function createTexture(
} catch (error) {
console.error(
// eslint-disable-next-line max-len
`Tiling: Error on createTexture in chunk ${chunkfile}:\n${error.message}`,
`Tiling: Error on createTexture in chunk ${chunkfile}: ${error.message}`,
);
}
}
@ -354,7 +356,10 @@ export async function createTexture(
} else {
for (let dy = 0; dy < amount; dy += 1) {
for (let dx = 0; dx < amount; dx += 1) {
chunk = await redisClient.get(`ch:${canvasId}:${dx}:${dy}`);
chunk = await redisClient.get(
commandOptions({ returnBuffers: true }),
`ch:${canvasId}:${dx}:${dy}`,
);
if (!chunk || chunk.length !== TILE_SIZE * TILE_SIZE) {
na.push([dx, dy]);
continue;
@ -388,7 +393,7 @@ export async function createTexture(
).toFile(filename);
} catch (error) {
console.error(
`Tiling: Error on createTexture:\n${error.message}`,
`Tiling: Error on createTexture: ${error.message}`,
);
return;
}

View File

@ -52,7 +52,9 @@ export async function executeIPAction(action, ips, logger = null) {
await Blacklist.findOrCreate({
where: { ip: ipKey },
});
await redis.set(key, 'y', 'EX', 24 * 3600);
await redis.set(key, 'y', {
EX: 24 * 3600,
});
break;
case 'unban':
await Blacklist.destroy({
@ -64,7 +66,9 @@ export async function executeIPAction(action, ips, logger = null) {
await Whitelist.findOrCreate({
where: { ip: ipKey },
});
await redis.set(key, 'n', 'EX', 24 * 3600);
await redis.set(key, 'n', {
EX: 24 * 3600,
});
break;
case 'unwhitelist':
await Whitelist.destroy({

View File

@ -12,12 +12,15 @@ import {
setPixelByCoords,
} from './setPixel';
import rankings from './ranking';
import rpgEvent from './event';
// eslint-disable-next-line import/no-unresolved
import canvases from './canvases.json';
import { THREE_CANVAS_HEIGHT, THREE_TILE_SIZE, TILE_SIZE } from './constants';
let coolDownFactor = 1;
export function setCoolDownFactor(fac) {
coolDownFactor = fac;
}
/**
*
@ -93,17 +96,6 @@ export async function drawByOffsets(
}
}
let coolDownFactor = 1;
if (rpgEvent.success) {
if (rpgEvent.success === 1) {
// if hourly event got won
coolDownFactor = 0.5;
} else {
// if hourly event got lost
coolDownFactor = 2;
}
}
while (pixels.length) {
const [offset, color] = pixels.pop();
@ -332,14 +324,8 @@ export async function drawByCoords(
? canvas.pcd : canvas.bcd;
if (isAdmin) {
coolDown = 0.0;
} else if (rpgEvent.success) {
if (rpgEvent.success === 1) {
// if HOURLY_EVENT got won
coolDown /= 2;
} else {
// if HOURLY_EVENT got lost
coolDown *= 2;
}
} else {
coolDown *= coolDownFactor;
}
const now = Date.now();

View File

@ -7,13 +7,10 @@ import fetch from '../utils/proxiedFetch';
import redis from '../data/redis';
import { getIPv6Subnet } from '../utils/ip';
import { Blacklist, Whitelist } from '../data/models';
import { proxyLogger } from './logger';
import { proxyLogger as logger } from './logger';
import { USE_PROXYCHECK } from './config';
const logger = proxyLogger;
/*
* check getipintel if IP is proxy
* Use proxiedFetch with random proxies and random mail for it, to not get blacklisted
@ -170,14 +167,11 @@ async function withCache(f, ip) {
const key = `isprox:${ipKey}`;
const cache = await redis.get(key);
if (cache) {
const str = cache.toString('utf8');
logger.debug('PROXYCHECK fetch isproxy from cache %s %s %s %s %s',
logger.debug('PROXYCHECK fetch isproxy from cache %s %s %s',
key,
cache,
typeof cache,
str,
typeof str);
return str === 'y';
typeof cache);
return cache === 'y';
}
logger.debug('PROXYCHECK fetch isproxy not from cache %s', key);
@ -190,7 +184,9 @@ async function withCache(f, ip) {
withoutCache(f, ip)
.then((result) => {
const value = result ? 'y' : 'n';
redis.set(key, value, 'EX', 3 * 24 * 3600); // cache for three days
redis.set(key, value, {
EX: 3 * 24 * 3600,
}); // cache for three days
const pos = checking.indexOf(ipKey);
if (~pos) checking.splice(pos, 1);
lock += 1;

View File

@ -23,14 +23,12 @@ class Ranks {
dailyRanking: [],
ranking: [],
};
this.loadPrevDayTop();
setInterval(this.updateRanking, 5 * MINUTE);
DailyCron.hook(this.resetDailyRanking);
}
async loadPrevDayTop() {
async initialize() {
this.prevTop = await loadDailyTop();
setInterval(this.updateRanking, 5 * MINUTE);
DailyCron.hook(this.resetDailyRanking);
}
async updateRanking() {

View File

@ -27,7 +27,7 @@ export function setPixelByOffset(
j,
offset,
) {
RedisCanvas.setPixelInChunk(i, j, offset, color, canvasId);
RedisCanvas.enqueuePixel(canvasId, color, i, j, offset);
pixelCache.append(canvasId, color, i, j, offset);
}

View File

@ -1,15 +1,15 @@
/*
* Offer functions for Canvas backups
*
* @flow
*/
/* eslint-disable no-console */
import sharp from 'sharp';
import fs from 'fs';
import Palette from './Palette';
import { commandOptions } from 'redis';
import Palette from './Palette';
import { TILE_SIZE } from './constants';
@ -40,11 +40,13 @@ export async function updateBackupRedis(canvasRedis, backupRedis, canvases) {
* in exchange for higher execution time is wanted.
*/
// eslint-disable-next-line no-await-in-loop
const chunk = await canvasRedis.get(key);
const chunk = await canvasRedis.get(
commandOptions({ returnBuffers: true }),
key,
);
if (chunk) {
const setNXArgs = [key, chunk];
// eslint-disable-next-line no-await-in-loop
await backupRedis.sendCommand('SET', setNXArgs);
await backupRedis.set(key, chunk);
amount += 1;
}
}
@ -65,7 +67,7 @@ export async function incrementialBackupRedis(
canvasRedis,
backupRedis,
canvases,
backupDir: string,
backupDir,
) {
const ids = Object.keys(canvases);
for (let i = 0; i < ids.length; i += 1) {
@ -108,12 +110,18 @@ export async function incrementialBackupRedis(
* in exchange for higher execution time is wanted.
*/
// eslint-disable-next-line no-await-in-loop
const curChunk = await canvasRedis.get(key);
const curChunk = await canvasRedis.get(
commandOptions({ returnBuffers: true }),
key,
);
let tileBuffer = null;
if (curChunk) {
if (curChunk.length === TILE_SIZE * TILE_SIZE) {
// eslint-disable-next-line no-await-in-loop
const oldChunk = await backupRedis.get(key);
const oldChunk = await backupRedis.get(
commandOptions({ returnBuffers: true }),
key,
);
if (oldChunk && oldChunk.length === TILE_SIZE * TILE_SIZE) {
let pxl = 0;
while (pxl < curChunk.length) {
@ -171,9 +179,9 @@ export async function incrementialBackupRedis(
* @param backupDir directory where to save png tiles
*/
export async function createPngBackup(
redisClient: Object,
canvases: Object,
backupDir: string,
redisClient,
canvases,
backupDir,
) {
const ids = Object.keys(canvases);
for (let i = 0; i < ids.length; i += 1) {
@ -207,7 +215,10 @@ export async function createPngBackup(
* in exchange for higher execution time is wanted.
*/
// eslint-disable-next-line no-await-in-loop
const chunk = await redisClient.get(key);
const chunk = await redisClient.get(
commandOptions({ returnBuffers: true }),
key,
);
if (chunk) {
if (chunk.length === TILE_SIZE * TILE_SIZE) {
const textureBuffer = palette.buffer2RGB(chunk);

File diff suppressed because one or more lines are too long

View File

@ -1,4 +1,7 @@
/* @flow */
/*
* canvas data on redis
*/
import { commandOptions } from 'redis';
import { getChunkOfPixel, getOffsetOfPixel } from '../../core/utils';
import {
@ -9,7 +12,7 @@ import {
// eslint-disable-next-line import/no-unresolved
import canvases from './canvases.json';
import redis from '../redis';
import redis, { redisV3 } from '../redis';
const UINT_SIZE = 'u8';
@ -22,7 +25,7 @@ const THREE_EMPTY_CACA = new Uint8Array(
const THREE_EMPTY_CHUNK_BUFFER = Buffer.from(THREE_EMPTY_CACA.buffer);
// cache existence of chunks
const chunks: Set<string> = new Set();
const chunks = new Set();
class RedisCanvas {
@ -39,19 +42,21 @@ class RedisCanvas {
}
static getChunk(
canvasId: number,
i: number,
j: number,
): Promise<Buffer> {
canvasId,
i,
j,
) {
// this key is also hardcoded into
// core/tilesBackup.js
// and ./EventData.js
const key = `ch:${canvasId}:${i}:${j}`;
return redis.get(key);
return redis.get(
commandOptions({ returnBuffers: true }),
key,
);
}
static async setChunk(i: number, j: number, chunk: Uint8Array,
canvasId: number) {
static async setChunk(i, j, chunk, canvasId) {
if (chunk.length !== TILE_SIZE * TILE_SIZE) {
// eslint-disable-next-line no-console
console.error(`Tried to set chunk with invalid length ${chunk.length}!`);
@ -63,7 +68,7 @@ class RedisCanvas {
return true;
}
static async delChunk(i: number, j: number, canvasId: number) {
static async delChunk(i, j, canvasId) {
const key = `ch:${canvasId}:${i}:${j}`;
await redis.del(key);
chunks.delete(key);
@ -72,11 +77,11 @@ class RedisCanvas {
}
static async setPixel(
canvasId: number,
color: number,
x: number,
y: number,
z: number = null,
canvasId,
color,
x,
y,
z = null,
) {
const canvasSize = canvases[canvasId].size;
const [i, j] = getChunkOfPixel(canvasSize, x, y, z);
@ -84,66 +89,104 @@ class RedisCanvas {
RedisCanvas.setPixelInChunk(i, j, offset, color, canvasId);
}
multi = null;
static enqueuePixel(
canvasId,
color,
i,
j,
offset,
) {
if (!RedisCanvas.multi) {
RedisCanvas.multi = redisV3.multi();
}
RedisCanvas.multi.v4.addCommand(
[
'BITFIELD',
`ch:${canvasId}:${i}:${j}`,
'SET',
UINT_SIZE,
`#${offset}`,
color,
],
);
RedisCanvas.execChunkChangeCallback(canvasId, [i, j]);
}
static flushPixels() {
if (RedisCanvas.multi) {
const { multi } = RedisCanvas;
RedisCanvas.multi = null;
return multi.execAsPipeline();
}
return null;
}
static async setPixelInChunk(
i: number,
j: number,
offset: number,
color: number,
canvasId: number,
i,
j,
offset,
color,
canvasId,
) {
const key = `ch:${canvasId}:${i}:${j}`;
if (!chunks.has(key)) {
if (canvases[canvasId].v) {
await redis.set(key, THREE_EMPTY_CHUNK_BUFFER, 'NX');
await redis.set(key, THREE_EMPTY_CHUNK_BUFFER, {
NX: true,
});
} else {
await redis.set(key, EMPTY_CHUNK_BUFFER, 'NX');
await redis.set(key, EMPTY_CHUNK_BUFFER, {
NX: true,
});
}
chunks.add(key);
}
const args = [key, 'SET', UINT_SIZE, `#${offset}`, color];
await redis.sendCommand('bitfield', args);
const args = ['BITFIELD', key, 'SET', UINT_SIZE, `#${offset}`, color];
await redis.sendCommand(args);
RedisCanvas.execChunkChangeCallback(canvasId, [i, j]);
}
static async getPixelIfExists(
canvasId: number,
i: number,
j: number,
offset: number,
): Promise<number> {
canvasId,
i,
j,
offset,
) {
// 1st bit -> protected or not
// 2nd bit -> unused
// rest (6 bits) -> index of color
const args = [
'BITFIELD',
`ch:${canvasId}:${i}:${j}`,
'GET',
UINT_SIZE,
`#${offset}`,
];
const result: ?number = await redis.sendCommand('bitfield', args);
const result = await redis.sendCommand(args);
if (!result) return null;
const color = result[0];
return color;
}
static async getPixelByOffset(
canvasId: number,
i: number,
j: number,
offset: number,
): Promise<number> {
canvasId,
i,
j,
offset,
) {
const clr = RedisCanvas.getPixelIfExists(canvasId, i, j, offset);
return (clr == null) ? 0 : clr;
}
static async getPixel(
canvasId: number,
x: number,
y: number,
z: number = null,
): Promise<number> {
canvasId,
x,
y,
z = null,
) {
const canvasSize = canvases[canvasId].size;
const [i, j] = getChunkOfPixel(canvasSize, x, y, z);
const offset = getOffsetOfPixel(canvasSize, x, y, z);
@ -153,4 +196,6 @@ class RedisCanvas {
}
}
setInterval(RedisCanvas.flushPixels, 100);
export default RedisCanvas;

View File

@ -139,9 +139,13 @@ class User {
async setWait(wait: number, canvasId: number): Promise<boolean> {
if (!wait) return false;
// PX is milliseconds expire
await redis.set(`cd:${canvasId}:ip:${this.ipSub}`, '', 'PX', wait);
await redis.set(`cd:${canvasId}:ip:${this.ipSub}`, '', {
PX: wait,
});
if (this.id != null) {
await redis.set(`cd:${canvasId}:id:${this.id}`, '', 'PX', wait);
await redis.set(`cd:${canvasId}:id:${this.id}`, '', {
PX: wait,
});
}
return true;
}

View File

@ -4,12 +4,27 @@ import { createClient } from 'redis';
import { REDIS_URL } from '../core/config';
const redis = createClient({
path: REDIS_URL,
// needed for connect-redis
legacyMode: true,
});
const redis = createClient(REDIS_URL.startsWith('redis://')
? {
url: REDIS_URL,
// needed for connect-redis
legacyMode: true,
}
: {
socket: {
path: REDIS_URL,
},
// needed for connect-redis
legacyMode: true,
},
);
export const redisV3 = redis;
export const connect = async () => {
// eslint-disable-next-line no-console
console.log(`Connecting to redis server at ${REDIS_URL}`);
await redis.connect();
};
export default redis.v4;

View File

@ -64,7 +64,8 @@ export default async (req, res, next) => {
// eslint-disable-next-line max-len
logger.warn(`Long redis response times of ${dur}ms for chunk ${c}:${x},${y}`);
}
} catch {
} catch (error) {
logger.error(`Error on routes/chunks: ${error.message}`);
res.status(503).end();
return;
}

View File

@ -12,15 +12,17 @@ import forceGC from './core/forceGC';
import logger from './core/logger';
import rankings from './core/ranking';
import models from './data/models';
import { redisV3 } from './data/redis';
import { connect as connectRedis } from './data/redis';
import routes from './routes';
import chatProvider from './core/ChatProvider';
import RpgEvent from './core/RpgEvent';
import canvasCleaner from './core/CanvasCleaner';
import SocketServer from './socket/SocketServer';
import APISocketServer from './socket/APISocketServer';
import { PORT, HOST } from './core/config';
import { PORT, HOST, HOURLY_EVENT } from './core/config';
import { SECOND } from './core/constants';
import { startAllCanvasLoops } from './core/tileserver';
@ -74,22 +76,46 @@ app.use(compression({
app.use(routes);
/*
/* Hourly Event
*/
const rpgEvent = new RpgEvent();
//
// ip config
// -----------------------------------------------------------------------------
// sync sql models
models.sync({ alter: { drop: false } })
.then(() => redisV3.connect())
// connect to redis
.then(() => connectRedis())
.then(() => {
rankings.updateRanking();
rankings.initialize();
chatProvider.initialize();
startAllCanvasLoops();
usersocket.initialize();
apisocket.initialize();
server.listen(PORT, HOST, () => {
const address = server.address();
logger.log(
'info',
`web is running at http://${address.host}:${address.port}/`,
if (HOURLY_EVENT) {
rpgEvent.initialize();
}
canvasCleaner.initialize();
// start http server
const startServer = () => {
server.listen(PORT, HOST, () => {
logger.log(
'info',
`HTTP Server listening on port ${PORT}`,
);
});
};
startServer();
// catch errors of server
server.on('error', (e) => {
logger.error(
`HTTP Server Error ${e.code} occured, trying again in 5s...`,
);
setTimeout(() => {
server.close();
startServer();
}, 5000);
});
});

View File

@ -85,6 +85,10 @@ class SocketServer {
ws.canvasId = null;
ws.startDate = Date.now();
const user = await authenticateClient(req);
if (!user) {
ws.close();
return;
}
ws.user = user;
ws.name = user.getName();
cheapDetector(user.ip);

View File

@ -73,7 +73,7 @@ function evaluateResult(captchaText, userText) {
* @param ip
* @param captchaid
*/
export function setCaptchaSolution(
export async function setCaptchaSolution(
text,
ip,
captchaid = null,
@ -82,7 +82,14 @@ export function setCaptchaSolution(
if (captchaid) {
key += `:${captchaid}`;
}
return redis.set(key, text, 'EX', CAPTCHA_TIMEOUT);
try {
await redis.set(key, text, {
EX: CAPTCHA_TIMEOUT,
});
} catch (error) {
// eslint-disable-next-line no-console
console.error(error);
}
}
/*
@ -109,10 +116,12 @@ export async function checkCaptchaSolution(
}
const solution = await redis.get(key);
if (solution) {
if (evaluateResult(solution.toString('utf8'), text)) {
if (evaluateResult(solution, text)) {
if (!onetime) {
const solvkey = `human:${ipn}`;
await redis.set(solvkey, '', 'EX', TTL_CACHE);
await redis.set(solvkey, '', {
EX: TTL_CACHE,
});
}
logger.info(`CAPTCHA ${ip} successfully solved captcha`);
return 0;

View File

@ -3,8 +3,6 @@
* basic functions to get data fromheaders and parse IPs
*/
import logger from '../core/logger';
import { USE_XREALIP } from '../core/config';
@ -35,7 +33,8 @@ export function getIPFromRequest(req) {
conip = conip || '0.0.0.1';
if (!USE_XREALIP) {
logger.warn(
// eslint-disable-next-line no-console
console.warn(
`Connection not going through reverse proxy! IP: ${conip}`, req.headers,
);
}
@ -47,7 +46,6 @@ export function getIPv6Subnet(ip) {
if (ip.includes(':')) {
// eslint-disable-next-line max-len
const ipv6sub = `${ip.split(':').slice(0, 4).join(':')}:0000:0000:0000:0000`;
// logger.warn("IPv6 subnet: ", ipv6sub);
return ipv6sub;
}
return ip;

View File

@ -2,24 +2,6 @@
* provide location and lang specific features
*/
import ccCoords from '../data/countrycode-coords-array.json';
/*
* takes country name in two letter ISO style,
* return canvas coords based on pre-made json list
* @param cc Two letter country code
* @return coords X/Y coordinates of the country on the canvas
*/
export function ccToCoords(cc) {
if (!cc) {
return [0, 0];
}
const country = cc.toLowerCase();
const coords = ccCoords[country];
return (coords) || [0, 0];
}
/*
* gets prefered language out of localisation string
* @param location string (like from accept-language header)
@ -64,5 +46,3 @@ const lang2CC = {
export function langCodeToCC(lang) {
return lang2CC[lang] || lang;
}
export default ccToCoords;

View File

@ -6,7 +6,7 @@
import { isMainThread, parentPort } from 'worker_threads';
import redisClient from '../data/redis';
import redis, { connect as connectRedis } from '../data/redis';
import {
createZoomTileFromChunk,
createZoomedTile,
@ -20,29 +20,33 @@ if (isMainThread) {
);
}
parentPort.on('message', async (msg) => {
const { task, args } = msg;
try {
switch (task) {
case 'createZoomTileFromChunk':
createZoomTileFromChunk(redisClient, ...args);
break;
case 'createZoomedTile':
createZoomedTile(...args);
break;
case 'createTexture':
createTexture(redisClient, ...args);
break;
case 'initializeTiles':
await initializeTiles(redisClient, ...args);
parentPort.postMessage('Done!');
break;
default:
console.warn(`Tiling: Main thread requested unknown task ${task}`);
}
} catch (error) {
console.warn(
`Tiling: Error on executing task ${task} args ${args}:\n${error.message}`,
);
}
});
connectRedis()
.then(() => {
parentPort.on('message', async (msg) => {
const { task, args } = msg;
try {
switch (task) {
case 'createZoomTileFromChunk':
createZoomTileFromChunk(redis, ...args);
break;
case 'createZoomedTile':
createZoomedTile(...args);
break;
case 'createTexture':
createTexture(redis, ...args);
break;
case 'initializeTiles':
await initializeTiles(redis, ...args);
parentPort.postMessage('Done!');
break;
default:
console.warn(`Tiling: Main thread requested unknown task ${task}`);
}
} catch (error) {
console.warn(
// eslint-disable-next-line max-len
`Tiling: Error on executing task ${task} args ${args}: ${error.message}`,
);
}
});
});

View File

@ -1,5 +1,8 @@
# Utils for map creation, conversion, 3d models and related stuff
Note:
- EVERY SCRIPT THAT USES REDIS IS JUST AS REFERENCE (node-redis and keys update and change over time and i am not keeping those up-to-date)
- we use blender 2.8
- js script are executed with babel-node

View File

@ -1,4 +1,3 @@
/* @flow */
// this scripts converts the old 64x64 chunks that were organiced relative to the center to 256x256 chunks with 0.0 being top-left corner
// it also goes from 2 pixel per byte to 1 pixel per byte
// old colors are converted to new order
@ -8,9 +7,12 @@ import { TILE_SIZE, CANVAS_SIZE, CANVAS_MIN_XY, CANVAS_MAX_XY } from '../src/cor
import redis from 'redis';
//ATTENTION Make suer to set the rdis URLs right!!!
const oldurl = "redis://localhost:6380";
const oldredis = redis.createClient(oldurl, { return_buffers: true });
const oldredis = redis.createClient({ url: oldurl, return_buffers: true });
const newurl = "redis://localhost:6379";
const newredis = redis.createClient(newurl, { return_buffers: true });
const newredis = redis.createClient({ url: newurl, return_buffers: true });
oldredis.connect();
newredis.connect();
const CHUNK_SIZE = 64; //old chunk size
const CHUNKS_IN_BASETILE = TILE_SIZE / CHUNK_SIZE;
@ -161,4 +163,6 @@ async function convert() {
}
}
convert();
oldredis.connect()
.then(() => newredis.connect())
.then(() => convert());

View File

@ -10,9 +10,12 @@ import {
//ATTENTION Make suer to set the rdis URLs right!!!
const oldurl = "redis://localhost:6380";
const oldredis = redis.createClient(oldurl, { return_buffers: true });
const oldredis = redis.createClient({ url: oldurl });
const newurl = "redis://localhost:6379";
const newredis = redis.createClient(newurl, { return_buffers: true });
const newredis = redis.createClient({ url: newurl });
oldredis.connect();
newredis.connect();
const CANVAS_SIZE = 1024;
const OUR_TILE_SIZE = THREE_TILE_SIZE;
@ -25,8 +28,7 @@ async function copyChunks() {
const newkey = `ch:2:${x}:${y}`;
const chunk = await oldredis.get(oldkey);
if (chunk) {
const setNXArgs = [newkey, chunk];
await newredis.sendCommand('SET', setNXArgs);
await newredis.set(newkey, chunk);
console.log("Created Chunk ", newkey);
}
}
@ -48,8 +50,7 @@ async function copyChunksByCoords(xMin, xMax, yMin, yMax) {
const newkey = `ch:2:${x}:${y}`;
const chunk = await oldredis.get(oldkey);
if (chunk) {
const setNXArgs = [newkey, chunk];
await newredis.sendCommand('SET', setNXArgs);
await newredis.set(newkey, chunk);
console.log("Created Chunk ", newkey);
} else {
await newredis.del(newkey);

View File

@ -1,11 +1,10 @@
/* @flow */
// this script moves chunks of a canvas, i.e. to center it after changing size
import redis from 'redis';
//ATTENTION Make suer to set the rdis URLs right!!!
const url = "redis://localhost:6379";
const redisc = redis.createClient(url, { return_buffers: true });
const redisc = redis.createClient({ url, return_buffers: true });
const CANVAS_SIZE = 4096;
const TILE_SIZE = 256;
@ -30,4 +29,5 @@ async function move() {
console.log("done");
}
move();
redisc.connect()
.then(() => move());