From 100bdb17b51af8b999385b9c485f2b1499df782c Mon Sep 17 00:00:00 2001 From: HF Date: Wed, 7 Sep 2022 02:37:48 +0200 Subject: [PATCH] use lua scripting in redis for setting pixels --- src/core/draw.js | 123 +++++++++++++----------------- src/core/isAllowed.js | 15 ++-- src/data/redis/allowPlace.js | 43 +++++++++++ src/data/redis/captcha.js | 8 +- src/data/redis/client.js | 16 +++- src/data/redis/isAllowedCache.js | 2 +- src/data/redis/lua/placePixel.lua | 110 ++++++++++++++++++++++++++ src/socket/SocketServer.js | 49 ++++-------- src/socket/packets/PixelReturn.js | 8 +- webpack.config.server.js | 4 + 10 files changed, 255 insertions(+), 123 deletions(-) create mode 100644 src/data/redis/allowPlace.js create mode 100644 src/data/redis/lua/placePixel.lua diff --git a/src/core/draw.js b/src/core/draw.js index 53c0c34..e70bd9c 100644 --- a/src/core/draw.js +++ b/src/core/draw.js @@ -7,6 +7,7 @@ import { } from './utils'; import logger, { pixelLogger } from './logger'; import RedisCanvas from '../data/redis/RedisCanvas'; +import allowPlace from '../data/redis/allowPlace'; import { setPixelByOffset, setPixelByCoords, @@ -68,6 +69,7 @@ export async function drawByOffsets( let retCode = 0; let pxlCnt = 0; let rankedPxlCnt = 0; + let needProxycheck = 0; const { ip } = user; try { @@ -91,8 +93,6 @@ export async function drawByOffsets( const canvasSize = canvas.size; const is3d = !!canvas.v; - wait = await user.getWait(canvasId); - const tileSize = (is3d) ? THREE_TILE_SIZE : TILE_SIZE; /* * canvas/chunk validation @@ -134,38 +134,34 @@ export async function drawByOffsets( } } - while (pixels.length) { - const [offset, color] = pixels.pop(); + const clrIgnore = canvas.cli || 0; + const factor = (isAdmin || (user.userlvl > 0 && pixels[0][1] < clrIgnore)) + ? 0.0 : coolDownFactor; + const bcd = canvas.bcd * factor; + const pcd = (canvas.pcd) ? canvas.pcd * factor : bcd; + const pxlOffsets = []; + /* + * validate pixels + */ + for (let u = 0; u < pixels.length; u += 1) { + const [offset, color] = pixels[u]; + pxlOffsets.push(offset); const [x, y, z] = getPixelFromChunkOffset(i, j, offset, canvasSize, is3d); - - // eslint-disable-next-line no-await-in-loop - const setColor = await RedisCanvas.getPixelByOffset( - canvasId, - i, j, - offset, - ); - pixelLogger.info( // eslint-disable-next-line max-len - `${startTime} ${user.ip} ${user.id} ${canvasId} ${x} ${y} ${z} ${color} ${setColor}`, + `${startTime} ${user.ip} ${user.id} ${canvasId} ${x} ${y} ${z} ${color}`, ); - const clrIgnore = canvas.cli || 0; - - /* - * pixel validation - */ const maxSize = (is3d) ? tileSize * tileSize * THREE_CANVAS_HEIGHT : tileSize * tileSize; if (offset >= maxSize) { // z out of bounds or weird stuff throw new Error(4); } - /* - * admins and mods can place unset pixels - */ + + // admins and mods can place unset pixels if (color >= canvas.colors.length || (color < clrIgnore && user.userlvl === 0 @@ -175,59 +171,46 @@ export async function drawByOffsets( throw new Error(5); } - if (setColor & 0x80 - /* 3D Canvas Minecraft Avatars */ - // && x >= 96 && x <= 128 && z >= 35 && z <= 100 - // 96 - 128 on x - // 32 - 128 on z - || (canvas.v && i === 19 && j >= 17 && j < 20 && !isAdmin) - ) { + /* 3D Canvas Minecraft Avatars */ + // && x >= 96 && x <= 128 && z >= 35 && z <= 100 + // 96 - 128 on x + // 32 - 128 on z + if (canvas.v && i === 19 && j >= 17 && j < 20 && !isAdmin) { // protected pixel throw new Error(8); } - coolDown = ((setColor & 0x3F) >= clrIgnore && canvas.pcd) - ? canvas.pcd : canvas.bcd; - /* - * admins have no cooldown - * mods have no cooldown when placing unset pixels - */ - if (isAdmin || (user.userlvl > 0 && color < clrIgnore)) { - coolDown = 0.0; - } else { - /* - * cooldown changes like from event - */ - coolDown *= coolDownFactor; - } - - wait += coolDown; - if (wait > canvas.cds) { - // cooldown stack used - wait -= coolDown; - coolDown = canvas.cds - wait - coolDown; - throw new Error(9); - } - - setPixelByOffset(canvasId, color, i, j, offset); - - pxlCnt += 1; - /* - * hardcode to not count pixels in antarctica - * do not count 0cd pixels - */ // eslint-disable-next-line eqeqeq - if (canvas.ranked && (canvasId != 0 || y < 14450) && coolDown) { + if (canvas.ranked && (canvasId != 0 || y < 14450) && pcd) { + pixels[u].push(true); + } + } + + [retCode, pxlCnt, wait, coolDown, needProxycheck] = await allowPlace( + ip, + user.id, + canvasId, + i, j, + clrIgnore, + bcd, pcd, + canvas.cds, + pxlOffsets, + ); + + for (let u = 0; u < pxlCnt; u += 1) { + const [offset, color, ranked] = pixels[u]; + setPixelByOffset(canvasId, color, i, j, offset); + if (ranked) { rankedPxlCnt += 1; } + } - const duration = Date.now() - startTime; - if (duration > 1000) { - logger.warn( - // eslint-disable-next-line max-len - `Long response time of ${duration}ms for placing ${pxlCnt} pixels for user ${user.id || user.ip}`, - ); - } + const duration = Date.now() - startTime; + if (duration > 1000) { + logger.warn( + // eslint-disable-next-line max-len + `Long response time of ${duration}ms for placing ${pxlCnt} pixels for user ${user.id || user.ip}`, + ); } } catch (e) { retCode = parseInt(e.message, 10); @@ -236,11 +219,8 @@ export async function drawByOffsets( } } - if (pxlCnt && wait) { - await user.setWait(wait, canvasId); - if (rankedPxlCnt) { - await user.incrementPixelcount(rankedPxlCnt); - } + if (rankedPxlCnt) { + await user.incrementPixelcount(rankedPxlCnt); } if (retCode !== 13) { @@ -253,6 +233,7 @@ export async function drawByOffsets( pxlCnt, rankedPxlCnt, retCode, + needProxycheck, }; } diff --git a/src/core/isAllowed.js b/src/core/isAllowed.js index 89e259a..29171a7 100644 --- a/src/core/isAllowed.js +++ b/src/core/isAllowed.js @@ -92,16 +92,15 @@ async function withCache(f, ip) { status: 4, }; } - // get from cache, if there const ipKey = getIPv6Subnet(ip); - const cache = await getCacheAllowed(ipKey); - if (cache) { - return cache; - } - - // else make asynchronous ipcheck and assume no proxy in the meantime - // do not check ip that currently gets checked if (checking.indexOf(ipKey) === -1) { + // get from cache, if there + const cache = await getCacheAllowed(ipKey); + if (cache) { + return cache; + } + // else make asynchronous ipcheck and assume no proxy in the meantime + // do not check ip that currently gets checked checking.push(ipKey); withoutCache(f, ip) .catch((error) => { diff --git a/src/data/redis/allowPlace.js b/src/data/redis/allowPlace.js new file mode 100644 index 0000000..f1426bb --- /dev/null +++ b/src/data/redis/allowPlace.js @@ -0,0 +1,43 @@ +/* + * redis script for user pixel placement + * this does not set any pixels itself, see lua/placePixel.lua + */ +import client from './client'; +import { getIPv6Subnet } from '../../utils/ip'; +import { PREFIX as CAPTCHA_PREFIX } from './captcha'; +import { PREFIX as ALLOWED_PREFIX } from './isAllowedCache'; +import { CAPTCHA_TIME } from '../../core/config'; + +/* + * gets pixels and chunk coords and checks if + * and how many a user can set and sets the cooldown accordingly + * @param ip ip of request + * @param id userId + * @param clrIgnore, bcd, pcd, cds incormations about canvas + * @param i, j chunk coordinates + * @param pxls Array with offsets of pixels + * @return see lua/placePixel.lua + */ +export default function allowPlace( + ip, + id, + canvasId, + i, j, + clrIgnore, + bcd, + pcd, + cds, + pxls, +) { + const ipn = getIPv6Subnet(ip); + const isalKey = `${ALLOWED_PREFIX}:${ipn}`; + const captKey = (CAPTCHA_TIME >= 0) ? `${CAPTCHA_PREFIX}:${ipn}` : 'nope'; + const ipCdKey = `cd:${canvasId}:ip:${ipn}`; + const idCdKey = (id) ? `cd:${canvasId}:id:${id}` : 'nope'; + const chunkKey = `ch:${canvasId}:${i}:${j}`; + return client.placePxl( + isalKey, captKey, ipCdKey, idCdKey, chunkKey, + clrIgnore, bcd, pcd, cds, + ...pxls, + ); +} diff --git a/src/data/redis/captcha.js b/src/data/redis/captcha.js index 410cf5c..4ae295e 100644 --- a/src/data/redis/captcha.js +++ b/src/data/redis/captcha.js @@ -13,6 +13,8 @@ import { const TTL_CACHE = CAPTCHA_TIME * 60; // seconds +export const PREFIX = 'human'; + /* * chars that are so similar that we allow them to get mixed up * left: captcha text @@ -119,7 +121,7 @@ export async function checkCaptchaSolution( return 2; } if (!onetime) { - const solvkey = `human:${ipn}`; + const solvkey = `${PREFIX}:${ipn}`; await client.set(solvkey, '', { EX: TTL_CACHE, }); @@ -148,7 +150,7 @@ export async function needCaptcha(ip) { if (CAPTCHA_TIME < 0) { return false; } - const key = `human:${getIPv6Subnet(ip)}`; + const key = `${PREFIX}:${getIPv6Subnet(ip)}`; const ttl = await client.ttl(key); if (ttl > 0) { return false; @@ -167,7 +169,7 @@ export async function forceCaptcha(ip) { if (CAPTCHA_TIME < 0) { return null; } - const key = `human:${getIPv6Subnet(ip)}`; + const key = `${PREFIX}:${getIPv6Subnet(ip)}`; const ret = await client.del(key); return (ret > 0); } diff --git a/src/data/redis/client.js b/src/data/redis/client.js index 13a7452..b8ee829 100644 --- a/src/data/redis/client.js +++ b/src/data/redis/client.js @@ -2,19 +2,31 @@ * redis client * REDIS_URL can be url or path to unix socket */ - -import { createClient } from 'redis'; +import fs from 'fs'; +import { createClient, defineScript } from 'redis'; import { REDIS_URL } from '../../core/config'; +const scripts = { + placePxl: defineScript({ + NUMBER_OF_KEYS: 5, + SCRIPT: fs.readFileSync('./workers/placePixel.lua'), + transformArguments(...args) { + return args.map((a) => ((typeof a === 'string') ? a : a.toString())); + }, + }), +}; + const client = createClient(REDIS_URL.startsWith('redis://') ? { url: REDIS_URL, + scripts, } : { socket: { path: REDIS_URL, }, + scripts, }, ); diff --git a/src/data/redis/isAllowedCache.js b/src/data/redis/isAllowedCache.js index e27edd9..7383364 100644 --- a/src/data/redis/isAllowedCache.js +++ b/src/data/redis/isAllowedCache.js @@ -5,7 +5,7 @@ import client from './client'; -const PREFIX = 'isal'; +export const PREFIX = 'isal'; const CACHE_DURATION = 14 * 24 * 3600; export function cacheAllowed(ip, status) { diff --git a/src/data/redis/lua/placePixel.lua b/src/data/redis/lua/placePixel.lua new file mode 100644 index 0000000..f8d3381 --- /dev/null +++ b/src/data/redis/lua/placePixel.lua @@ -0,0 +1,110 @@ +-- Checking requirements and calculating cooldown of user wthin +-- redis itself. Does not set pixels directly. Pixels are set in batches +-- in RedisCanvas.js +-- This script will get copied into the dist/workers directory from webpack +-- Keys: +-- isAlloweed: 'isal:ip' (proxycheck, blacklist, whitelist) +-- isHuman 'human:ip' (captcha needed when expired) +-- ipCD: 'cd:canvasId:ip:ip' +-- uCD: 'cd:canvasId:id:userId' +-- chunk: 'ch:canvasId:i:j' +-- Args: +-- clrIgnore: integer number of what colors are considered unset +-- bcd: number baseColldown (fixed to cdFactor and 0 if admin) +-- pcd: number set pixel cooldown (fixed to cdFactor and 0 if admin) +-- cds: max cooldown of canvas +-- off1, chonk offset of first pixel +-- off2, chonk offset of second pixel +-- ..., infinie pixels possible +-- Returns: +-- { +-- 1: pixel return status code (check ui/placePixel.js) +-- 2: amount of successfully set pixels +-- 3: total cooldown of user +-- 4: info about placed pixel cooldown (addition of last pixel) +-- 5: if we have to update isAllowed( proxycheck) +-- } +local ret = {0, 0, 0, 0, 0} +-- check if isAllowed +local ia = redis.call('get', KEYS[1]) +if not ia then + ret[5] = 1 +else + ia = tonumber(ia) + if ia > 0 then + if ia == 1 then + -- proxy + ret[1] = 11 + elseif ia == 2 then + -- banned + ret[1] = 14 + elseif ia == 3 then + -- range banned + ret[1] = 15 + end + return ret + end +end +-- check if captcha is needed +if KEYS[2] ~= "nope" and not redis.call('get', KEYS[2]) then + -- captcha + ret[1] = 10 + return ret +end +-- get cooldown of user +local cd = redis.call('pttl', KEYS[3]) +if cd < 0 then + cd = 0 +end +if KEYS[4] ~= "nope" then + local icd = redis.call('pttl', KEYS[4]) + if icd > cd then + cd = icd + end +end +-- set pixels +local pxlcd = 0 +local pxlcnt = 0 +local cli = tonumber(ARGV[1]) +local bcd = tonumber(ARGV[2]) +local pcd = tonumber(ARGV[3]) +local cds = tonumber(ARGV[4]) +for c = 5,#ARGV do + local off = tonumber(ARGV[c]) * 8 + local clr = tonumber(ARGV[c + 1]) + -- get color of pixel on canvas + local sclr = redis.call('bitfield', KEYS[5], 'get', 'u8', off) + sclr = sclr[1] + -- check if protected (protected is last bit in u8) + if sclr >= 128 then + -- pixel protected + ret[1] = 8 + break + end + -- calculate cooldown of pixel + pxlcd = bcd + if sclr >= cli then + pxlcd = pcd + end + cd = cd + pxlcd + if cd > cds then + cd = cd - pxlcd + pxlcd = cds - cd - pxlcd + -- pixelstack used up + ret[1] = 9 + break + end + pxlcnt = pxlcnt + 1 +end + +if pxlcnt > 0 and cd > 0 then + redis.call('set', KEYS[3], '', 'px', cd) + if KEYS[4] ~= "nope" then + redis.call('set', KEYS[4], '', 'px', cd) + end +end + +ret[2] = pxlcnt +ret[3] = cd +ret[4] = pxlcd +return ret diff --git a/src/socket/SocketServer.js b/src/socket/SocketServer.js index 8a3c73a..bb4b5e1 100644 --- a/src/socket/SocketServer.js +++ b/src/socket/SocketServer.js @@ -23,7 +23,6 @@ import socketEvents from './SocketEvents'; import chatProvider, { ChatProvider } from '../core/ChatProvider'; import authenticateClient from './authenticateClient'; import { drawByOffsets } from '../core/draw'; -import { needCaptcha } from '../data/redis/captcha'; import isIPAllowed from '../core/isAllowed'; @@ -495,39 +494,6 @@ class SocketServer { return; } - let failureRet = null; - // check if captcha needed - if (await needCaptcha(ip)) { - // need captcha - failureRet = PixelReturn.dehydrate(10); - } else { - // (re)check for Proxy - const allowed = await isIPAllowed(ip); - if (!allowed.allowed) { - // proxy - let failureStatus = 11; - if (allowed.status === 2) { - // banned - failureStatus = 14; - } else if (allowed.status === 3) { - // range banned - failureStatus = 15; - } - failureRet = PixelReturn.dehydrate(failureStatus); - } - } - if (failureRet !== null) { - const now = Date.now(); - if (limiter && limiter[0] > now) { - limiter[0] += 1000; - } else { - rateLimit.set(ip, [now + 1000, false]); - } - ws.send(failureRet); - break; - } - - // receive pixels here const { i, j, pixels, } = PixelUpdate.hydrate(buffer); @@ -537,12 +503,27 @@ class SocketServer { pxlCnt, rankedPxlCnt, retCode, + needProxycheck, } = await drawByOffsets( ws.user, canvasId, i, j, pixels, ); + + if (needProxycheck) { + isIPAllowed(ip); + } + + if (retCode > 9 && retCode !== 13) { + const now = Date.now(); + if (limiter && limiter[0] > now) { + limiter[0] += 1000; + } else { + rateLimit.set(ip, [now + 1000, false]); + } + } + ws.send(PixelReturn.dehydrate( retCode, wait, diff --git a/src/socket/packets/PixelReturn.js b/src/socket/packets/PixelReturn.js index 8ba9285..9306f44 100644 --- a/src/socket/packets/PixelReturn.js +++ b/src/socket/packets/PixelReturn.js @@ -19,10 +19,10 @@ export default { }, dehydrate( retCode, - wait = 0, - coolDown = 0, - pxlCnt = 0, - rankedPxlCnt = 0, + wait, + coolDown, + pxlCnt, + rankedPxlCnt, ) { // Server (sender) const buffer = Buffer.allocUnsafe(1 + 1 + 4 + 2 + 1 + 1); diff --git a/webpack.config.server.js b/webpack.config.server.js index 16efe72..3a14ebf 100644 --- a/webpack.config.server.js +++ b/webpack.config.server.js @@ -159,6 +159,10 @@ module.exports = ({ from: path.resolve('deployment', 'captchaFonts'), to: path.resolve('dist', 'captchaFonts'), }, + { + from: path.resolve('src', 'data', 'redis', 'lua', 'placePixel.lua'), + to: path.resolve('dist', 'workers', 'placePixel.lua'), + }, ], }), ],