From 725e23dbabff42cd17eb91d80699f1633d98f6da Mon Sep 17 00:00:00 2001 From: HF Date: Mon, 20 Jun 2022 22:54:05 +0200 Subject: [PATCH] move captchas into their own thread closes #3 --- README.md | 2 +- deployment/example-ecosystem-captchas.yml | 9 -- deployment/githook.sh | 4 - src/captchaserver.js | 108 ---------------------- src/components/Captcha.jsx | 2 +- src/core/captchaserver.js | 67 ++++++++++++++ src/core/config.js | 2 - src/{utils => data/redis}/captcha.js | 12 +-- src/routes/api/auth/register.js | 2 +- src/routes/api/captcha.js | 2 +- src/routes/api/me.js | 1 - src/routes/captcha.js | 38 ++++++++ src/routes/index.js | 9 +- src/routes/ranking.js | 6 +- src/socket/SocketServer.js | 2 +- src/ssr-components/Main.jsx | 3 +- src/workers/README.md | 4 + src/workers/captchaloader.js | 62 +++++++++++++ src/workers/tilewriter.js | 13 ++- webpack.config.client.babel.js | 2 +- webpack.config.server.babel.js | 29 ++++-- 21 files changed, 220 insertions(+), 159 deletions(-) delete mode 100644 deployment/example-ecosystem-captchas.yml delete mode 100644 src/captchaserver.js create mode 100644 src/core/captchaserver.js rename src/{utils => data/redis}/captcha.js (94%) create mode 100644 src/routes/captcha.js create mode 100644 src/workers/README.md create mode 100644 src/workers/captchaloader.js diff --git a/README.md b/README.md index d0db8f9..fd5e1bc 100644 --- a/README.md +++ b/README.md @@ -85,8 +85,8 @@ Configuration takes place in the environment variables that are defined in ecosy | USE_PROXYCHECK | Check users for Proxies | 0 | | APISOCKET_KEY | Key for API Socket for SpecialAccess™ | "SDfasife3" | | ADMIN_IDS | Ids of users with Admin rights | "1,12,3" | -| CAPTCHA_URL | URL where captcha is served | "http://localhost:8080" | | CAPTCHA_TIME | time in minutes between captchas | 30 | +| | 0: always captcha -1: never captcha | | | SESSION_SECRET | random sting for express sessions | "ayylmao" | | LOG_MYSQL | if sql queries should get logged | 0 | | USE_XREALIP | see ngins / CDN section | 1 | diff --git a/deployment/example-ecosystem-captchas.yml b/deployment/example-ecosystem-captchas.yml deleted file mode 100644 index 8ed0599..0000000 --- a/deployment/example-ecosystem-captchas.yml +++ /dev/null @@ -1,9 +0,0 @@ -apps: - - script : ./captchaserver.js - name : 'ppfun-captchas' - node_args: --nouse-idle-notification --expose-gc - env: - PORT: 8080 - HOST: "localhost" - REDIS_URL: 'redis://localhost:6379' - CAPTCHA_TIMEOUT: 120 diff --git a/deployment/githook.sh b/deployment/githook.sh index 3b4bebc..a015052 100755 --- a/deployment/githook.sh +++ b/deployment/githook.sh @@ -58,15 +58,12 @@ do cd "$PFOLDER" pm2 stop ppfun-server pm2 stop ppfun-backups - pm2 stop ppfun-captchs [ $DO_REINSTALL -eq 0 ] && npm_reinstall pm2 start ecosystem.yml pm2 start ecosystem-backup.yml - pm2 start ecosystem-captchas.yml else echo "---UPDATING REPO ON DEV SERVER---" pm2 stop ppfun-server-dev - pm2 stop ppfun-captchas-dev GIT_WORK_TREE="$BUILDDIR" GIT_DIR="${BUILDDIR}/.git" git reset --hard "origin/$branch" COMMITS=`git log --pretty=format:'- %s%b' $newrev ^$oldrev` COMMITS=`echo "$COMMITS" | sed ':a;N;$!ba;s/\n/\\\n/g'` @@ -88,6 +85,5 @@ do cd "$DEVFOLDER" [ $DO_REINSTALL -eq 0 ] && npm_reinstall pm2 start ecosystem.yml - pm2 start ecosystem-captchas.yml fi done diff --git a/src/captchaserver.js b/src/captchaserver.js deleted file mode 100644 index 1143078..0000000 --- a/src/captchaserver.js +++ /dev/null @@ -1,108 +0,0 @@ -/* - * serving captchas - */ - -/* eslint-disable no-console */ - -import path from 'path'; -import fs from 'fs'; -import process from 'process'; -import http from 'http'; -import url from 'url'; -import ppfunCaptcha from 'ppfun-captcha'; - -import { connect as connectRedis } from './data/redis/client'; -import { getIPFromRequest } from './utils/ip'; -import { setCaptchaSolution } from './utils/captcha'; -import { getRandomString } from './core/utils'; - -const PORT = process.env.PORT || 8080; -const HOST = process.env.HOST || 'localhost'; - -const font = fs.readdirSync(path.resolve(__dirname, 'captchaFonts')) - .filter((e) => e.endsWith('.ttf')) - .map((e) => ppfunCaptcha.loadFont( - path.resolve(__dirname, 'captchaFonts', e), - )); - -const server = http.createServer((req, res) => { - console.log(req.url); - - req.on('error', (err) => { - console.error(err); - }); - - const urlObject = url.parse(req.url, true); - - if (req.method === 'GET' && urlObject.pathname.endsWith('.svg')) { - try { - const captcha = ppfunCaptcha.create({ - width: 500, - height: 300, - fontSize: 180, - stroke: 'black', - fill: 'none', - nodeDeviation: 2.5, - connectionPathDeviation: 10.0, - style: 'stroke-width: 4;', - background: '#EFEFEF', - font, - }); - - const ip = getIPFromRequest(req); - const captchaid = getRandomString(); - - setCaptchaSolution(captcha.text, ip, captchaid); - console.log(`Serving ${captcha.text} to ${ip} / ${captchaid}`); - - res.writeHead(200, { - 'Content-Type': 'image/svg+xml', - 'Cache-Control': 'no-cache', - 'Captcha-Id': captchaid, - }); - res.write(captcha.data); - res.end(); - } catch (error) { - console.error(error); - res.writeHead(503, { - 'Content-Type': 'text/html', - 'Cache-Control': 'no-cache', - }); - res.end( - // eslint-disable-next-line max-len - '

Captchaserver: 503 Server Error

Captchas are accessible via *.svp paths', - ); - } - } else { - res.writeHead(404, { - 'Content-Type': 'text/html', - 'Cache-Control': 'no-cache', - }); - res.end( - // eslint-disable-next-line max-len - '

Captchaserver: 404 Not Found

Captchas are accessible via *.svp paths', - ); - } -}); - -// connect to redis -connectRedis() - .then(() => { - // start http server - const startServer = () => { - server.listen(PORT, HOST, () => { - console.log(`Captcha Server listening on port ${PORT}`); - }); - }; - startServer(); - // catch errors of server - server.on('error', (e) => { - console.error( - `Captcha Server Error ${e.code} occured, trying again in 5s...`, - ); - setTimeout(() => { - server.close(); - startServer(); - }, 5000); - }); - }); diff --git a/src/components/Captcha.jsx b/src/components/Captcha.jsx index ab0bc7e..4f70c7b 100644 --- a/src/components/Captcha.jsx +++ b/src/components/Captcha.jsx @@ -12,7 +12,7 @@ import { t } from 'ttag'; import { IoReloadCircleSharp } from 'react-icons/io5'; async function getUrlAndId() { - const url = window.ssv.captchaurl; + const url = './captcha.svg'; const resp = await fetch(url, { cache: 'no-cache', }); diff --git a/src/core/captchaserver.js b/src/core/captchaserver.js new file mode 100644 index 0000000..da58153 --- /dev/null +++ b/src/core/captchaserver.js @@ -0,0 +1,67 @@ +/* + * creation of captchas + */ + +import { Worker } from 'worker_threads'; + +import logger from './logger'; + +const MAX_WAIT = 20 * 1000; + +/* + * worker thread + */ +const worker = new Worker('./workers/captchaloader.js'); + +/* + * queue of captcha-generation tasks + * [[ timestamp, callbackFunction ],...] + */ +let captchaQueue = []; + +/* + * generate a captcha in the worker thread + * calls callback with arguments: + * (error, captcha.text, captcha.svgdata, captcha.id) + */ +function requestCaptcha(cb) { + worker.postMessage('createCaptcha'); + captchaQueue.push([ + Date.now(), + cb, + ]); +} + +/* + * answer of worker thread + */ +worker.on('message', (msg) => { + const task = captchaQueue.shift(); + task[1](...msg); +}); + +/* + * checks queue of captcha requests for stale + * unanswered requests + */ +function clearOldQueue() { + const now = Date.now(); + captchaQueue = captchaQueue.filter((task) => { + if (now - task[0] > MAX_WAIT) { + logger.warn( + 'Captchas: Thread took longer than 30s to generate captcha', + ); + try { + task[1]('TIMEOUT'); + } catch { + // nothing + } + return false; + } + return true; + }); +} + +setInterval(clearOldQueue, MAX_WAIT); + +export default requestCaptcha; diff --git a/src/core/config.js b/src/core/config.js index c785a92..0f54dff 100644 --- a/src/core/config.js +++ b/src/core/config.js @@ -20,8 +20,6 @@ export const TILE_FOLDER = path.join(__dirname, `./${TILE_FOLDER_REL}`); export const ASSET_SERVER = process.env.ASSET_SERVER || '.'; -export const CAPTCHA_URL = process.env.CAPTCHA_URL || null; - export const USE_XREALIP = process.env.USE_XREALIP || false; export const BACKUP_URL = process.env.BACKUP_URL || null; diff --git a/src/utils/captcha.js b/src/data/redis/captcha.js similarity index 94% rename from src/utils/captcha.js rename to src/data/redis/captcha.js index 9c4ea7d..9a2e62e 100644 --- a/src/utils/captcha.js +++ b/src/data/redis/captcha.js @@ -3,14 +3,13 @@ * check for captcha requirement */ -import logger from '../core/logger'; -import redis from '../data/redis/client'; -import { getIPv6Subnet } from './ip'; +import logger from '../../core/logger'; +import redis from './client'; +import { getIPv6Subnet } from '../../utils/ip'; import { - CAPTCHA_URL, CAPTCHA_TIME, CAPTCHA_TIMEOUT, -} from '../core/config'; +} from '../../core/config'; const TTL_CACHE = CAPTCHA_TIME * 60; // seconds @@ -142,10 +141,9 @@ export async function checkCaptchaSolution( * @return boolean true if needed */ export async function needCaptcha(ip) { - if (!CAPTCHA_URL) { + if (CAPTCHA_TIME < 0) { return false; } - const key = `human:${getIPv6Subnet(ip)}`; const ttl = await redis.ttl(key); if (ttl > 0) { diff --git a/src/routes/api/auth/register.js b/src/routes/api/auth/register.js index 430acbf..894d618 100644 --- a/src/routes/api/auth/register.js +++ b/src/routes/api/auth/register.js @@ -19,7 +19,7 @@ import { } from '../../../utils/validation'; import { checkCaptchaSolution, -} from '../../../utils/captcha'; +} from '../../../data/redis/captcha'; async function validate(email, name, password, captcha, captchaid, t, gettext) { const errors = []; diff --git a/src/routes/api/captcha.js b/src/routes/api/captcha.js index b1fe9d9..e8c5f00 100644 --- a/src/routes/api/captcha.js +++ b/src/routes/api/captcha.js @@ -6,7 +6,7 @@ */ import logger from '../../core/logger'; -import { checkCaptchaSolution } from '../../utils/captcha'; +import { checkCaptchaSolution } from '../../data/redis/captcha'; import { getIPFromRequest } from '../../utils/ip'; export default async (req, res) => { diff --git a/src/routes/api/me.js b/src/routes/api/me.js index 2e5c9ca..57eff75 100644 --- a/src/routes/api/me.js +++ b/src/routes/api/me.js @@ -26,7 +26,6 @@ export default async (req, res, next) => { // https://stackoverflow.com/questions/49547/how-to-control-web-page-caching-across-all-browsers res.set({ 'Cache-Control': 'no-cache, no-store, must-revalidate', - Pragma: 'no-cache', Expires: '0', }); res.json(userdata); diff --git a/src/routes/captcha.js b/src/routes/captcha.js new file mode 100644 index 0000000..ecdb675 --- /dev/null +++ b/src/routes/captcha.js @@ -0,0 +1,38 @@ +/* + * route providing captcha + */ +import logger from '../core/logger'; +import requestCaptcha from '../core/captchaserver'; +import { getIPFromRequest } from '../utils/ip'; +import { setCaptchaSolution } from '../data/redis/captcha'; + +export default (req, res) => { + res.set({ + 'Cache-Control': 'no-cache, no-store, must-revalidate', + }); + + requestCaptcha((err, text, data, id) => { + if (res.writableEnded) { + return; + } + + if (err) { + res.status(503); + res.send( + // eslint-disable-next-line max-len + '

Captchaserver: 503 Server Error

Captchas are accessible via *.svp paths', + ); + return; + } + + const ip = getIPFromRequest(req); + setCaptchaSolution(text, ip, id); + logger.info(`Captchas: ${ip} got captcha with text: ${text}`); + + res.set({ + 'Content-Type': 'image/svg+xml', + 'Captcha-Id': id, + }); + res.end(data); + }); +}; diff --git a/src/routes/index.js b/src/routes/index.js index a05b8f7..3477380 100644 --- a/src/routes/index.js +++ b/src/routes/index.js @@ -1,6 +1,5 @@ /** * - * @flow */ import express from 'express'; @@ -11,9 +10,10 @@ import ranking from './ranking'; import history from './history'; import tiles from './tiles'; import chunks from './chunks'; +import adminapi from './adminapi'; +import captcha from './captcha'; import resetPassword from './reset_password'; import api from './api'; -import adminapi from './adminapi'; import assets from './assets.json'; // eslint-disable-line import/no-unresolved import { expressTTag } from '../core/ttag'; @@ -47,6 +47,11 @@ router.use('/tiles', tiles); */ router.use('/adminapi', adminapi); +/* + * serve captcha + */ +router.get('/captcha.svg', captcha); + /* * public folder * (this should be served with nginx or other webserver) diff --git a/src/routes/ranking.js b/src/routes/ranking.js index ebc29c5..c893357 100644 --- a/src/routes/ranking.js +++ b/src/routes/ranking.js @@ -1,13 +1,9 @@ /* * send global ranking - * @flow */ -import type { Request, Response } from 'express'; - import rankings from '../core/ranking'; - -export default async (req: Request, res: Response) => { +export default (req, res) => { res.json(rankings.ranks); }; diff --git a/src/socket/SocketServer.js b/src/socket/SocketServer.js index a003edd..18a52e1 100644 --- a/src/socket/SocketServer.js +++ b/src/socket/SocketServer.js @@ -24,7 +24,7 @@ import socketEvents from './SocketEvents'; import chatProvider, { ChatProvider } from '../core/ChatProvider'; import authenticateClient from './authenticateClient'; import { drawByOffsets } from '../core/draw'; -import { needCaptcha } from '../utils/captcha'; +import { needCaptcha } from '../data/redis/captcha'; import { cheapDetector } from '../core/isProxy'; diff --git a/src/ssr-components/Main.jsx b/src/ssr-components/Main.jsx index dabdf3e..c6e89e9 100644 --- a/src/ssr-components/Main.jsx +++ b/src/ssr-components/Main.jsx @@ -17,7 +17,7 @@ import assets from './assets.json'; // eslint-disable-next-line import/no-unresolved import styleassets from './styleassets.json'; -import { CAPTCHA_URL, ASSET_SERVER, BACKUP_URL } from '../core/config'; +import { ASSET_SERVER, BACKUP_URL } from '../core/config'; /* * generate language list @@ -31,7 +31,6 @@ const langs = Object.keys(ttags) */ const ssv = { assetserver: ASSET_SERVER, - captchaurl: CAPTCHA_URL, availableStyles: styleassets, langs, }; diff --git a/src/workers/README.md b/src/workers/README.md new file mode 100644 index 0000000..e5363b0 --- /dev/null +++ b/src/workers/README.md @@ -0,0 +1,4 @@ +# Worker Threads + +Every single .js file here automatically gets its own webpack entry point and +will get build into its own worker/filename file. diff --git a/src/workers/captchaloader.js b/src/workers/captchaloader.js new file mode 100644 index 0000000..2c6016a --- /dev/null +++ b/src/workers/captchaloader.js @@ -0,0 +1,62 @@ +/* + * worker thread for creating captchas + */ + +/* eslint-disable no-console */ + +import fs from 'fs'; +import path from 'path'; +import ppfunCaptcha from 'ppfun-captcha'; +import { isMainThread, parentPort } from 'worker_threads'; + +import { getRandomString } from '../core/utils'; + +const FONT_FOLDER = 'captchaFonts'; + +if (isMainThread) { + throw new Error( + 'Tilewriter is run as a worker thread, not as own process', + ); +} + +const font = fs.readdirSync(path.resolve(__dirname, '..', FONT_FOLDER)) + .filter((e) => e.endsWith('.ttf')) + .map((e) => ppfunCaptcha.loadFont( + path.resolve(__dirname, '..', FONT_FOLDER, e), + )); + +function createCaptcha() { + return ppfunCaptcha.create({ + width: 500, + height: 300, + fontSize: 180, + stroke: 'black', + fill: 'none', + nodeDeviation: 2.5, + connectionPathDeviation: 10.0, + style: 'stroke-width: 4;', + background: '#EFEFEF', + font, + }); +} + +parentPort.on('message', (msg) => { + try { + if (msg === 'createCaptcha') { + const captcha = createCaptcha(); + const captchaid = getRandomString(); + parentPort.postMessage([ + null, + captcha.text, + captcha.data, + captchaid, + ]); + } + } catch (error) { + console.warn( + // eslint-disable-next-line max-len + `Captchas: Error on createCaptcha: ${error.message}`, + ); + parentPort.postMessage(['Failure!']); + } +}); diff --git a/src/workers/tilewriter.js b/src/workers/tilewriter.js index 8c5b02c..21828dc 100644 --- a/src/workers/tilewriter.js +++ b/src/workers/tilewriter.js @@ -36,8 +36,16 @@ connectRedis() createTexture(...args); break; case 'initializeTiles': - await initializeTiles(...args); - parentPort.postMessage('Done!'); + try { + await initializeTiles(...args); + parentPort.postMessage('Done!'); + } catch (err) { + console.warn( + // eslint-disable-next-line max-len + `Tiling: Error on initializeTiles args ${args}: ${err.message}`, + ); + parentPort.postMessage('Failure!'); + } break; default: console.warn(`Tiling: Main thread requested unknown task ${task}`); @@ -47,7 +55,6 @@ connectRedis() // eslint-disable-next-line max-len `Tiling: Error on executing task ${task} args ${args}: ${error.message}`, ); - parentPort.postMessage('Failure!'); } }); }); diff --git a/webpack.config.client.babel.js b/webpack.config.client.babel.js index 67d3d0d..7c7aeea 100644 --- a/webpack.config.client.babel.js +++ b/webpack.config.client.babel.js @@ -1,8 +1,8 @@ /** */ -import path from 'path'; import fs from 'fs'; +import path from 'path'; import webpack from 'webpack'; import AssetsPlugin from 'assets-webpack-plugin'; import { BundleAnalyzerPlugin } from 'webpack-bundle-analyzer'; diff --git a/webpack.config.server.babel.js b/webpack.config.server.babel.js index 1949e1a..874735e 100644 --- a/webpack.config.server.babel.js +++ b/webpack.config.server.babel.js @@ -1,6 +1,7 @@ /* */ +import fs from 'fs'; import path from 'path'; import webpack from 'webpack'; import nodeExternals from 'webpack-node-externals'; @@ -34,6 +35,9 @@ const babelPlugins = [ export default ({ development, extract, }) => { + /* + * write template files for translations + */ if (extract) { ttag.extract = { output: path.resolve(__dirname, 'i18n', 'template-ssr.pot'), @@ -41,6 +45,20 @@ export default ({ ttag.discover = ['t', 'jt']; } + /* + * worker threads need to be their own + * entry points + */ + const workersDir = path.resolve(__dirname, 'src', 'workers'); + const workerEntries = {}; + fs.readdirSync(workersDir) + .filter((e) => e.endsWith('.js')) + .forEach((filename) => { + const name = `workers/${filename.slice(0, -3)}`; + const fullPath = path.resolve(workersDir, filename); + workerEntries[name] = fullPath; + }); + return { name: 'server', target: 'node', @@ -51,8 +69,7 @@ export default ({ entry: { server: [path.resolve(__dirname, 'src', 'server.js')], backup: [path.resolve(__dirname, 'src', 'backup.js')], - 'workers/tilewriter': [path.resolve(__dirname, 'src', 'workers', 'tilewriter.js')], - captchaserver: [path.resolve(__dirname, 'src', 'captchaserver.js')], + ...workerEntries, }, output: { @@ -147,14 +164,6 @@ export default ({ from: path.resolve(__dirname, 'deployment', 'captchaFonts'), to: path.resolve(__dirname, 'dist', 'captchaFonts'), }, - { - from: path.resolve( - __dirname, 'deployment', 'example-ecosystem-captchas.yml' - ), - to: path.resolve( - __dirname, 'dist', 'ecosystem-captchas.yml' - ), - }, ], }), ],