move captchas into their own thread

closes #3

README.md

@@ -85,8 +85,8 @@ Configuration takes place in the environment variables that are defined in ecosy
 | USE_PROXYCHECK    | Check users for Proxies               | 0                       |
 | APISOCKET_KEY     | Key for API Socket for SpecialAccess™ | "SDfasife3"             |
 | ADMIN_IDS         | Ids of users with Admin rights        | "1,12,3"                |
-| CAPTCHA_URL       | URL where captcha is served           | "http://localhost:8080" |
 | CAPTCHA_TIME      | time in minutes between captchas      | 30                      |
+|                   |  0: always captcha -1: never captcha  |                         |
 | SESSION_SECRET    | random sting for express sessions     | "ayylmao"               |
 | LOG_MYSQL         | if sql queries should get logged      | 0                       |
 | USE_XREALIP       | see ngins / CDN  section              | 1                       |

deployment/example-ecosystem-captchas.yml

@@ -1,9 +0,0 @@
-apps:
-  - script   : ./captchaserver.js
-    name     : 'ppfun-captchas'
-    node_args: --nouse-idle-notification --expose-gc
-    env:
-      PORT: 8080
-      HOST: "localhost"
-      REDIS_URL: 'redis://localhost:6379'
-      CAPTCHA_TIMEOUT: 120

deployment/githook.sh

@@ -58,15 +58,12 @@ do
         cd "$PFOLDER"
         pm2 stop ppfun-server
         pm2 stop ppfun-backups
-        pm2 stop ppfun-captchs
         [ $DO_REINSTALL -eq 0 ] && npm_reinstall
         pm2 start ecosystem.yml
         pm2 start ecosystem-backup.yml
-       pm2 start ecosystem-captchas.yml
     else
         echo "---UPDATING REPO ON DEV SERVER---"
         pm2 stop ppfun-server-dev
-        pm2 stop ppfun-captchas-dev
         GIT_WORK_TREE="$BUILDDIR" GIT_DIR="${BUILDDIR}/.git" git reset --hard "origin/$branch"
         COMMITS=`git log --pretty=format:'- %s%b' $newrev ^$oldrev`
         COMMITS=`echo "$COMMITS" | sed ':a;N;$!ba;s/\n/\\\n/g'`
@@ -88,6 +85,5 @@ do
         cd "$DEVFOLDER"
         [ $DO_REINSTALL -eq 0 ] && npm_reinstall
         pm2 start ecosystem.yml
-        pm2 start ecosystem-captchas.yml
     fi
 done

src/captchaserver.js

@@ -1,108 +0,0 @@
-/*
- * serving captchas
- */
-
-/* eslint-disable no-console */
-
-import path from 'path';
-import fs from 'fs';
-import process from 'process';
-import http from 'http';
-import url from 'url';
-import ppfunCaptcha from 'ppfun-captcha';
-
-import { connect as connectRedis } from './data/redis/client';
-import { getIPFromRequest } from './utils/ip';
-import { setCaptchaSolution } from './utils/captcha';
-import { getRandomString } from './core/utils';
-
-const PORT = process.env.PORT || 8080;
-const HOST = process.env.HOST || 'localhost';
-
-const font = fs.readdirSync(path.resolve(__dirname, 'captchaFonts'))
-  .filter((e) => e.endsWith('.ttf'))
-  .map((e) => ppfunCaptcha.loadFont(
-    path.resolve(__dirname, 'captchaFonts', e),
-  ));
-
-const server = http.createServer((req, res) => {
-  console.log(req.url);
-
-  req.on('error', (err) => {
-    console.error(err);
-  });
-
-  const urlObject = url.parse(req.url, true);
-
-  if (req.method === 'GET' && urlObject.pathname.endsWith('.svg')) {
-    try {
-      const captcha = ppfunCaptcha.create({
-        width: 500,
-        height: 300,
-        fontSize: 180,
-        stroke: 'black',
-        fill: 'none',
-        nodeDeviation: 2.5,
-        connectionPathDeviation: 10.0,
-        style: 'stroke-width: 4;',
-        background: '#EFEFEF',
-        font,
-      });
-
-      const ip = getIPFromRequest(req);
-      const captchaid = getRandomString();
-
-      setCaptchaSolution(captcha.text, ip, captchaid);
-      console.log(`Serving ${captcha.text} to ${ip} / ${captchaid}`);
-
-      res.writeHead(200, {
-        'Content-Type': 'image/svg+xml',
-        'Cache-Control': 'no-cache',
-        'Captcha-Id': captchaid,
-      });
-      res.write(captcha.data);
-      res.end();
-    } catch (error) {
-      console.error(error);
-      res.writeHead(503, {
-        'Content-Type': 'text/html',
-        'Cache-Control': 'no-cache',
-      });
-      res.end(
-        // eslint-disable-next-line max-len
-        '<html><body><h1>Captchaserver: 503 Server Error</h1>Captchas are accessible via *.svp paths</body></html>',
-      );
-    }
-  } else {
-    res.writeHead(404, {
-      'Content-Type': 'text/html',
-      'Cache-Control': 'no-cache',
-    });
-    res.end(
-      // eslint-disable-next-line max-len
-      '<html><body><h1>Captchaserver: 404 Not Found</h1>Captchas are accessible via *.svp paths</body></html>',
-    );
-  }
-});
-
-// connect to redis
-connectRedis()
-  .then(() => {
-    // start http server
-    const startServer = () => {
-      server.listen(PORT, HOST, () => {
-        console.log(`Captcha Server listening on port ${PORT}`);
-      });
-    };
-    startServer();
-    // catch errors of server
-    server.on('error', (e) => {
-      console.error(
-        `Captcha Server Error ${e.code} occured, trying again in 5s...`,
-      );
-      setTimeout(() => {
-        server.close();
-        startServer();
-      }, 5000);
-    });
-  });

src/components/Captcha.jsx

@@ -12,7 +12,7 @@ import { t } from 'ttag';
 import { IoReloadCircleSharp } from 'react-icons/io5';
 
 async function getUrlAndId() {
-  const url = window.ssv.captchaurl;
+  const url = './captcha.svg';
   const resp = await fetch(url, {
     cache: 'no-cache',
   });

src/core/captchaserver.js

@@ -0,0 +1,67 @@
+/*
+ * creation of captchas
+ */
+
+import { Worker } from 'worker_threads';
+
+import logger from './logger';
+
+const MAX_WAIT = 20 * 1000;
+
+/*
+ * worker thread
+ */
+const worker = new Worker('./workers/captchaloader.js');
+
+/*
+ * queue of captcha-generation tasks
+ * [[ timestamp, callbackFunction ],...]
+ */
+let captchaQueue = [];
+
+/*
+ * generate a captcha in the worker thread
+ * calls callback with arguments:
+ *  (error, captcha.text, captcha.svgdata, captcha.id)
+ */
+function requestCaptcha(cb) {
+  worker.postMessage('createCaptcha');
+  captchaQueue.push([
+    Date.now(),
+    cb,
+  ]);
+}
+
+/*
+ * answer of worker thread
+ */
+worker.on('message', (msg) => {
+  const task = captchaQueue.shift();
+  task[1](...msg);
+});
+
+/*
+ * checks queue of captcha requests for stale
+ * unanswered requests
+ */
+function clearOldQueue() {
+  const now = Date.now();
+  captchaQueue = captchaQueue.filter((task) => {
+    if (now - task[0] > MAX_WAIT) {
+      logger.warn(
+        'Captchas: Thread took longer than 30s to generate captcha',
+      );
+      try {
+        task[1]('TIMEOUT');
+      } catch {
+        // nothing
+      }
+      return false;
+    }
+    return true;
+  });
+}
+
+setInterval(clearOldQueue, MAX_WAIT);
+
+export default requestCaptcha;

src/core/config.js

@@ -20,8 +20,6 @@ export const TILE_FOLDER = path.join(__dirname, `./${TILE_FOLDER_REL}`);
 
 export const ASSET_SERVER = process.env.ASSET_SERVER || '.';
 
-export const CAPTCHA_URL = process.env.CAPTCHA_URL || null;
-
 export const USE_XREALIP = process.env.USE_XREALIP || false;
 
 export const BACKUP_URL = process.env.BACKUP_URL || null;

src/data/redis/captcha.js

@@ -3,14 +3,13 @@
  * check for captcha requirement
  */
 
-import logger from '../core/logger';
-import redis from '../data/redis/client';
-import { getIPv6Subnet } from './ip';
+import logger from '../../core/logger';
+import redis from './client';
+import { getIPv6Subnet } from '../../utils/ip';
 import {
-  CAPTCHA_URL,
   CAPTCHA_TIME,
   CAPTCHA_TIMEOUT,
-} from '../core/config';
+} from '../../core/config';
 
 const TTL_CACHE = CAPTCHA_TIME * 60; // seconds
 
@@ -142,10 +141,9 @@ export async function checkCaptchaSolution(
  * @return boolean true if needed
  */
 export async function needCaptcha(ip) {
-  if (!CAPTCHA_URL) {
+  if (CAPTCHA_TIME < 0) {
     return false;
   }
-
   const key = `human:${getIPv6Subnet(ip)}`;
   const ttl = await redis.ttl(key);
   if (ttl > 0) {

src/routes/api/auth/register.js

@@ -19,7 +19,7 @@ import {
 } from '../../../utils/validation';
 import {
   checkCaptchaSolution,
-} from '../../../utils/captcha';
+} from '../../../data/redis/captcha';
 
 async function validate(email, name, password, captcha, captchaid, t, gettext) {
   const errors = [];

src/routes/api/captcha.js

@@ -6,7 +6,7 @@
  */
 
 import logger from '../../core/logger';
-import { checkCaptchaSolution } from '../../utils/captcha';
+import { checkCaptchaSolution } from '../../data/redis/captcha';
 import { getIPFromRequest } from '../../utils/ip';
 
 export default async (req, res) => {

src/routes/api/me.js

@@ -26,7 +26,6 @@ export default async (req, res, next) => {
     // https://stackoverflow.com/questions/49547/how-to-control-web-page-caching-across-all-browsers
     res.set({
       'Cache-Control': 'no-cache, no-store, must-revalidate',
-      Pragma: 'no-cache',
       Expires: '0',
     });
     res.json(userdata);

src/routes/captcha.js

@@ -0,0 +1,38 @@
+/*
+ * route providing captcha
+ */
+import logger from '../core/logger';
+import requestCaptcha from '../core/captchaserver';
+import { getIPFromRequest } from '../utils/ip';
+import { setCaptchaSolution } from '../data/redis/captcha';
+
+export default (req, res) => {
+  res.set({
+    'Cache-Control': 'no-cache, no-store, must-revalidate',
+  });
+
+  requestCaptcha((err, text, data, id) => {
+    if (res.writableEnded) {
+      return;
+    }
+
+    if (err) {
+      res.status(503);
+      res.send(
+        // eslint-disable-next-line max-len
+        '<html><body><h1>Captchaserver: 503 Server Error</h1>Captchas are accessible via *.svp paths</body></html>',
+      );
+      return;
+    }
+
+    const ip = getIPFromRequest(req);
+    setCaptchaSolution(text, ip, id);
+    logger.info(`Captchas: ${ip} got captcha with text: ${text}`);
+
+    res.set({
+      'Content-Type': 'image/svg+xml',
+      'Captcha-Id': id,
+    });
+    res.end(data);
+  });
+};

src/routes/index.js

@@ -1,6 +1,5 @@
 /**
  *
- * @flow
  */
 
 import express from 'express';
@@ -11,9 +10,10 @@ import ranking from './ranking';
 import history from './history';
 import tiles from './tiles';
 import chunks from './chunks';
+import adminapi from './adminapi';
+import captcha from './captcha';
 import resetPassword from './reset_password';
 import api from './api';
-import adminapi from './adminapi';
 
 import assets from './assets.json'; // eslint-disable-line import/no-unresolved
 import { expressTTag } from '../core/ttag';
@@ -47,6 +47,11 @@ router.use('/tiles', tiles);
  */
 router.use('/adminapi', adminapi);
 
+/*
+ * serve captcha
+ */
+router.get('/captcha.svg', captcha);
+
 /*
  * public folder
  * (this should be served with nginx or other webserver)

src/routes/ranking.js

@@ -1,13 +1,9 @@
 /*
  * send global ranking
- * @flow
  */
 
-import type { Request, Response } from 'express';
-
 import rankings from '../core/ranking';
 
-
-export default async (req: Request, res: Response) => {
+export default (req, res) => {
   res.json(rankings.ranks);
 };

src/socket/SocketServer.js

@@ -24,7 +24,7 @@ import socketEvents from './SocketEvents';
 import chatProvider, { ChatProvider } from '../core/ChatProvider';
 import authenticateClient from './authenticateClient';
 import { drawByOffsets } from '../core/draw';
-import { needCaptcha } from '../utils/captcha';
+import { needCaptcha } from '../data/redis/captcha';
 import { cheapDetector } from '../core/isProxy';
 
 

src/ssr-components/Main.jsx

@@ -17,7 +17,7 @@ import assets from './assets.json';
 // eslint-disable-next-line import/no-unresolved
 import styleassets from './styleassets.json';
 
-import { CAPTCHA_URL, ASSET_SERVER, BACKUP_URL } from '../core/config';
+import { ASSET_SERVER, BACKUP_URL } from '../core/config';
 
 /*
  * generate language list
@@ -31,7 +31,6 @@ const langs = Object.keys(ttags)
  */
 const ssv = {
   assetserver: ASSET_SERVER,
-  captchaurl: CAPTCHA_URL,
   availableStyles: styleassets,
   langs,
 };

src/workers/README.md

@@ -0,0 +1,4 @@
+# Worker Threads
+
+Every single .js file here automatically gets its own webpack entry point and
+will get build into its own worker/filename file.

src/workers/captchaloader.js

@@ -0,0 +1,62 @@
+/*
+ * worker thread for creating captchas
+ */
+
+/* eslint-disable no-console */
+
+import fs from 'fs';
+import path from 'path';
+import ppfunCaptcha from 'ppfun-captcha';
+import { isMainThread, parentPort } from 'worker_threads';
+
+import { getRandomString } from '../core/utils';
+
+const FONT_FOLDER = 'captchaFonts';
+
+if (isMainThread) {
+  throw new Error(
+    'Tilewriter is run as a worker thread, not as own process',
+  );
+}
+
+const font = fs.readdirSync(path.resolve(__dirname, '..', FONT_FOLDER))
+  .filter((e) => e.endsWith('.ttf'))
+  .map((e) => ppfunCaptcha.loadFont(
+    path.resolve(__dirname, '..', FONT_FOLDER, e),
+  ));
+
+function createCaptcha() {
+  return ppfunCaptcha.create({
+    width: 500,
+    height: 300,
+    fontSize: 180,
+    stroke: 'black',
+    fill: 'none',
+    nodeDeviation: 2.5,
+    connectionPathDeviation: 10.0,
+    style: 'stroke-width: 4;',
+    background: '#EFEFEF',
+    font,
+  });
+}
+
+parentPort.on('message', (msg) => {
+  try {
+    if (msg === 'createCaptcha') {
+      const captcha = createCaptcha();
+      const captchaid = getRandomString();
+      parentPort.postMessage([
+        null,
+        captcha.text,
+        captcha.data,
+        captchaid,
+      ]);
+    }
+  } catch (error) {
+    console.warn(
+      // eslint-disable-next-line max-len
+      `Captchas: Error on createCaptcha: ${error.message}`,
+    );
+    parentPort.postMessage(['Failure!']);
+  }
+});

src/workers/tilewriter.js

@@ -36,8 +36,16 @@ connectRedis()
             createTexture(...args);
             break;
           case 'initializeTiles':
-            await initializeTiles(...args);
-            parentPort.postMessage('Done!');
+            try {
+              await initializeTiles(...args);
+              parentPort.postMessage('Done!');
+            } catch (err) {
+              console.warn(
+                // eslint-disable-next-line max-len
+                `Tiling: Error on initializeTiles args ${args}: ${err.message}`,
+              );
+              parentPort.postMessage('Failure!');
+            }
             break;
           default:
             console.warn(`Tiling: Main thread requested unknown task ${task}`);
@@ -47,7 +55,6 @@ connectRedis()
           // eslint-disable-next-line max-len
           `Tiling: Error on executing task ${task} args ${args}: ${error.message}`,
         );
-        parentPort.postMessage('Failure!');
       }
     });
   });

webpack.config.client.babel.js

@@ -1,8 +1,8 @@
 /**
  */
 
-import path from 'path';
 import fs from 'fs';
+import path from 'path';
 import webpack from 'webpack';
 import AssetsPlugin from 'assets-webpack-plugin';
 import { BundleAnalyzerPlugin } from 'webpack-bundle-analyzer';

webpack.config.server.babel.js

@@ -1,6 +1,7 @@
 /*
  */
 
+import fs from 'fs';
 import path from 'path';
 import webpack from 'webpack';
 import nodeExternals from 'webpack-node-externals';
@@ -34,6 +35,9 @@ const babelPlugins = [
 export default ({
   development, extract,
 }) => {
+  /*
+   * write template files for translations
+   */
   if (extract) {
     ttag.extract = {
       output: path.resolve(__dirname, 'i18n', 'template-ssr.pot'),
@@ -41,6 +45,20 @@ export default ({
     ttag.discover = ['t', 'jt'];
   }
 
+  /*
+   * worker threads need to be their own
+   * entry points
+   */
+  const workersDir = path.resolve(__dirname, 'src', 'workers');
+  const workerEntries = {};
+  fs.readdirSync(workersDir)
+    .filter((e) => e.endsWith('.js'))
+    .forEach((filename) => {
+      const name = `workers/${filename.slice(0, -3)}`;
+      const fullPath = path.resolve(workersDir, filename);
+      workerEntries[name] = fullPath;
+    });
+
   return {
     name: 'server',
     target: 'node',
@@ -51,8 +69,7 @@ export default ({
     entry: {
       server: [path.resolve(__dirname, 'src', 'server.js')],
       backup: [path.resolve(__dirname, 'src', 'backup.js')],
-      'workers/tilewriter': [path.resolve(__dirname, 'src', 'workers', 'tilewriter.js')],
-      captchaserver: [path.resolve(__dirname, 'src', 'captchaserver.js')],
+      ...workerEntries,
     },
 
     output: {
@@ -147,14 +164,6 @@ export default ({
             from: path.resolve(__dirname, 'deployment', 'captchaFonts'),
             to: path.resolve(__dirname, 'dist', 'captchaFonts'),
           },
-          {
-            from: path.resolve(
-              __dirname, 'deployment', 'example-ecosystem-captchas.yml'
-            ),
-            to: path.resolve(
-              __dirname, 'dist', 'ecosystem-captchas.yml'
-            ),
-          },
         ],
       }),
     ],