use proxycheck keys instead of proxiesFetch

README.md

@@ -83,6 +83,7 @@ Configuration takes place in the environment variables that are defined in ecosy
 |-------------------|:--------------------------------------|---------------------------|
 | ASSET_SERVER      | URL for assets                        | "http://localhost"        |
 | USE_PROXYCHECK    | Check users for Proxies               | 0                         |
+| PROXYCHECK_KEY    | Key for proxycheck.io                 | "asfas-xcsc-ewef-sdfsd"   |
 | APISOCKET_KEY     | Key for API Socket for SpecialAccess™ | "SDfasife3"               |
 | ADMIN_IDS         | Ids of users with Admin rights        | "1,12,3"                  |
 | CAPTCHA_TIME      | time in minutes between captchas      | 30                        |
@@ -115,7 +116,7 @@ Configuration takes place in the environment variables that are defined in ecosy
 Notes:
 
 - HOST / PORT is the host on which the ppfun server is listening. In example: If you have a reverse proxy on the same machine, HOST should still be unset or localhost, because it's where the proxy forwards to.
-- to be able to use USE_PROXYCHECK, you have to have an account on proxycheck.io or getipintel or another checker setup and you might set some proxies in`proxies.json` that get used for making proxycheck requests. Look into `src/isProxy.js` to see how things work, but keep in mind that this isn't neccessarily how pixelplanet.fun uses it.
+- to be able to use USE_PROXYCHECK effectively, you have to have an account on proxycheck.io and PROXYCHECK_KEY set.
 - Admins are users with 0cd and access to `Admintools`in their User Menu for image-upload and whatever
 - You can find out the id of a user by looking into the logs (i.e. `info: {ip} / {id} wants to place 2 in (1701, -8315)`) when he places a pixel or by checking the MySql Users database
 - pixelplanet uses the unix command sendmail for sending verification and password reset mails. If you don't want to set up your own mail server, look into [ssmtp](https://wiki.archlinux.org/title/SSMTP), which provides a sendmail interface that forwards to other providers like gmail.

package.json

@@ -29,9 +29,7 @@
     "etag": "^1.8.1",
     "express": "^4.17.2",
     "express-session": "^1.17.2",
-    "http-proxy-agent": "^5.0.0",
     "image-q": "^4.0.0",
-    "isomorphic-fetch": "^3.0.0",
     "js-file-download": "^0.4.12",
     "localforage": "^1.10.0",
     "morgan": "^1.10.0",

src/client.js

@@ -2,9 +2,6 @@
  * Entrypoint for main client script
  */
 
-// eslint-disable-next-line no-unused-vars
-import fetch from 'isomorphic-fetch'; // TODO put in the beggining with webpack!
-
 import createKeyPressHandler from './controls/keypress';
 import {
   fetchMe,

src/core/config.js

@@ -1,5 +1,7 @@
-// general config that is also available from client code can be found in
-// src/core/constants.js
+/*
+ * general config that is also available from client code can be found in
+ * src/core/constants.js
+ */
 import path from 'path';
 
 if (process.env.BROWSER) {
@@ -27,6 +29,7 @@ export const BACKUP_DIR = process.env.BACKUP_DIR || null;
 
 // Proxycheck
 export const USE_PROXYCHECK = parseInt(process.env.USE_PROXYCHECK, 10) || false;
+export const { PROXYCHECK_KEY } = process.env;
 
 export const REDIS_URL = process.env.REDIS_URL || 'redis://localhost:6379';
 // Database

src/core/isAllowed.js

@@ -2,10 +2,9 @@
  * decide if IP is allowed
  * does proxycheck and check bans and whitelists
  */
-import fetch from '../utils/proxiedFetch';
-
 import { getIPv6Subnet } from '../utils/ip';
 import whois from '../utils/whois';
+import getProxyCheck from '../utils/proxycheck';
 import { IPInfo } from '../data/sql';
 import { isIPBanned } from '../data/sql/Ban';
 import { isWhitelisted } from '../data/sql/Whitelist';
@@ -17,76 +16,6 @@ import { proxyLogger as logger } from './logger';
 
 import { USE_PROXYCHECK } from './config';
 
-/*
- * check getipintel if IP is proxy
- * Use proxiedFetch with random proxies and random mail for it, to not get blacklisted
- * @param ip IP to check
- * @return true if proxy, false if not
- */
-// eslint-disable-next-line no-unused-vars
-async function getIPIntel(ip) {
-  // eslint-disable-next-line max-len
-  const email = `${Math.random().toString(36).substring(8)}-${Math.random().toString(36).substring(4)}@gmail.com`;
-  // eslint-disable-next-line max-len
-  const url = `http://check.getipintel.net/check.php?ip=${ip}&contact=${email}&flags=m`;
-  logger.info(`fetching getipintel ${url}`);
-  const response = await fetch(url, {
-    headers: {
-      Accept: '*/*',
-      'Accept-Language': 'de,en-US;q=0.7,en;q=0.3',
-      Referer: 'http://check.getipintel.net/',
-      // eslint-disable-next-line max-len
-      'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36',
-    },
-  });
-  if (!response.ok) {
-    const text = await response.text();
-    throw new Error(`getipintel not ok ${response.status}/${text}`);
-  }
-  const body = await response.text();
-  logger.info('PROXYCHECK %s : %s', ip, body);
-  // returns tru iff we found 1 in the response and was ok (http code = 200)
-  const value = parseFloat(body);
-  return [
-    value > 0.995,
-    `score:${value}`,
-  ];
-}
-
-/*
- * check proxycheck.io if IP is proxy
- * Use proxiedFetch with random proxies
- * @param ip IP to check
- * @return [ isProxy, info] true if proxy and extra info
- */
-async function getProxyCheck(ip) {
-  const url = `http://proxycheck.io/v2/${ip}?risk=1&vpn=1&asn=1`;
-  logger.info('fetching proxycheck %s', url);
-  const response = await fetch(url, {
-    headers: {
-      // eslint-disable-next-line max-len
-      'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36',
-    },
-  });
-  if (!response.ok) {
-    const text = await response.text();
-    throw new Error(`proxycheck not ok ${response.status}/${text}`);
-  }
-  const data = await response.json();
-  logger.info('PROXYCHECK', data);
-  if (!data.status) {
-    return [
-      false,
-      'status not ok',
-    ];
-  }
-  const ipData = data[ip];
-  return [
-    ipData.proxy === 'yes',
-    `${ipData.type},${ipData.city}`,
-  ];
-}
-
 /*
  * dummy function to include if you don't want any proxycheck
  */
@@ -94,6 +23,9 @@ async function dummy() {
   return [false, 'dummy'];
 }
 
+/*
+ * save information of ip into database
+ */
 async function saveIPInfo(ip, whoisRet, allowed, info) {
   try {
     await IPInfo.upsert({
@@ -117,26 +49,30 @@ async function withoutCache(f, ip) {
   const ipKey = getIPv6Subnet(ip);
   let allowed = true;
   let status = -2;
-  let pcInfo = null;
+  let pcheck = null;
   let whoisRet = null;
 
   try {
     if (await isWhitelisted(ipKey)) {
       allowed = true;
-      pcInfo = 'wl';
+      pcheck = 'wl';
       status = -1;
     } else if (await isIPBanned(ipKey)) {
       allowed = false;
-      pcInfo = 'bl';
+      pcheck = 'bl';
       status = 2;
     } else {
-      [allowed, pcInfo] = await f(ip);
-      allowed = !allowed;
-      status = (allowed) ? 0 : 1;
+      const res = await f(ip);
+      status = res.status;
+      allowed = res.allowed;
+      pcheck = res.pcheck;
+      if (status === -2) {
+        throw new Error('Proxycheck request did not return yet');
+      }
     }
-    whoisRet = await whois(ip) || {};
+    whoisRet = await whois(ip);
   } finally {
-    await saveIPInfo(ipKey, whoisRet, status, pcInfo);
+    await saveIPInfo(ipKey, whoisRet || {}, status, pcheck);
   }
 
   return {
@@ -152,7 +88,6 @@ async function withoutCache(f, ip) {
  * @param ip IP to check
  * @return true if proxy or blacklisted, false if not or whitelisted
  */
-let lock = 4;
 const checking = [];
 async function withCache(f, ip) {
   if (!ip || ip === '0.0.0.1') {
@@ -169,10 +104,8 @@ async function withCache(f, ip) {
   }
 
   // else make asynchronous ipcheck and assume no proxy in the meantime
-  // use lock to just check three at a time
   // do not check ip that currently gets checked
-  if (checking.indexOf(ipKey) === -1 && lock > 0) {
-    lock -= 1;
+  if (checking.indexOf(ipKey) === -1) {
     checking.push(ipKey);
     withoutCache(f, ip)
       .then((result) => {
@@ -184,7 +117,6 @@ async function withCache(f, ip) {
       .finally(() => {
         const pos = checking.indexOf(ipKey);
         if (~pos) checking.splice(pos, 1);
-        lock += 1;
       });
   }
   return {

src/data/redis/isAllowedCache.js

@@ -6,7 +6,7 @@
 import client from './client';
 
 const PREFIX = 'isal:';
-const CACHE_DURATION = 3 * 24 * 3600;
+const CACHE_DURATION = 14 * 24 * 3600;
 
 export function cacheAllowed(ip, allowed) {
   const key = `${PREFIX}:${ip}`;

src/proxies.json

@@ -1 +0,0 @@
-[]

src/store/middleware/audio.js

@@ -95,9 +95,6 @@ export default (store) => (next) => (action) => {
       }
 
       case 'ALERT': {
-        if (action.alertType !== 'error') {
-          break;
-        }
         const oscillatorNode = context.createOscillator();
         const gainNode = context.createGain();
 

src/utils/proxiedFetch.js

@@ -1,33 +0,0 @@
-/*
- *
- * implements a fetch that always chooses a random proxy from a list
- * of http proxies
- *
-*/
-
-import isoFetch from 'isomorphic-fetch';
-import HttpProxyAgent from 'http-proxy-agent';
-// eslint-disable-next-line import/no-unresolved
-import proxylist from './proxies.json';
-
-import logger from '../core/logger';
-
-function randomProxyURL() {
-  const rand = proxylist[Math.floor(Math.random() * proxylist.length)];
-  logger.info(`choosesn fetch proxy ${rand}`);
-  return rand;
-}
-
-function fetch(url, options = {}) {
-  if (proxylist.length === 0) {
-    return isoFetch(url, options);
-  }
-  const agent = new HttpProxyAgent(randomProxyURL());
-
-  return isoFetch(url, {
-    ...options,
-    agent,
-  });
-}
-
-export default fetch;

src/utils/proxycheck.js

@@ -0,0 +1,159 @@
+/*
+ * check if an ip is a proxy via proxycheck.io
+ */
+import { proxyLogger as logger } from '../core/logger';
+import { PROXYCHECK_KEY } from '../core/config';
+
+const http = require('http');
+
+const pcKeys = PROXYCHECK_KEY.split(',');
+
+/*
+ * queue of ip-checking tasks
+ * [[ip, callbackFunction],...]
+ */
+const ipQueue = [];
+
+let fetching = false;
+
+function reqProxyCheck(ips) {
+  return new Promise((resolve, reject) => {
+    const postData = `ips=${ips.join(',')}`;
+    logger.info(`Request for ${postData}`);
+
+    let path = '/v2/?vpn=1&asn=1';
+    const key = pcKeys[Math.floor(Math.random() * pcKeys.length)];
+    if (key) path += `&key=${key}`;
+
+    const options = {
+      hostname: 'proxycheck.io',
+      port: 80,
+      path,
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/x-www-form-urlencoded',
+        'Content-Length': Buffer.byteLength(postData),
+      },
+    };
+
+    const req = http.request(options, (res) => {
+      if (res.statusCode !== 200) {
+        reject(new Error(`Status not 200: ${res.statusCode}`));
+        return;
+      }
+      res.setEncoding('utf8');
+      const data = [];
+
+      res.on('data', (chunk) => {
+        data.push(chunk);
+      });
+
+      res.on('end', () => {
+        try {
+          const result = JSON.parse(data.join(''));
+          if (result.status !== 'ok') {
+            if (result.status === 'error' && ips.length === 1) {
+              /*
+               * invalid ip, like a link local address
+               * Error is either thrown in the top, when requesting only one ip
+               * or in the ip-part as "error": "No valid.." when multiple
+               * */
+              resolve({
+                [ips[0]]: {
+                  proxy: 'yes',
+                  type: 'Invalid IP',
+                },
+              });
+              return;
+            }
+            if (result.status !== 'warning') {
+              throw new Error(`${key}: ${result.message}`);
+            } else {
+              logger.warn(`Warning: ${key}: ${result.message}`);
+            }
+          }
+          ips.forEach((ip) => {
+            if (result[ip] && result[ip].error) {
+              result[ip] = {
+                proxy: 'yes',
+                type: 'Invalid IP',
+              };
+            }
+          });
+          resolve(result);
+        } catch (err) {
+          reject(err);
+        }
+      });
+    });
+
+    req.on('error', (err) => {
+      reject(err);
+    });
+    req.write(postData);
+    req.end();
+  });
+}
+
+async function checkFromQueue() {
+  if (!ipQueue.length) {
+    fetching = false;
+    return;
+  }
+  fetching = true;
+  const tasks = ipQueue.slice(0, 50);
+  const ips = tasks.map((i) => i[0]);
+  let res = {};
+  try {
+    res = await reqProxyCheck(ips);
+  } catch (err) {
+    logger.error(`Eroor: ${err.message}`);
+  }
+  for (let i = 0; i < tasks.length; i += 1) {
+    const task = tasks[i];
+
+    const pos = ipQueue.indexOf(task);
+    if (~pos) ipQueue.splice(pos, 1);
+
+    const [ip, cb] = task;
+
+    let allowed = true;
+    let status = -2;
+    let pcheck = 'N/A';
+
+    if (res[ip]) {
+      const { proxy, type, city } = res[ip];
+      allowed = proxy === 'no';
+      status = (allowed) ? 0 : 1;
+      pcheck = `${type},${city}`;
+    }
+
+    cb({
+      allowed,
+      status,
+      pcheck,
+    });
+  }
+  setTimeout(checkFromQueue, 10);
+}
+
+/*
+ * check if ip is proxy in queue
+ * @param ip
+ * @return Promise that resolves to
+ * {
+ *   status, 0: no proxy 1: proxy -2: any failure
+ *   allowed, boolean if ip should be allowed to place
+ *   pcheck, string info of proxycheck return (like type and city)
+ * }
+ */
+function checkForProxy(ip) {
+  return new Promise((resolve) => {
+    ipQueue.push([ip, resolve]);
+    if (!fetching) {
+      checkFromQueue();
+    }
+  });
+}
+
+export default checkForProxy;

webpack.config.server.babel.js

@@ -118,7 +118,6 @@ export default ({
     },
 
     externals: [
-      /\/proxies\.json$/,
       /\/canvases\.json$/,
       /\/styleassets\.json$/,
       /\/assets\.json$/,
@@ -143,7 +142,6 @@ export default ({
             to: path.resolve(__dirname, 'dist', 'public'),
           },
           path.resolve(__dirname, 'src', 'canvases.json'),
-          path.resolve(__dirname, 'src', 'proxies.json'),
           {
             from: path.resolve(
               __dirname, 'deployment', 'example-ecosystem.yml'