From 61ddfb0181a12c604bff822e220f4c5e6381f7e7 Mon Sep 17 00:00:00 2001 From: HF Date: Sun, 7 Aug 2022 00:53:14 +0200 Subject: [PATCH] use proxycheck keys instead of proxiesFetch --- README.md | 3 +- package.json | 2 - src/client.js | 3 - src/core/config.js | 7 +- src/core/isAllowed.js | 102 ++++---------------- src/data/redis/isAllowedCache.js | 2 +- src/proxies.json | 1 - src/store/middleware/audio.js | 3 - src/utils/proxiedFetch.js | 33 ------- src/utils/proxycheck.js | 159 +++++++++++++++++++++++++++++++ webpack.config.server.babel.js | 2 - 11 files changed, 184 insertions(+), 133 deletions(-) delete mode 100644 src/proxies.json delete mode 100644 src/utils/proxiedFetch.js create mode 100644 src/utils/proxycheck.js diff --git a/README.md b/README.md index ea990e0..ed7cbcc 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,7 @@ Configuration takes place in the environment variables that are defined in ecosy |-------------------|:--------------------------------------|---------------------------| | ASSET_SERVER | URL for assets | "http://localhost" | | USE_PROXYCHECK | Check users for Proxies | 0 | +| PROXYCHECK_KEY | Key for proxycheck.io | "asfas-xcsc-ewef-sdfsd" | | APISOCKET_KEY | Key for API Socket for SpecialAccess™ | "SDfasife3" | | ADMIN_IDS | Ids of users with Admin rights | "1,12,3" | | CAPTCHA_TIME | time in minutes between captchas | 30 | @@ -115,7 +116,7 @@ Configuration takes place in the environment variables that are defined in ecosy Notes: - HOST / PORT is the host on which the ppfun server is listening. In example: If you have a reverse proxy on the same machine, HOST should still be unset or localhost, because it's where the proxy forwards to. -- to be able to use USE_PROXYCHECK, you have to have an account on proxycheck.io or getipintel or another checker setup and you might set some proxies in`proxies.json` that get used for making proxycheck requests. Look into `src/isProxy.js` to see how things work, but keep in mind that this isn't neccessarily how pixelplanet.fun uses it. +- to be able to use USE_PROXYCHECK effectively, you have to have an account on proxycheck.io and PROXYCHECK_KEY set. - Admins are users with 0cd and access to `Admintools`in their User Menu for image-upload and whatever - You can find out the id of a user by looking into the logs (i.e. `info: {ip} / {id} wants to place 2 in (1701, -8315)`) when he places a pixel or by checking the MySql Users database - pixelplanet uses the unix command sendmail for sending verification and password reset mails. If you don't want to set up your own mail server, look into [ssmtp](https://wiki.archlinux.org/title/SSMTP), which provides a sendmail interface that forwards to other providers like gmail. diff --git a/package.json b/package.json index ae93d87..540fc79 100644 --- a/package.json +++ b/package.json @@ -29,9 +29,7 @@ "etag": "^1.8.1", "express": "^4.17.2", "express-session": "^1.17.2", - "http-proxy-agent": "^5.0.0", "image-q": "^4.0.0", - "isomorphic-fetch": "^3.0.0", "js-file-download": "^0.4.12", "localforage": "^1.10.0", "morgan": "^1.10.0", diff --git a/src/client.js b/src/client.js index 2e9e8e8..f4d90a1 100644 --- a/src/client.js +++ b/src/client.js @@ -2,9 +2,6 @@ * Entrypoint for main client script */ -// eslint-disable-next-line no-unused-vars -import fetch from 'isomorphic-fetch'; // TODO put in the beggining with webpack! - import createKeyPressHandler from './controls/keypress'; import { fetchMe, diff --git a/src/core/config.js b/src/core/config.js index af4999a..fae0a35 100644 --- a/src/core/config.js +++ b/src/core/config.js @@ -1,5 +1,7 @@ -// general config that is also available from client code can be found in -// src/core/constants.js +/* + * general config that is also available from client code can be found in + * src/core/constants.js + */ import path from 'path'; if (process.env.BROWSER) { @@ -27,6 +29,7 @@ export const BACKUP_DIR = process.env.BACKUP_DIR || null; // Proxycheck export const USE_PROXYCHECK = parseInt(process.env.USE_PROXYCHECK, 10) || false; +export const { PROXYCHECK_KEY } = process.env; export const REDIS_URL = process.env.REDIS_URL || 'redis://localhost:6379'; // Database diff --git a/src/core/isAllowed.js b/src/core/isAllowed.js index 538f6ee..20bd9c8 100644 --- a/src/core/isAllowed.js +++ b/src/core/isAllowed.js @@ -2,10 +2,9 @@ * decide if IP is allowed * does proxycheck and check bans and whitelists */ -import fetch from '../utils/proxiedFetch'; - import { getIPv6Subnet } from '../utils/ip'; import whois from '../utils/whois'; +import getProxyCheck from '../utils/proxycheck'; import { IPInfo } from '../data/sql'; import { isIPBanned } from '../data/sql/Ban'; import { isWhitelisted } from '../data/sql/Whitelist'; @@ -17,76 +16,6 @@ import { proxyLogger as logger } from './logger'; import { USE_PROXYCHECK } from './config'; -/* - * check getipintel if IP is proxy - * Use proxiedFetch with random proxies and random mail for it, to not get blacklisted - * @param ip IP to check - * @return true if proxy, false if not - */ -// eslint-disable-next-line no-unused-vars -async function getIPIntel(ip) { - // eslint-disable-next-line max-len - const email = `${Math.random().toString(36).substring(8)}-${Math.random().toString(36).substring(4)}@gmail.com`; - // eslint-disable-next-line max-len - const url = `http://check.getipintel.net/check.php?ip=${ip}&contact=${email}&flags=m`; - logger.info(`fetching getipintel ${url}`); - const response = await fetch(url, { - headers: { - Accept: '*/*', - 'Accept-Language': 'de,en-US;q=0.7,en;q=0.3', - Referer: 'http://check.getipintel.net/', - // eslint-disable-next-line max-len - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36', - }, - }); - if (!response.ok) { - const text = await response.text(); - throw new Error(`getipintel not ok ${response.status}/${text}`); - } - const body = await response.text(); - logger.info('PROXYCHECK %s : %s', ip, body); - // returns tru iff we found 1 in the response and was ok (http code = 200) - const value = parseFloat(body); - return [ - value > 0.995, - `score:${value}`, - ]; -} - -/* - * check proxycheck.io if IP is proxy - * Use proxiedFetch with random proxies - * @param ip IP to check - * @return [ isProxy, info] true if proxy and extra info - */ -async function getProxyCheck(ip) { - const url = `http://proxycheck.io/v2/${ip}?risk=1&vpn=1&asn=1`; - logger.info('fetching proxycheck %s', url); - const response = await fetch(url, { - headers: { - // eslint-disable-next-line max-len - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36', - }, - }); - if (!response.ok) { - const text = await response.text(); - throw new Error(`proxycheck not ok ${response.status}/${text}`); - } - const data = await response.json(); - logger.info('PROXYCHECK', data); - if (!data.status) { - return [ - false, - 'status not ok', - ]; - } - const ipData = data[ip]; - return [ - ipData.proxy === 'yes', - `${ipData.type},${ipData.city}`, - ]; -} - /* * dummy function to include if you don't want any proxycheck */ @@ -94,6 +23,9 @@ async function dummy() { return [false, 'dummy']; } +/* + * save information of ip into database + */ async function saveIPInfo(ip, whoisRet, allowed, info) { try { await IPInfo.upsert({ @@ -117,26 +49,30 @@ async function withoutCache(f, ip) { const ipKey = getIPv6Subnet(ip); let allowed = true; let status = -2; - let pcInfo = null; + let pcheck = null; let whoisRet = null; try { if (await isWhitelisted(ipKey)) { allowed = true; - pcInfo = 'wl'; + pcheck = 'wl'; status = -1; } else if (await isIPBanned(ipKey)) { allowed = false; - pcInfo = 'bl'; + pcheck = 'bl'; status = 2; } else { - [allowed, pcInfo] = await f(ip); - allowed = !allowed; - status = (allowed) ? 0 : 1; + const res = await f(ip); + status = res.status; + allowed = res.allowed; + pcheck = res.pcheck; + if (status === -2) { + throw new Error('Proxycheck request did not return yet'); + } } - whoisRet = await whois(ip) || {}; + whoisRet = await whois(ip); } finally { - await saveIPInfo(ipKey, whoisRet, status, pcInfo); + await saveIPInfo(ipKey, whoisRet || {}, status, pcheck); } return { @@ -152,7 +88,6 @@ async function withoutCache(f, ip) { * @param ip IP to check * @return true if proxy or blacklisted, false if not or whitelisted */ -let lock = 4; const checking = []; async function withCache(f, ip) { if (!ip || ip === '0.0.0.1') { @@ -169,10 +104,8 @@ async function withCache(f, ip) { } // else make asynchronous ipcheck and assume no proxy in the meantime - // use lock to just check three at a time // do not check ip that currently gets checked - if (checking.indexOf(ipKey) === -1 && lock > 0) { - lock -= 1; + if (checking.indexOf(ipKey) === -1) { checking.push(ipKey); withoutCache(f, ip) .then((result) => { @@ -184,7 +117,6 @@ async function withCache(f, ip) { .finally(() => { const pos = checking.indexOf(ipKey); if (~pos) checking.splice(pos, 1); - lock += 1; }); } return { diff --git a/src/data/redis/isAllowedCache.js b/src/data/redis/isAllowedCache.js index f01c807..3ca5f0d 100644 --- a/src/data/redis/isAllowedCache.js +++ b/src/data/redis/isAllowedCache.js @@ -6,7 +6,7 @@ import client from './client'; const PREFIX = 'isal:'; -const CACHE_DURATION = 3 * 24 * 3600; +const CACHE_DURATION = 14 * 24 * 3600; export function cacheAllowed(ip, allowed) { const key = `${PREFIX}:${ip}`; diff --git a/src/proxies.json b/src/proxies.json deleted file mode 100644 index fe51488..0000000 --- a/src/proxies.json +++ /dev/null @@ -1 +0,0 @@ -[] diff --git a/src/store/middleware/audio.js b/src/store/middleware/audio.js index cb8b0c7..db990c1 100644 --- a/src/store/middleware/audio.js +++ b/src/store/middleware/audio.js @@ -95,9 +95,6 @@ export default (store) => (next) => (action) => { } case 'ALERT': { - if (action.alertType !== 'error') { - break; - } const oscillatorNode = context.createOscillator(); const gainNode = context.createGain(); diff --git a/src/utils/proxiedFetch.js b/src/utils/proxiedFetch.js deleted file mode 100644 index 5764479..0000000 --- a/src/utils/proxiedFetch.js +++ /dev/null @@ -1,33 +0,0 @@ -/* - * - * implements a fetch that always chooses a random proxy from a list - * of http proxies - * -*/ - -import isoFetch from 'isomorphic-fetch'; -import HttpProxyAgent from 'http-proxy-agent'; -// eslint-disable-next-line import/no-unresolved -import proxylist from './proxies.json'; - -import logger from '../core/logger'; - -function randomProxyURL() { - const rand = proxylist[Math.floor(Math.random() * proxylist.length)]; - logger.info(`choosesn fetch proxy ${rand}`); - return rand; -} - -function fetch(url, options = {}) { - if (proxylist.length === 0) { - return isoFetch(url, options); - } - const agent = new HttpProxyAgent(randomProxyURL()); - - return isoFetch(url, { - ...options, - agent, - }); -} - -export default fetch; diff --git a/src/utils/proxycheck.js b/src/utils/proxycheck.js new file mode 100644 index 0000000..99a33cc --- /dev/null +++ b/src/utils/proxycheck.js @@ -0,0 +1,159 @@ +/* + * check if an ip is a proxy via proxycheck.io + */ +import { proxyLogger as logger } from '../core/logger'; +import { PROXYCHECK_KEY } from '../core/config'; + +const http = require('http'); + +const pcKeys = PROXYCHECK_KEY.split(','); + +/* + * queue of ip-checking tasks + * [[ip, callbackFunction],...] + */ +const ipQueue = []; + +let fetching = false; + +function reqProxyCheck(ips) { + return new Promise((resolve, reject) => { + const postData = `ips=${ips.join(',')}`; + logger.info(`Request for ${postData}`); + + let path = '/v2/?vpn=1&asn=1'; + const key = pcKeys[Math.floor(Math.random() * pcKeys.length)]; + if (key) path += `&key=${key}`; + + const options = { + hostname: 'proxycheck.io', + port: 80, + path, + method: 'POST', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + 'Content-Length': Buffer.byteLength(postData), + }, + }; + + const req = http.request(options, (res) => { + if (res.statusCode !== 200) { + reject(new Error(`Status not 200: ${res.statusCode}`)); + return; + } + res.setEncoding('utf8'); + const data = []; + + res.on('data', (chunk) => { + data.push(chunk); + }); + + res.on('end', () => { + try { + const result = JSON.parse(data.join('')); + if (result.status !== 'ok') { + if (result.status === 'error' && ips.length === 1) { + /* + * invalid ip, like a link local address + * Error is either thrown in the top, when requesting only one ip + * or in the ip-part as "error": "No valid.." when multiple + * */ + resolve({ + [ips[0]]: { + proxy: 'yes', + type: 'Invalid IP', + }, + }); + return; + } + if (result.status !== 'warning') { + throw new Error(`${key}: ${result.message}`); + } else { + logger.warn(`Warning: ${key}: ${result.message}`); + } + } + ips.forEach((ip) => { + if (result[ip] && result[ip].error) { + result[ip] = { + proxy: 'yes', + type: 'Invalid IP', + }; + } + }); + resolve(result); + } catch (err) { + reject(err); + } + }); + }); + + req.on('error', (err) => { + reject(err); + }); + req.write(postData); + req.end(); + }); +} + +async function checkFromQueue() { + if (!ipQueue.length) { + fetching = false; + return; + } + fetching = true; + const tasks = ipQueue.slice(0, 50); + const ips = tasks.map((i) => i[0]); + let res = {}; + try { + res = await reqProxyCheck(ips); + } catch (err) { + logger.error(`Eroor: ${err.message}`); + } + for (let i = 0; i < tasks.length; i += 1) { + const task = tasks[i]; + + const pos = ipQueue.indexOf(task); + if (~pos) ipQueue.splice(pos, 1); + + const [ip, cb] = task; + + let allowed = true; + let status = -2; + let pcheck = 'N/A'; + + if (res[ip]) { + const { proxy, type, city } = res[ip]; + allowed = proxy === 'no'; + status = (allowed) ? 0 : 1; + pcheck = `${type},${city}`; + } + + cb({ + allowed, + status, + pcheck, + }); + } + setTimeout(checkFromQueue, 10); +} + +/* + * check if ip is proxy in queue + * @param ip + * @return Promise that resolves to + * { + * status, 0: no proxy 1: proxy -2: any failure + * allowed, boolean if ip should be allowed to place + * pcheck, string info of proxycheck return (like type and city) + * } + */ +function checkForProxy(ip) { + return new Promise((resolve) => { + ipQueue.push([ip, resolve]); + if (!fetching) { + checkFromQueue(); + } + }); +} + +export default checkForProxy; diff --git a/webpack.config.server.babel.js b/webpack.config.server.babel.js index 874735e..755f326 100644 --- a/webpack.config.server.babel.js +++ b/webpack.config.server.babel.js @@ -118,7 +118,6 @@ export default ({ }, externals: [ - /\/proxies\.json$/, /\/canvases\.json$/, /\/styleassets\.json$/, /\/assets\.json$/, @@ -143,7 +142,6 @@ export default ({ to: path.resolve(__dirname, 'dist', 'public'), }, path.resolve(__dirname, 'src', 'canvases.json'), - path.resolve(__dirname, 'src', 'proxies.json'), { from: path.resolve( __dirname, 'deployment', 'example-ecosystem.yml'