implement own connect-redis

closes #1

package.json

@@ -12,7 +12,7 @@
     "build": "babel-node scripts/build.js && npm run minify-css",
     "build:dev": "webpack --env extract --config ./webpack.config.server.babel.js && webpack --env extract --env development --config ./webpack.config.client.babel.js && npm run minify-css",
     "minify-css": "babel-node scripts/minifyCss.js",
-    "babel-node": "cd $INIT_CWD && babel-node",
+    "babel-node": "babel-node",
     "lint": "cd $INIT_CWD && eslint --ext .jsx --ext .js",
     "lint:src": "eslint --ext .jsx --ext .js src"
   },
@@ -25,7 +25,6 @@
   "dependencies": {
     "bcrypt": "^5.0.1",
     "compression": "^1.7.3",
-    "connect-redis": "^6.1.3",
     "cookie": "^0.4.1",
     "core-js": "^3.20.2",
     "etag": "^1.8.1",
@@ -56,7 +55,7 @@
     "react-responsive": "^8.2.0",
     "react-stay-scrolled": "^7.4.0",
     "react-toggle-button": "^2.1.0",
-    "redis": "^4.0.6",
+    "redis": "^4.1.0",
     "redux": "^4.1.2",
     "redux-logger": "^3.0.6",
     "redux-persist": "^6.0.0",

src/core/session.js

@@ -2,15 +2,14 @@
  *
  */
 import expressSession from 'express-session';
-import connectRedis from 'connect-redis';
+import RedisStore from '../utils/connectRedis';
 
-import { redisV3 } from '../data/redis';
+import redis from '../data/redis';
 import { HOUR, COOKIE_SESSION_NAME } from './constants';
 import { SESSION_SECRET } from './config';
 
 
-const RedisStore = connectRedis(expressSession);
-export const store = new RedisStore({ client: redisV3 });
+export const store = new RedisStore({ client: redis });
 
 const session = expressSession({
   name: COOKIE_SESSION_NAME,

src/data/models/Event.js

@@ -58,7 +58,6 @@ export async function getEventArea() {
 
 /*
  * restore area effected by last event
- * TODO: notify RedisCanvas
  */
 export async function clearOldEvent() {
   const pos = await getEventArea();

src/data/redis.js

@@ -1,4 +1,7 @@
-/* @flow */
+/*
+ * redis client
+ * REDIS_URL can be url or path to unix socket
+ */
 
 import { createClient } from 'redis';
 
@@ -7,31 +10,18 @@ import { REDIS_URL } from '../core/config';
 const redis = createClient(REDIS_URL.startsWith('redis://')
   ? {
     url: REDIS_URL,
-    // needed for connect-redis
-    legacyMode: true,
   }
   : {
     socket: {
       path: REDIS_URL,
     },
-    // needed for connect-redis
-    legacyMode: true,
   },
 );
 
-export const redisV3 = redis;
-
 export const connect = async () => {
   // eslint-disable-next-line no-console
   console.log(`Connecting to redis server at ${REDIS_URL}`);
   await redis.connect();
 };
 
-/*
- * multi is not in .v4 in legacyMode,
- * might be fixed in the future
- * https://github.com/redis/node-redis/blob/329885b4ae3167d0092e856095b726e2adf89c97/packages/client/lib/client/multi-command.ts
- */
-redis.v4.multi = () => redis.multi().v4;
-
-export default redis.v4;
+export default redis;

src/utils/connectRedis.js

@@ -0,0 +1,239 @@
+/*
+ * forked from
+ * https://github.com/tj/connect-redis
+ *
+ * Copyright(c) 2010-2020 TJ Holowaychuk <tj@vision-media.ca>
+ * MIT Licensed
+ *
+ * and adjusted to work with node-redis v4 without legacyMode: true.
+ *
+ * Since connect-redis needs to support other redis clients (i.e.: ioredis)
+ * as well, there is no chance for this to be merged upstream
+ *
+ * Links:
+ *   express-session readmy with Session Store Implementation section:
+ *     https://github.com/expressjs/session#readme
+ *
+ */
+import { Store } from 'express-session';
+
+// All callbacks should have a noop if none provided for compatibility
+// with the most Redis clients.
+const noop = () => {};
+
+class RedisStore extends Store {
+  constructor(options = {}) {
+    super(options);
+    if (!options.client) {
+      throw new Error('A client must be directly provided to the RedisStore');
+    }
+
+    this.prefix = options.prefix == null ? 'sess:' : options.prefix;
+    this.scanCount = Number(options.scanCount) || 100;
+    this.serializer = options.serializer || JSON;
+    this.client = options.client;
+    this.ttl = options.ttl || 86400; // One day in seconds.
+    this.disableTTL = options.disableTTL || false;
+    this.disableTouch = options.disableTouch || false;
+  }
+
+  get(sid, cb = noop) {
+    const key = this.prefix + sid;
+    this.client
+      .get(key)
+      .then((data) => {
+        if (!data) {
+          cb(null, null);
+          return;
+        }
+        const result = this.serializer.parse(data);
+        cb(null, result);
+      })
+      .catch((err) => {
+        cb(err);
+      });
+  }
+
+  set(sid, sess, cb = noop) {
+    const key = this.prefix + sid;
+    let value;
+    try {
+      value = this.serializer.stringify(sess);
+    } catch (er) {
+      cb(er);
+      return;
+    }
+
+    let options;
+    if (!this.disableTTL) {
+      const ttl = this.getTTL(sess);
+      if (ttl <= 0) {
+        this.desroy(sid, cb);
+        return;
+      }
+      options = {
+        EX: ttl,
+      };
+    }
+
+    this.client
+      .set(key, value, options)
+      .then(() => cb(null))
+      .catch((err) => {
+        cb(err);
+      });
+  }
+
+  touch(sid, sess, cb = noop) {
+    if (this.disableTouch || this.disableTTL) {
+      cb(null, 'OK');
+      return;
+    }
+    const key = this.prefix + sid;
+    this.client
+      .expire(key, this.getTTL(sess))
+      .then((ret) => {
+        if (!ret) {
+          cb(null, 'EXPIRED');
+          return;
+        }
+        cb(null, 'OK');
+      })
+      .catch((err) => {
+        cb(err);
+      });
+  }
+
+  destroy(sid, cb = noop) {
+    const key = this.prefix + sid;
+    this.client
+      .del(key)
+      .then((amount) => {
+        if (amount === 0) {
+          throw new Error('No such session exists');
+        }
+        cb(null);
+      })
+      .catch((err) => {
+        cb(err);
+      });
+  }
+
+  clear(cb = noop) {
+    this.getAllKeys((er, keys) => {
+      if (er) {
+        cb(er);
+        return;
+      }
+      this.client
+        .del(keys)
+        .then(() => cb(null))
+        .catch((err) => {
+          cb(err);
+        });
+    });
+  }
+
+  length(cb = noop) {
+    this.getAllKeys((err, keys) => {
+      if (err) {
+        cb(err);
+        return;
+      }
+      cb(null, keys.length);
+    });
+  }
+
+  ids(cb = noop) {
+    const prefixLen = this.prefix.length;
+
+    this.getAllKeys((err, keys) => {
+      if (err) {
+        cb(err);
+        return;
+      }
+      keys = keys.map((key) => key.substr(prefixLen));
+      cb(null, keys);
+    });
+  }
+
+  all(cb = noop) {
+    const prefixLen = this.prefix.length;
+
+    this.getAllKeys((er, keys) => {
+      if (er) {
+        cb(er);
+        return;
+      }
+      if (keys.length === 0) {
+        cb(null, []);
+        return;
+      }
+
+      this.client
+        .MGET(keys)
+        .then((sessions) => {
+          let errReduce = null;
+          let result;
+          try {
+            result = sessions.reduce((accum, data, index) => {
+              if (!data) return accum;
+              data = this.serializer.parse(data);
+              data.id = keys[index].substr(prefixLen);
+              accum.push(data);
+              return accum;
+            }, []);
+          } catch (e) {
+            errReduce = e;
+          }
+          cb(errReduce, result);
+        })
+        .catch((err) => {
+          cb(err);
+        });
+    });
+  }
+
+  getTTL(sess) {
+    let ttl;
+    if (sess && sess.cookie && sess.cookie.expires) {
+      const ms = Number(new Date(sess.cookie.expires)) - Date.now();
+      ttl = Math.ceil(ms / 1000);
+    } else {
+      ttl = this.ttl;
+    }
+    return ttl;
+  }
+
+  getAllKeys(cb = noop) {
+    const pattern = `${this.prefix}*`;
+    this.scanKeys({}, 0, pattern, this.scanCount, cb);
+  }
+
+  scanKeys(keys, cursor, pattern, count, cb = noop) {
+    this.client
+      .scan(cursor, {
+        MATCH: pattern,
+        COUNT: count,
+      })
+      .then((data) => {
+        const { cursor: nextCursorId, keys: scanKeys } = data;
+        for (let i = 0; i < scanKeys.length; i += 1) {
+          keys[scanKeys[i]] = true;
+        }
+
+        // This can be a string or a number. We check both.
+        if (Number(nextCursorId) !== 0) {
+          this.scanKeys(keys, nextCursorId, pattern, count, cb);
+          return;
+        }
+
+        cb(null, Object.keys(keys));
+      })
+      .catch((err) => {
+        cb(err);
+      });
+  }
+}
+
+export default RedisStore;

utils/README.md

@@ -4,7 +4,7 @@ Note:
 
 - EVERY SCRIPT THAT USES REDIS IS JUST AS REFERENCE (node-redis and keys update and change over time and i am not keeping those up-to-date) 
 - we use blender 2.8
-- js script are executed with babel-node
+- js script are executed with `npm run babel-node utils/[scriptname].js`
 
 ## sphere-protection.blend
 This blend file includes the sphere we use to display the globe with two UV maps, one for protection like it's used on many globe textures of the earth like [here](http://blog.mastermaps.com/2013/09/creating-webgl-earth-with-threejs.html) and [here](http://www.shadedrelief.com/natural3/pages/textures.html) and one for our mercator projection that is the same as on OpenStreetMap, with additional changes for poles.
@@ -60,3 +60,6 @@ just a script that got run once to add the missing tiles in historical view when
 
 ## uploadImage.js
 nodejs script to upload a Image to the canvas without checks and without caring about what was previously there, don't use it for anything other than initially loading a very large image to the canvas.
+
+## testStore.js
+used to test our own [connect-redis](https://github.com/tj/connect-redis) fork in src/utils/connectRedis.js

utils/redisConvert.js

@@ -11,9 +11,6 @@ const oldredis = redis.createClient({ url: oldurl, return_buffers: true });
 const newurl = "redis://localhost:6379";
 const newredis = redis.createClient({ url: newurl, return_buffers: true });
 
-oldredis.connect();
-newredis.connect();
-
 const CHUNK_SIZE = 64; //old chunk size
 const CHUNKS_IN_BASETILE = TILE_SIZE / CHUNK_SIZE;
 const CHUNK_MIN_XY = Math.floor(CANVAS_MIN_XY / CHUNK_SIZE);

utils/testStore.js

@@ -0,0 +1,42 @@
+/*
+ * This script is to test our fork of connect-redis
+ * It gets all sessions, prints how many there are and the sets,
+ * gets and destroys a testsession.
+ * run with:
+ * npm run babel-node utils/testStore.js
+ */
+import { createClient } from 'redis';
+
+import RedisStore from '../src/utils/connectRedis';
+
+const redis = createClient({
+  url: 'redis://localhost:6379',
+});
+
+redis.connect()
+  .then(() => {
+    const store = new RedisStore({ client: redis });
+    console.log('Test getting all sessions');
+    store.all((err, sessions) => {
+      console.log('Error', err);
+      console.log('Amount of sessions', Object.keys(sessions).length);
+      const firstkey = 'test';
+      console.log('Trying to create session', firstkey);
+      store.set(firstkey, {LOL: 'haha'}, (err) => {
+        console.log('Error', err);
+        console.log('Trying to get session', firstkey);
+        store.get(firstkey, (err, session) => {
+          console.log('Error', err);
+          console.log('session', session);
+          store.touch(firstkey, {}, (err, result) => {
+            console.log('Error', err);
+            console.log('touch result', result);
+            console.log('Trying to destroy session', firstkey);
+            store.destroy(firstkey, (err) => {
+              console.log('Error', err);
+            });
+          })
+        });
+      })
+    });
+  });