diff --git a/src/data/User.js b/src/data/User.js index 7771521..74b51f2 100644 --- a/src/data/User.js +++ b/src/data/User.js @@ -180,9 +180,19 @@ class User { } incrementPixelcount(amount = 1) { - if (this.regUser) { - incrementPixelcount(this.regUser, amount); + if (!this.id) { + return; } + /* + * incrementation gets queued to be processed + * in batches + */ + if (!this.queuedPxlIncrement) { + this.queuedPxlIncrement = amount; + incrementPixelcount(this); + return; + } + this.queuedPxlIncrement += amount; } async getTotalPixels() { diff --git a/src/data/sql/RegUser.js b/src/data/sql/RegUser.js index b0f9158..e367a87 100644 --- a/src/data/sql/RegUser.js +++ b/src/data/sql/RegUser.js @@ -205,44 +205,75 @@ export async function getNamesToIds(ids) { } /* - * increment user pixelcount in a batched transaction + * increment user pixelcount in batches sequentially + * Queue directly accesses queuedPxlIncrement in user object */ const incrementQueue = []; let pushLoop = null; +let lastLog = 0; const incrementLoop = async () => { if (!incrementQueue.length) { pushLoop = null; return; } try { - await sequelize.transaction(async (t) => { - while (incrementQueue.length) { - const [model, amount] = incrementQueue.pop(); - // eslint-disable-next-line no-await-in-loop - await model.increment( - ['totalPixels', 'dailyTotalPixels'], - { by: amount, transaction: t }, - ); - } - return true; + incrementQueue.sort((a, b) => { + const aa = a.queuedPxlIncrement; + const ba = b.queuedPxlIncrement; + if (aa < ba) return -1; + if (aa > ba) return 1; + return 0; }); + let cnt = incrementQueue.length; + let user = incrementQueue.pop(); + while (cnt) { + cnt -= 1; + const amount = user.queuedPxlIncrement; + user.queuedPxlIncrement = 0; + const ids = [user.id]; + while (cnt) { + cnt -= 1; + user = incrementQueue.pop(); + if (user.queuedPxlIncrement !== amount) { + break; + } + user.queuedPxlIncrement = 0; + ids.push(user.id); + } + // eslint-disable-next-line no-await-in-loop + await RegUser.increment(['totalPixels', 'dailyTotalPixels'], { + by: amount, + where: { + id: ids, + }, + silent: true, + raw: true, + // TODO: remove this after testing + logging: (msg) => { + const now = Date.now(); + if (now - 5000 > lastLog) { + lastLog = now; + logger.warn(msg); + } + }, + }); + } } catch (err) { - logger.warn(`Error on batched incrementing pixelcounts: ${err.message}`); + logger.warn(`Error on pixel increment: ${err.message}`); + incrementQueue.forEach((q) => { + if (q) q.queuedPxlIncrement = 0; + }); + incrementQueue.length = 0; } - pushLoop = setTimeout(incrementLoop, 50); + pushLoop = setTimeout(incrementLoop, 250); }; // TODO remove this after testing setInterval(() => { // eslint-disable-next-line no-console console.log('INCREMENTATION QUEUE SIZE', incrementQueue.length, pushLoop); }, 300000); -export async function incrementPixelcount(model, amount) { - const exists = incrementQueue.find((q) => q[0] === model); - if (exists) { - exists[1] += amount; - } else { - incrementQueue.push([model, amount]); - } +export async function incrementPixelcount(user) { + incrementQueue.push(user); if (!pushLoop) { pushLoop = setTimeout(incrementLoop, 0); }