Node.js based forum software built for the modern web
perf: update old upgrade scripts to use bulkSet/Add
Barış Soner Uşaklı2b987d09ce48
diff --git a/src/upgrades/1.10.2/upgrade_bans_to_hashes.js b/src/upgrades/1.10.2/upgrade_bans_to_hashes.jsindex 84c7a0ed4d..2bc55b4667 100644--- a/src/upgrades/1.10.2/upgrade_bans_to_hashes.js+++ b/src/upgrades/1.10.2/upgrade_bans_to_hashes.js@@ -11,14 +11,23 @@ module.exports = {method: async function () {const { progress } = this;+ progress.total = await db.sortedSetCard('users:joindate');+await batch.processSortedSet('users:joindate', async (uids) => {- for (const uid of uids) {- progress.incr();- const [bans, reasons, userData] = await Promise.all([- db.getSortedSetRevRangeWithScores(`uid:${uid}:bans`, 0, -1),- db.getSortedSetRevRangeWithScores(`banned:${uid}:reasons`, 0, -1),- db.getObjectFields(`user:${uid}`, ['banned', 'banned:expire', 'joindate', 'lastposttime', 'lastonline']),- ]);+ progress.incr(uids.length);+ const [allUserData, allBans] = await Promise.all([+ db.getObjectsFields(+ uids.map(uid => `user:${uid}`),+ ['banned', 'banned:expire', 'joindate', 'lastposttime', 'lastonline'],+ ),+ db.getSortedSetsMembersWithScores(+ uids.map(uid => `uid:${uid}:bans`)+ ),+ ]);++ await Promise.all(uids.map(async (uid, index) => {+ const userData = allUserData[index];+ const bans = allBans[index] || [];// has no history, but is banned, create plain object with just uid and timestmapif (!bans.length && parseInt(userData.banned, 10)) {@@ -31,6 +40,7 @@ module.exports = {const banKey = `uid:${uid}:ban:${banTimestamp}`;await addBan(uid, banKey, { uid: uid, timestamp: banTimestamp });} else if (bans.length) {+ const reasons = await db.getSortedSetRevRangeWithScores(`banned:${uid}:reasons`, 0, -1);// process ban historyfor (const ban of bans) {const reasonData = reasons.find(reasonData => reasonData.score === ban.score);@@ -46,14 +56,16 @@ module.exports = {await addBan(uid, banKey, data);}}- }+ }));}, {- progress: this.progress,+ batch: 500,});},};async function addBan(uid, key, data) {- await db.setObject(key, data);- await db.sortedSetAdd(`uid:${uid}:bans:timestamp`, data.timestamp, key);+ await Promise.all([+ db.setObject(key, data),+ db.sortedSetAdd(`uid:${uid}:bans:timestamp`, data.timestamp, key),+ ]);}diff --git a/src/upgrades/1.10.2/username_email_history.js b/src/upgrades/1.10.2/username_email_history.jsindex 3b03568a69..8ee4306e3d 100644--- a/src/upgrades/1.10.2/username_email_history.js+++ b/src/upgrades/1.10.2/username_email_history.js@@ -11,27 +11,34 @@ module.exports = {method: async function () {const { progress } = this;+ progress.total = await db.sortedSetCard('users:joindate');+await batch.processSortedSet('users:joindate', async (uids) => {- async function updateHistory(uid, set, fieldName) {- const count = await db.sortedSetCard(set);- if (count <= 0) {- // User has not changed their username/email before, record original username- const userData = await user.getUserFields(uid, [fieldName, 'joindate']);- if (userData && userData.joindate && userData[fieldName]) {- await db.sortedSetAdd(set, userData.joindate, [userData[fieldName], userData.joindate].join(':'));- }- }- }+ const [usernameHistory, emailHistory, userData] = await Promise.all([+ db.sortedSetsCard(uids.map(uid => `user:${uid}:usernames`)),+ db.sortedSetsCard(uids.map(uid => `user:${uid}:emails`)),+ user.getUsersFields(uids, ['uid', 'username', 'email', 'joindate']),+ ]);- await Promise.all(uids.map(async (uid) => {- await Promise.all([- updateHistory(uid, `user:${uid}:usernames`, 'username'),- updateHistory(uid, `user:${uid}:emails`, 'email'),- ]);- progress.incr();- }));+ const bulkAdd = [];+ userData.forEach((data, index) => {+ const thisUsernameHistory = usernameHistory[index];+ const thisEmailHistory = emailHistory[index];+ if (thisUsernameHistory <= 0 && data && data.joindate && data.username) {+ bulkAdd.push([+ `user:${data.uid}:usernames`, data.joindate, [data.username, data.joindate].join(':'),+ ]);+ }+ if (thisEmailHistory <= 0 && data && data.joindate && data.email) {+ bulkAdd.push([+ `user:${data.uid}:emails`, data.joindate, [data.email, data.joindate].join(':'),+ ]);+ }+ });+ await db.sortedSetAddBulk(bulkAdd);+ progress.incr(uids.length);}, {- progress: this.progress,+ batch: 500,});},};diff --git a/src/upgrades/1.12.1/clear_username_email_history.js b/src/upgrades/1.12.1/clear_username_email_history.jsindex 822b500884..0d36534502 100644--- a/src/upgrades/1.12.1/clear_username_email_history.js+++ b/src/upgrades/1.12.1/clear_username_email_history.js@@ -1,45 +1,32 @@'use strict';-const async = require('async');+const db = require('../../database');const user = require('../../user');+const batch = require('../../batch');module.exports = {name: 'Delete username email history for deleted users',timestamp: Date.UTC(2019, 2, 25),- method: function (callback) {+ method: async function () {const { progress } = this;- let currentUid = 1;- db.getObjectField('global', 'nextUid', (err, nextUid) => {- if (err) {- return callback(err);- }- progress.total = nextUid;- async.whilst((next) => {- next(null, currentUid < nextUid);- },- (next) => {- progress.incr();- user.exists(currentUid, (err, exists) => {- if (err) {- return next(err);- }- if (exists) {- currentUid += 1;- return next();- }- db.deleteAll([`user:${currentUid}:usernames`, `user:${currentUid}:emails`], (err) => {- if (err) {- return next(err);- }- currentUid += 1;- next();- });- });- },- (err) => {- callback(err);- });++ progress.total = await db.getObjectField('global', 'nextUid');+ const allUids = [];+ for (let i = 1; i < progress.total; i += 1) {+ allUids.push(i);+ }+ await batch.processArray(allUids, async (uids) => {+ const exists = await user.exists(uids);+ const missingUids = uids.filter((uid, index) => !exists[index]);+ const keysToDelete = [+ ...missingUids.map(uid => `user:${uid}:usernames`),+ ...missingUids.map(uid => `user:${uid}:emails`),+ ];+ await db.deleteAll(keysToDelete);+ progress.incr(uids.length);+ }, {+ batch: 500,});},};diff --git a/src/upgrades/1.12.1/moderation_notes_refactor.js b/src/upgrades/1.12.1/moderation_notes_refactor.jsindex 390273d74a..85118a9a0c 100644--- a/src/upgrades/1.12.1/moderation_notes_refactor.js+++ b/src/upgrades/1.12.1/moderation_notes_refactor.js@@ -12,10 +12,12 @@ module.exports = {const { progress } = this;await batch.processSortedSet('users:joindate', async (uids) => {- await Promise.all(uids.map(async (uid) => {- progress.incr();-- const notes = await db.getSortedSetRevRange(`uid:${uid}:moderation:notes`, 0, -1);+ progress.incr(uids.length);+ const allNotes = await db.getSortedSetsMembers(+ uids.map(uid => `uid:${uid}:moderation:notes`)+ );+ await Promise.all(uids.map(async (uid, index) => {+ const notes = allNotes[index];for (const note of notes) {const noteData = JSON.parse(note);noteData.timestamp = noteData.timestamp || Date.now();diff --git a/src/upgrades/1.13.0/clean_post_topic_hash.js b/src/upgrades/1.13.0/clean_post_topic_hash.jsindex caa6dbd8f6..20cfd78c22 100644--- a/src/upgrades/1.13.0/clean_post_topic_hash.js+++ b/src/upgrades/1.13.0/clean_post_topic_hash.js@@ -8,6 +8,7 @@ module.exports = {timestamp: Date.UTC(2019, 9, 7),method: async function () {const { progress } = this;+ progress.total = await db.sortedSetCard('posts:pid') + await db.sortedSetCard('topics:tid');await cleanPost(progress);await cleanTopic(progress);},@@ -51,7 +52,6 @@ async function cleanPost(progress) {}));}, {batch: 500,- progress: progress,});}@@ -90,6 +90,5 @@ async function cleanTopic(progress) {}));}, {batch: 500,- progress: progress,});}diff --git a/src/upgrades/1.6.2/topics_lastposttime_zset.js b/src/upgrades/1.6.2/topics_lastposttime_zset.jsindex 1dee9feb1a..f299b19c01 100644--- a/src/upgrades/1.6.2/topics_lastposttime_zset.js+++ b/src/upgrades/1.6.2/topics_lastposttime_zset.js@@ -1,29 +1,30 @@'use strict';-const async = require('async');-const db = require('../../database');+const batch = require('../../batch');module.exports = {name: 'New sorted set cid:<cid>:tids:lastposttime',timestamp: Date.UTC(2017, 9, 30),- method: function (callback) {+ method: async function () {const { progress } = this;+ progress.total = await db.sortedSetCard('topics:tid');- require('../../batch').processSortedSet('topics:tid', (tids, next) => {- async.eachSeries(tids, (tid, next) => {- db.getObjectFields(`topic:${tid}`, ['cid', 'timestamp', 'lastposttime'], (err, topicData) => {- if (err || !topicData) {- return next(err);- }- progress.incr();-- const timestamp = topicData.lastposttime || topicData.timestamp || Date.now();- db.sortedSetAdd(`cid:${topicData.cid}:tids:lastposttime`, timestamp, tid, next);- }, next);- }, next);+ await batch.processSortedSet('topics:tid', async (tids) => {+ const topicData = await db.getObjectsFields(+ tids.map(tid => `topic:${tid}`), ['tid', 'cid', 'timestamp', 'lastposttime']+ );+ const bulkAdd = [];+ topicData.forEach((data) => {+ if (data && data.cid && data.tid) {+ const timestamp = data.lastposttime || data.timestamp || Date.now();+ bulkAdd.push([`cid:${data.cid}:tids:lastposttime`, timestamp, data.tid]);+ }+ });+ await db.sortedSetAddBulk(bulkAdd);+ progress.incr(tids.length);}, {- progress: this.progress,- }, callback);+ batch: 500,+ });},};diff --git a/src/upgrades/1.7.1/notification-settings.js b/src/upgrades/1.7.1/notification-settings.jsindex fed592effb..e3693d4f04 100644--- a/src/upgrades/1.7.1/notification-settings.js+++ b/src/upgrades/1.7.1/notification-settings.js@@ -8,23 +8,38 @@ module.exports = {timestamp: Date.UTC(2017, 10, 15),method: async function () {const { progress } = this;-+ progress.total = await db.sortedSetCard('users:joindate');await batch.processSortedSet('users:joindate', async (uids) => {- await Promise.all(uids.map(async (uid) => {- progress.incr();- const userSettings = await db.getObjectFields(`user:${uid}:settings`, ['sendChatNotifications', 'sendPostNotifications']);- if (userSettings) {++ const userSettings = await db.getObjectsFields(+ uids.map(uid => `user:${uid}:settings`),+ ['sendChatNotifications', 'sendPostNotifications'],+ );++ const bulkSet = [];+ userSettings.forEach((settings, index) => {+ const set = {};+ if (settings) {if (parseInt(userSettings.sendChatNotifications, 10) === 1) {- await db.setObjectField(`user:${uid}:settings`, 'notificationType_new-chat', 'notificationemail');+ set['notificationType_new-chat'] = 'notificationemail';}if (parseInt(userSettings.sendPostNotifications, 10) === 1) {- await db.setObjectField(`user:${uid}:settings`, 'notificationType_new-reply', 'notificationemail');+ set['notificationType_new-reply'] = 'notificationemail';+ }+ if (Object.keys(set).length) {+ bulkSet.push([`user:${uids[index]}:settings`, set]);}}- await db.deleteObjectFields(`user:${uid}:settings`, ['sendChatNotifications', 'sendPostNotifications']);- }));+ });+ await db.setObjectBulk(bulkSet);++ await db.deleteObjectFields(+ uids.map(uid => `user:${uid}:settings`),+ ['sendChatNotifications', 'sendPostNotifications'],+ );++ progress.incr(uids.length);}, {- progress: progress,batch: 500,});},diff --git a/src/upgrades/1.7.3/topic_votes.js b/src/upgrades/1.7.3/topic_votes.jsindex 008aaece0a..d5f6b9fd57 100644--- a/src/upgrades/1.7.3/topic_votes.js+++ b/src/upgrades/1.7.3/topic_votes.js@@ -10,32 +10,42 @@ module.exports = {method: async function () {const { progress } = this;- batch.processSortedSet('topics:tid', async (tids) => {- await Promise.all(tids.map(async (tid) => {- progress.incr();- const topicData = await db.getObjectFields(`topic:${tid}`, ['mainPid', 'cid', 'pinned']);- if (topicData.mainPid && topicData.cid) {- const postData = await db.getObject(`post:${topicData.mainPid}`);- if (postData) {- const upvotes = parseInt(postData.upvotes, 10) || 0;- const downvotes = parseInt(postData.downvotes, 10) || 0;- const data = {- upvotes: upvotes,- downvotes: downvotes,- };- const votes = upvotes - downvotes;- await Promise.all([- db.setObject(`topic:${tid}`, data),- db.sortedSetAdd('topics:votes', votes, tid),- ]);- if (parseInt(topicData.pinned, 10) !== 1) {- await db.sortedSetAdd(`cid:${topicData.cid}:tids:votes`, votes, tid);- }+ progress.total = await db.sortedSetCard('topics:tid');++ await batch.processSortedSet('topics:tid', async (tids) => {+ const topicsData = await db.getObjectsFields(+ tids.map(tid => `topic:${tid}`),+ ['tid', 'mainPid', 'cid', 'pinned'],+ );+ const mainPids = topicsData.map(topicData => topicData && topicData.mainPid);+ const mainPosts = await db.getObjects(mainPids.map(pid => `post:${pid}`));++ const bulkSet = [];+ const bulkAdd = [];++ topicsData.forEach((topicData, index) => {+ const mainPost = mainPosts[index];+ if (mainPost && topicData && topicData.cid) {+ const upvotes = parseInt(mainPost.upvotes, 10) || 0;+ const downvotes = parseInt(mainPost.downvotes, 10) || 0;+ const data = {+ upvotes: upvotes,+ downvotes: downvotes,+ };+ const votes = upvotes - downvotes;+ bulkSet.push([`topic:${topicData.tid}`, data]);+ bulkAdd.push(['topics:votes', votes, topicData.tid]);+ if (parseInt(topicData.pinned, 10) !== 1) {+ bulkAdd.push([`cid:${topicData.cid}:tids:votes`, votes, topicData.tid]);}}- }));+ });++ await db.setObjectBulk(bulkSet);+ await db.sortedSetAddBulk('topics:votes', bulkAdd);++ progress.incr(tids.length);}, {- progress: progress,batch: 500,});},diff --git a/src/upgrades/1.8.1/diffs_zset_to_listhash.js b/src/upgrades/1.8.1/diffs_zset_to_listhash.jsindex 370242fba1..277418a79e 100644--- a/src/upgrades/1.8.1/diffs_zset_to_listhash.js+++ b/src/upgrades/1.8.1/diffs_zset_to_listhash.js@@ -1,57 +1,40 @@'use strict';-const async = require('async');const db = require('../../database');const batch = require('../../batch');-module.exports = {name: 'Reformatting post diffs to be stored in lists and hash instead of single zset',timestamp: Date.UTC(2018, 2, 15),- method: function (callback) {+ method: async function () {const { progress } = this;- batch.processSortedSet('posts:pid', (pids, next) => {- async.each(pids, (pid, next) => {- db.getSortedSetRangeWithScores(`post:${pid}:diffs`, 0, -1, (err, diffs) => {- if (err) {- return next(err);- }+ progress.total = await db.sortedSetCard('posts:pid');- if (!diffs || !diffs.length) {- progress.incr();- return next();- }+ await batch.processSortedSet('posts:pid', async (pids) => {+ const postDiffs = await db.getSortedSetsMembersWithScores(+ pids.map(pid => `post:${pid}:diffs`),+ );- // For each diff, push to list- async.each(diffs, (diff, next) => {- async.series([- async.apply(db.delete.bind(db), `post:${pid}:diffs`),- async.apply(db.listPrepend.bind(db), `post:${pid}:diffs`, diff.score),- async.apply(db.setObject.bind(db), `diff:${pid}.${diff.score}`, {- pid: pid,- patch: diff.value,- }),- ], next);- }, (err) => {- if (err) {- return next(err);- }+ await db.deleteAll(pids.map(pid => `post:${pid}:diffs`));- progress.incr();- return next();- });- });- }, (err) => {- if (err) {- // Probably type error, ok to incr and continue- progress.incr();+ await Promise.all(postDiffs.map(async (diffs, index) => {+ if (!diffs || !diffs.length) {+ return;}-- return next();- });+ diffs.reverse();+ const pid = pids[index];+ await db.listAppend(`post:${pid}:diffs`, diffs.map(d => d.score));+ await db.setObjectBulk(+ diffs.map(d => ([`diff:${pid}.${d.score}`, {+ pid: pid,+ patch: d.value,+ }]))+ );+ }));+ progress.incr(pids.length);}, {- progress: progress,- }, callback);+ batch: 500,+ });},};diff --git a/src/upgrades/1.9.0/refresh_post_upload_associations.js b/src/upgrades/1.9.0/refresh_post_upload_associations.jsindex 44acfc079f..6183529641 100644--- a/src/upgrades/1.9.0/refresh_post_upload_associations.js+++ b/src/upgrades/1.9.0/refresh_post_upload_associations.js@@ -1,21 +1,20 @@'use strict';-const async = require('async');+const db = require('../../database');const posts = require('../../posts');+const batch = require('../../batch');module.exports = {name: 'Refresh post-upload associations',timestamp: Date.UTC(2018, 3, 16),- method: function (callback) {+ method: async function () {const { progress } = this;-- require('../../batch').processSortedSet('posts:pid', (pids, next) => {- async.each(pids, (pid, next) => {- posts.uploads.sync(pid, next);- progress.incr();- }, next);+ progress.total = await db.sortedSetCard('posts:pid');+ await batch.processSortedSet('posts:pid', async (pids) => {+ await Promise.all(pids.map(pid => posts.uploads.sync(pid)));+ progress.incr(pids.length);}, {- progress: this.progress,- }, callback);+ batch: 500,+ });},};diff --git a/src/upgrades/2.8.7/fix-email-sorted-sets.js b/src/upgrades/2.8.7/fix-email-sorted-sets.jsindex fcab69a8f4..84919e6774 100644--- a/src/upgrades/2.8.7/fix-email-sorted-sets.js+++ b/src/upgrades/2.8.7/fix-email-sorted-sets.js@@ -26,7 +26,7 @@ module.exports = {}// user has email but doesn't match whats stored in user hash, gh#11259- if (userData.email && userData.email.toLowerCase() !== email.toLowerCase()) {+ if (userData.email && email && String(userData.email).toLowerCase() !== email.toLowerCase()) {bulkRemove.push(['email:uid', email]);bulkRemove.push(['email:sorted', `${email.toLowerCase()}:${uid}`]);}diff --git a/src/upgrades/3.7.0/category-read-by-uid.js b/src/upgrades/3.7.0/category-read-by-uid.jsindex 4ef564f53a..971620613e 100644--- a/src/upgrades/3.7.0/category-read-by-uid.js+++ b/src/upgrades/3.7.0/category-read-by-uid.js@@ -9,6 +9,7 @@ module.exports = {method: async function () {const { progress } = this;const nextCid = await db.getObjectField('global', 'nextCid');+ progress.total = nextCid;const allCids = [];for (let i = 1; i <= nextCid; i++) {allCids.push(i);@@ -18,7 +19,6 @@ module.exports = {progress.incr(cids.length);}, {batch: 500,- progress,});},};