Skip to content

Node.js based forum software built for the modern web

perf: update old upgrade scripts to use bulkSet/Add

Barış Soner Uşaklı32d0ee480844
diff --git a/src/upgrades/1.10.2/upgrade_bans_to_hashes.js b/src/upgrades/1.10.2/upgrade_bans_to_hashes.js
index 84c7a0ed4d..2bc55b4667 100644
--- a/src/upgrades/1.10.2/upgrade_bans_to_hashes.js
+++ b/src/upgrades/1.10.2/upgrade_bans_to_hashes.js
@@ -11,14 +11,23 @@ module.exports = {
method: async function () {
const { progress } = this;
+ progress.total = await db.sortedSetCard('users:joindate');
+
await batch.processSortedSet('users:joindate', async (uids) => {
- for (const uid of uids) {
- progress.incr();
- const [bans, reasons, userData] = await Promise.all([
- db.getSortedSetRevRangeWithScores(`uid:${uid}:bans`, 0, -1),
- db.getSortedSetRevRangeWithScores(`banned:${uid}:reasons`, 0, -1),
- db.getObjectFields(`user:${uid}`, ['banned', 'banned:expire', 'joindate', 'lastposttime', 'lastonline']),
- ]);
+ progress.incr(uids.length);
+ const [allUserData, allBans] = await Promise.all([
+ db.getObjectsFields(
+ uids.map(uid => `user:${uid}`),
+ ['banned', 'banned:expire', 'joindate', 'lastposttime', 'lastonline'],
+ ),
+ db.getSortedSetsMembersWithScores(
+ uids.map(uid => `uid:${uid}:bans`)
+ ),
+ ]);
+
+ await Promise.all(uids.map(async (uid, index) => {
+ const userData = allUserData[index];
+ const bans = allBans[index] || [];
// has no history, but is banned, create plain object with just uid and timestmap
if (!bans.length && parseInt(userData.banned, 10)) {
@@ -31,6 +40,7 @@ module.exports = {
const banKey = `uid:${uid}:ban:${banTimestamp}`;
await addBan(uid, banKey, { uid: uid, timestamp: banTimestamp });
} else if (bans.length) {
+ const reasons = await db.getSortedSetRevRangeWithScores(`banned:${uid}:reasons`, 0, -1);
// process ban history
for (const ban of bans) {
const reasonData = reasons.find(reasonData => reasonData.score === ban.score);
@@ -46,14 +56,16 @@ module.exports = {
await addBan(uid, banKey, data);
}
}
- }
+ }));
}, {
- progress: this.progress,
+ batch: 500,
});
},
};
async function addBan(uid, key, data) {
- await db.setObject(key, data);
- await db.sortedSetAdd(`uid:${uid}:bans:timestamp`, data.timestamp, key);
+ await Promise.all([
+ db.setObject(key, data),
+ db.sortedSetAdd(`uid:${uid}:bans:timestamp`, data.timestamp, key),
+ ]);
}
diff --git a/src/upgrades/1.10.2/username_email_history.js b/src/upgrades/1.10.2/username_email_history.js
index 3b03568a69..8ee4306e3d 100644
--- a/src/upgrades/1.10.2/username_email_history.js
+++ b/src/upgrades/1.10.2/username_email_history.js
@@ -11,27 +11,34 @@ module.exports = {
method: async function () {
const { progress } = this;
+ progress.total = await db.sortedSetCard('users:joindate');
+
await batch.processSortedSet('users:joindate', async (uids) => {
- async function updateHistory(uid, set, fieldName) {
- const count = await db.sortedSetCard(set);
- if (count <= 0) {
- // User has not changed their username/email before, record original username
- const userData = await user.getUserFields(uid, [fieldName, 'joindate']);
- if (userData && userData.joindate && userData[fieldName]) {
- await db.sortedSetAdd(set, userData.joindate, [userData[fieldName], userData.joindate].join(':'));
- }
- }
- }
+ const [usernameHistory, emailHistory, userData] = await Promise.all([
+ db.sortedSetsCard(uids.map(uid => `user:${uid}:usernames`)),
+ db.sortedSetsCard(uids.map(uid => `user:${uid}:emails`)),
+ user.getUsersFields(uids, ['uid', 'username', 'email', 'joindate']),
+ ]);
- await Promise.all(uids.map(async (uid) => {
- await Promise.all([
- updateHistory(uid, `user:${uid}:usernames`, 'username'),
- updateHistory(uid, `user:${uid}:emails`, 'email'),
- ]);
- progress.incr();
- }));
+ const bulkAdd = [];
+ userData.forEach((data, index) => {
+ const thisUsernameHistory = usernameHistory[index];
+ const thisEmailHistory = emailHistory[index];
+ if (thisUsernameHistory <= 0 && data && data.joindate && data.username) {
+ bulkAdd.push([
+ `user:${data.uid}:usernames`, data.joindate, [data.username, data.joindate].join(':'),
+ ]);
+ }
+ if (thisEmailHistory <= 0 && data && data.joindate && data.email) {
+ bulkAdd.push([
+ `user:${data.uid}:emails`, data.joindate, [data.email, data.joindate].join(':'),
+ ]);
+ }
+ });
+ await db.sortedSetAddBulk(bulkAdd);
+ progress.incr(uids.length);
}, {
- progress: this.progress,
+ batch: 500,
});
},
};
diff --git a/src/upgrades/1.12.1/clear_username_email_history.js b/src/upgrades/1.12.1/clear_username_email_history.js
index 822b500884..0d36534502 100644
--- a/src/upgrades/1.12.1/clear_username_email_history.js
+++ b/src/upgrades/1.12.1/clear_username_email_history.js
@@ -1,45 +1,32 @@
'use strict';
-const async = require('async');
+
const db = require('../../database');
const user = require('../../user');
+const batch = require('../../batch');
module.exports = {
name: 'Delete username email history for deleted users',
timestamp: Date.UTC(2019, 2, 25),
- method: function (callback) {
+ method: async function () {
const { progress } = this;
- let currentUid = 1;
- db.getObjectField('global', 'nextUid', (err, nextUid) => {
- if (err) {
- return callback(err);
- }
- progress.total = nextUid;
- async.whilst((next) => {
- next(null, currentUid < nextUid);
- },
- (next) => {
- progress.incr();
- user.exists(currentUid, (err, exists) => {
- if (err) {
- return next(err);
- }
- if (exists) {
- currentUid += 1;
- return next();
- }
- db.deleteAll([`user:${currentUid}:usernames`, `user:${currentUid}:emails`], (err) => {
- if (err) {
- return next(err);
- }
- currentUid += 1;
- next();
- });
- });
- },
- (err) => {
- callback(err);
- });
+
+ progress.total = await db.getObjectField('global', 'nextUid');
+ const allUids = [];
+ for (let i = 1; i < progress.total; i += 1) {
+ allUids.push(i);
+ }
+ await batch.processArray(allUids, async (uids) => {
+ const exists = await user.exists(uids);
+ const missingUids = uids.filter((uid, index) => !exists[index]);
+ const keysToDelete = [
+ ...missingUids.map(uid => `user:${uid}:usernames`),
+ ...missingUids.map(uid => `user:${uid}:emails`),
+ ];
+ await db.deleteAll(keysToDelete);
+ progress.incr(uids.length);
+ }, {
+ batch: 500,
});
},
};
diff --git a/src/upgrades/1.12.1/moderation_notes_refactor.js b/src/upgrades/1.12.1/moderation_notes_refactor.js
index 390273d74a..85118a9a0c 100644
--- a/src/upgrades/1.12.1/moderation_notes_refactor.js
+++ b/src/upgrades/1.12.1/moderation_notes_refactor.js
@@ -12,10 +12,12 @@ module.exports = {
const { progress } = this;
await batch.processSortedSet('users:joindate', async (uids) => {
- await Promise.all(uids.map(async (uid) => {
- progress.incr();
-
- const notes = await db.getSortedSetRevRange(`uid:${uid}:moderation:notes`, 0, -1);
+ progress.incr(uids.length);
+ const allNotes = await db.getSortedSetsMembers(
+ uids.map(uid => `uid:${uid}:moderation:notes`)
+ );
+ await Promise.all(uids.map(async (uid, index) => {
+ const notes = allNotes[index];
for (const note of notes) {
const noteData = JSON.parse(note);
noteData.timestamp = noteData.timestamp || Date.now();
diff --git a/src/upgrades/1.13.0/clean_post_topic_hash.js b/src/upgrades/1.13.0/clean_post_topic_hash.js
index caa6dbd8f6..20cfd78c22 100644
--- a/src/upgrades/1.13.0/clean_post_topic_hash.js
+++ b/src/upgrades/1.13.0/clean_post_topic_hash.js
@@ -8,6 +8,7 @@ module.exports = {
timestamp: Date.UTC(2019, 9, 7),
method: async function () {
const { progress } = this;
+ progress.total = await db.sortedSetCard('posts:pid') + await db.sortedSetCard('topics:tid');
await cleanPost(progress);
await cleanTopic(progress);
},
@@ -51,7 +52,6 @@ async function cleanPost(progress) {
}));
}, {
batch: 500,
- progress: progress,
});
}
@@ -90,6 +90,5 @@ async function cleanTopic(progress) {
}));
}, {
batch: 500,
- progress: progress,
});
}
diff --git a/src/upgrades/1.6.2/topics_lastposttime_zset.js b/src/upgrades/1.6.2/topics_lastposttime_zset.js
index 1dee9feb1a..f299b19c01 100644
--- a/src/upgrades/1.6.2/topics_lastposttime_zset.js
+++ b/src/upgrades/1.6.2/topics_lastposttime_zset.js
@@ -1,29 +1,30 @@
'use strict';
-const async = require('async');
-
const db = require('../../database');
+const batch = require('../../batch');
module.exports = {
name: 'New sorted set cid:<cid>:tids:lastposttime',
timestamp: Date.UTC(2017, 9, 30),
- method: function (callback) {
+ method: async function () {
const { progress } = this;
+ progress.total = await db.sortedSetCard('topics:tid');
- require('../../batch').processSortedSet('topics:tid', (tids, next) => {
- async.eachSeries(tids, (tid, next) => {
- db.getObjectFields(`topic:${tid}`, ['cid', 'timestamp', 'lastposttime'], (err, topicData) => {
- if (err || !topicData) {
- return next(err);
- }
- progress.incr();
-
- const timestamp = topicData.lastposttime || topicData.timestamp || Date.now();
- db.sortedSetAdd(`cid:${topicData.cid}:tids:lastposttime`, timestamp, tid, next);
- }, next);
- }, next);
+ await batch.processSortedSet('topics:tid', async (tids) => {
+ const topicData = await db.getObjectsFields(
+ tids.map(tid => `topic:${tid}`), ['tid', 'cid', 'timestamp', 'lastposttime']
+ );
+ const bulkAdd = [];
+ topicData.forEach((data) => {
+ if (data && data.cid && data.tid) {
+ const timestamp = data.lastposttime || data.timestamp || Date.now();
+ bulkAdd.push([`cid:${data.cid}:tids:lastposttime`, timestamp, data.tid]);
+ }
+ });
+ await db.sortedSetAddBulk(bulkAdd);
+ progress.incr(tids.length);
}, {
- progress: this.progress,
- }, callback);
+ batch: 500,
+ });
},
};
diff --git a/src/upgrades/1.7.1/notification-settings.js b/src/upgrades/1.7.1/notification-settings.js
index fed592effb..e3693d4f04 100644
--- a/src/upgrades/1.7.1/notification-settings.js
+++ b/src/upgrades/1.7.1/notification-settings.js
@@ -8,23 +8,38 @@ module.exports = {
timestamp: Date.UTC(2017, 10, 15),
method: async function () {
const { progress } = this;
-
+ progress.total = await db.sortedSetCard('users:joindate');
await batch.processSortedSet('users:joindate', async (uids) => {
- await Promise.all(uids.map(async (uid) => {
- progress.incr();
- const userSettings = await db.getObjectFields(`user:${uid}:settings`, ['sendChatNotifications', 'sendPostNotifications']);
- if (userSettings) {
+
+ const userSettings = await db.getObjectsFields(
+ uids.map(uid => `user:${uid}:settings`),
+ ['sendChatNotifications', 'sendPostNotifications'],
+ );
+
+ const bulkSet = [];
+ userSettings.forEach((settings, index) => {
+ const set = {};
+ if (settings) {
if (parseInt(userSettings.sendChatNotifications, 10) === 1) {
- await db.setObjectField(`user:${uid}:settings`, 'notificationType_new-chat', 'notificationemail');
+ set['notificationType_new-chat'] = 'notificationemail';
}
if (parseInt(userSettings.sendPostNotifications, 10) === 1) {
- await db.setObjectField(`user:${uid}:settings`, 'notificationType_new-reply', 'notificationemail');
+ set['notificationType_new-reply'] = 'notificationemail';
+ }
+ if (Object.keys(set).length) {
+ bulkSet.push([`user:${uids[index]}:settings`, set]);
}
}
- await db.deleteObjectFields(`user:${uid}:settings`, ['sendChatNotifications', 'sendPostNotifications']);
- }));
+ });
+ await db.setObjectBulk(bulkSet);
+
+ await db.deleteObjectFields(
+ uids.map(uid => `user:${uid}:settings`),
+ ['sendChatNotifications', 'sendPostNotifications'],
+ );
+
+ progress.incr(uids.length);
}, {
- progress: progress,
batch: 500,
});
},
diff --git a/src/upgrades/1.7.3/topic_votes.js b/src/upgrades/1.7.3/topic_votes.js
index 008aaece0a..d5f6b9fd57 100644
--- a/src/upgrades/1.7.3/topic_votes.js
+++ b/src/upgrades/1.7.3/topic_votes.js
@@ -10,32 +10,42 @@ module.exports = {
method: async function () {
const { progress } = this;
- batch.processSortedSet('topics:tid', async (tids) => {
- await Promise.all(tids.map(async (tid) => {
- progress.incr();
- const topicData = await db.getObjectFields(`topic:${tid}`, ['mainPid', 'cid', 'pinned']);
- if (topicData.mainPid && topicData.cid) {
- const postData = await db.getObject(`post:${topicData.mainPid}`);
- if (postData) {
- const upvotes = parseInt(postData.upvotes, 10) || 0;
- const downvotes = parseInt(postData.downvotes, 10) || 0;
- const data = {
- upvotes: upvotes,
- downvotes: downvotes,
- };
- const votes = upvotes - downvotes;
- await Promise.all([
- db.setObject(`topic:${tid}`, data),
- db.sortedSetAdd('topics:votes', votes, tid),
- ]);
- if (parseInt(topicData.pinned, 10) !== 1) {
- await db.sortedSetAdd(`cid:${topicData.cid}:tids:votes`, votes, tid);
- }
+ progress.total = await db.sortedSetCard('topics:tid');
+
+ await batch.processSortedSet('topics:tid', async (tids) => {
+ const topicsData = await db.getObjectsFields(
+ tids.map(tid => `topic:${tid}`),
+ ['tid', 'mainPid', 'cid', 'pinned'],
+ );
+ const mainPids = topicsData.map(topicData => topicData && topicData.mainPid);
+ const mainPosts = await db.getObjects(mainPids.map(pid => `post:${pid}`));
+
+ const bulkSet = [];
+ const bulkAdd = [];
+
+ topicsData.forEach((topicData, index) => {
+ const mainPost = mainPosts[index];
+ if (mainPost && topicData && topicData.cid) {
+ const upvotes = parseInt(mainPost.upvotes, 10) || 0;
+ const downvotes = parseInt(mainPost.downvotes, 10) || 0;
+ const data = {
+ upvotes: upvotes,
+ downvotes: downvotes,
+ };
+ const votes = upvotes - downvotes;
+ bulkSet.push([`topic:${topicData.tid}`, data]);
+ bulkAdd.push(['topics:votes', votes, topicData.tid]);
+ if (parseInt(topicData.pinned, 10) !== 1) {
+ bulkAdd.push([`cid:${topicData.cid}:tids:votes`, votes, topicData.tid]);
}
}
- }));
+ });
+
+ await db.setObjectBulk(bulkSet);
+ await db.sortedSetAddBulk('topics:votes', bulkAdd);
+
+ progress.incr(tids.length);
}, {
- progress: progress,
batch: 500,
});
},
diff --git a/src/upgrades/1.8.1/diffs_zset_to_listhash.js b/src/upgrades/1.8.1/diffs_zset_to_listhash.js
index 370242fba1..277418a79e 100644
--- a/src/upgrades/1.8.1/diffs_zset_to_listhash.js
+++ b/src/upgrades/1.8.1/diffs_zset_to_listhash.js
@@ -1,57 +1,40 @@
'use strict';
-const async = require('async');
const db = require('../../database');
const batch = require('../../batch');
-
module.exports = {
name: 'Reformatting post diffs to be stored in lists and hash instead of single zset',
timestamp: Date.UTC(2018, 2, 15),
- method: function (callback) {
+ method: async function () {
const { progress } = this;
- batch.processSortedSet('posts:pid', (pids, next) => {
- async.each(pids, (pid, next) => {
- db.getSortedSetRangeWithScores(`post:${pid}:diffs`, 0, -1, (err, diffs) => {
- if (err) {
- return next(err);
- }
+ progress.total = await db.sortedSetCard('posts:pid');
- if (!diffs || !diffs.length) {
- progress.incr();
- return next();
- }
+ await batch.processSortedSet('posts:pid', async (pids) => {
+ const postDiffs = await db.getSortedSetsMembersWithScores(
+ pids.map(pid => `post:${pid}:diffs`),
+ );
- // For each diff, push to list
- async.each(diffs, (diff, next) => {
- async.series([
- async.apply(db.delete.bind(db), `post:${pid}:diffs`),
- async.apply(db.listPrepend.bind(db), `post:${pid}:diffs`, diff.score),
- async.apply(db.setObject.bind(db), `diff:${pid}.${diff.score}`, {
- pid: pid,
- patch: diff.value,
- }),
- ], next);
- }, (err) => {
- if (err) {
- return next(err);
- }
+ await db.deleteAll(pids.map(pid => `post:${pid}:diffs`));
- progress.incr();
- return next();
- });
- });
- }, (err) => {
- if (err) {
- // Probably type error, ok to incr and continue
- progress.incr();
+ await Promise.all(postDiffs.map(async (diffs, index) => {
+ if (!diffs || !diffs.length) {
+ return;
}
-
- return next();
- });
+ diffs.reverse();
+ const pid = pids[index];
+ await db.listAppend(`post:${pid}:diffs`, diffs.map(d => d.score));
+ await db.setObjectBulk(
+ diffs.map(d => ([`diff:${pid}.${d.score}`, {
+ pid: pid,
+ patch: d.value,
+ }]))
+ );
+ }));
+ progress.incr(pids.length);
}, {
- progress: progress,
- }, callback);
+ batch: 500,
+ });
},
};
diff --git a/src/upgrades/1.9.0/refresh_post_upload_associations.js b/src/upgrades/1.9.0/refresh_post_upload_associations.js
index 44acfc079f..6183529641 100644
--- a/src/upgrades/1.9.0/refresh_post_upload_associations.js
+++ b/src/upgrades/1.9.0/refresh_post_upload_associations.js
@@ -1,21 +1,20 @@
'use strict';
-const async = require('async');
+const db = require('../../database');
const posts = require('../../posts');
+const batch = require('../../batch');
module.exports = {
name: 'Refresh post-upload associations',
timestamp: Date.UTC(2018, 3, 16),
- method: function (callback) {
+ method: async function () {
const { progress } = this;
-
- require('../../batch').processSortedSet('posts:pid', (pids, next) => {
- async.each(pids, (pid, next) => {
- posts.uploads.sync(pid, next);
- progress.incr();
- }, next);
+ progress.total = await db.sortedSetCard('posts:pid');
+ await batch.processSortedSet('posts:pid', async (pids) => {
+ await Promise.all(pids.map(pid => posts.uploads.sync(pid)));
+ progress.incr(pids.length);
}, {
- progress: this.progress,
- }, callback);
+ batch: 500,
+ });
},
};
diff --git a/src/upgrades/2.8.7/fix-email-sorted-sets.js b/src/upgrades/2.8.7/fix-email-sorted-sets.js
index fcab69a8f4..84919e6774 100644
--- a/src/upgrades/2.8.7/fix-email-sorted-sets.js
+++ b/src/upgrades/2.8.7/fix-email-sorted-sets.js
@@ -26,7 +26,7 @@ module.exports = {
}
// user has email but doesn't match whats stored in user hash, gh#11259
- if (userData.email && userData.email.toLowerCase() !== email.toLowerCase()) {
+ if (userData.email && email && String(userData.email).toLowerCase() !== email.toLowerCase()) {
bulkRemove.push(['email:uid', email]);
bulkRemove.push(['email:sorted', `${email.toLowerCase()}:${uid}`]);
}
diff --git a/src/upgrades/3.7.0/category-read-by-uid.js b/src/upgrades/3.7.0/category-read-by-uid.js
index 4ef564f53a..971620613e 100644
--- a/src/upgrades/3.7.0/category-read-by-uid.js
+++ b/src/upgrades/3.7.0/category-read-by-uid.js
@@ -9,6 +9,7 @@ module.exports = {
method: async function () {
const { progress } = this;
const nextCid = await db.getObjectField('global', 'nextCid');
+ progress.total = nextCid;
const allCids = [];
for (let i = 1; i <= nextCid; i++) {
allCids.push(i);
@@ -18,7 +19,6 @@ module.exports = {
progress.incr(cids.length);
}, {
batch: 500,
- progress,
});
},
};