Skip to content

Node.js based forum software built for the modern web

refactor: closes #13547, process user uploads via batch

Barış Soner Uşaklı1ad97ac19425
diff --git a/src/upgrades/4.3.0/normalize_thumbs_uploads.js b/src/upgrades/4.3.0/normalize_thumbs_uploads.js
index ef12d54f81..3a33daee9f 100644
--- a/src/upgrades/4.3.0/normalize_thumbs_uploads.js
+++ b/src/upgrades/4.3.0/normalize_thumbs_uploads.js
@@ -89,34 +89,36 @@ module.exports = {
const keys = uids.map(uid => `uid:${uid}:uploads`);
const userUploadData = await db.getSortedSetsMembersWithScores(keys);
- const bulkAdd = [];
- const bulkRemove = [];
- const promises = [];
- userUploadData.forEach((userUploads, idx) => {
+ await Promise.all(userUploadData.map(async (allUserUploads, idx) => {
const uid = uids[idx];
- if (Array.isArray(userUploads)) {
- userUploads.forEach((userUpload) => {
- const normalizedPath = normalizePath(userUpload.value);
- if (normalizedPath !== userUpload.value) {
- bulkAdd.push([`uid:${uid}:uploads`, userUpload.score, normalizedPath]);
- promises.push(db.setObjectField(`upload:${md5(normalizedPath)}`, 'uid', uid));
-
- bulkRemove.push([`uid:${uid}:uploads`, userUpload.value]);
- promises.push(db.delete(`upload:${md5(userUpload.value)}`));
- }
+ if (Array.isArray(allUserUploads)) {
+ await batch.processArray(allUserUploads, async (userUploads) => {
+ const bulkAdd = [];
+ const bulkRemove = [];
+ const promises = [];
+ userUploads.forEach((userUpload) => {
+ const normalizedPath = normalizePath(userUpload.value);
+ if (normalizedPath !== userUpload.value) {
+ bulkAdd.push([`uid:${uid}:uploads`, userUpload.score, normalizedPath]);
+ promises.push(db.setObjectField(`upload:${md5(normalizedPath)}`, 'uid', uid));
+
+ bulkRemove.push([`uid:${uid}:uploads`, userUpload.value]);
+ promises.push(db.delete(`upload:${md5(userUpload.value)}`));
+ }
+ });
+ await Promise.all(promises);
+ await db.sortedSetRemoveBulk(bulkRemove);
+ await db.sortedSetAddBulk(bulkAdd);
+ }, {
+ batch: 500,
});
-
}
- });
-
- await Promise.all(promises);
- await db.sortedSetRemoveBulk(bulkRemove);
- await db.sortedSetAddBulk(bulkAdd);
+ }));
progress.incr(uids.length);
}, {
- batch: 500,
+ batch: 100,
});
},
};