Skip to content

Node.js based forum software built for the modern web

refactor: use opendir instead of loading all files

Barış Soner Uşaklıce9bd0bb6805
diff --git a/src/upgrades/3.8.0/user-upload-folders.js b/src/upgrades/3.8.0/user-upload-folders.js
index fbacbc64c4..dbca210322 100644
--- a/src/upgrades/3.8.0/user-upload-folders.js
+++ b/src/upgrades/3.8.0/user-upload-folders.js
@@ -20,13 +20,28 @@ module.exports = {
await mkdirp(folder);
const userPicRegex = /^\d+-profile/;
- const files = (await fs.promises.readdir(folder, { withFileTypes: true }))
- .filter(item => !item.isDirectory() && String(item.name).match(userPicRegex))
- .map(item => item.name);
+ const dir = await fs.promises.opendir(folder);
- progress.total = files.length;
- await batch.processArray(files, async (files) => {
- progress.incr(files.length);
+ let batchBuffer = [];
+ const BATCH_SIZE = 500;
+
+ for await (const entry of dir) {
+ if (!entry.isDirectory() && userPicRegex.test(entry.name)) {
+ batchBuffer.push(entry.name);
+ }
+ if (batchBuffer.length >= BATCH_SIZE) {
+ await processBatch(batchBuffer);
+ progress.incr(batchBuffer.length);
+ batchBuffer = [];
+ }
+ }
+ if (batchBuffer.length > 0) {
+ await processBatch(batchBuffer);
+ progress.incr(batchBuffer.length);
+ batchBuffer = [];
+ }
+
+ async function processBatch(files) {
await Promise.all(files.map(async (file) => {
const uid = file.split('-')[0];
if (parseInt(uid, 10) > 0) {
@@ -37,9 +52,7 @@ module.exports = {
);
}
}));
- }, {
- batch: 500,
- });
+ }
await batch.processSortedSet('users:joindate', async (uids) => {
progress.incr(uids.length);