Skip to content

Node.js based forum software built for the modern web

refactor: switch to cursor

Barış Soner Uşaklı1c7daf0db9aa
diff --git a/src/database/mongo/main.js b/src/database/mongo/main.js
index 67ff7ac8fa..77f1f1409c 100644
--- a/src/database/mongo/main.js
+++ b/src/database/mongo/main.js
@@ -40,9 +40,19 @@ module.exports = function (module) {
module.scan = async function (params) {
const match = dbHelpers.globToRegex(params.match);
- return await module.client.collection('objects').distinct(
- '_key', { _key: { $regex: new RegExp(match) } }
- );
+ const cursor = await module.client.collection('objects').find({
+ _key: { $regex: new RegExp(match) },
+ }, {
+ projection: { _id: 0, _key: 1 },
+ }).batchSize(params.batch || 1000);
+ const found = new Set();
+ // eslint-disable-next-line no-await-in-loop
+ while (await cursor.hasNext()) {
+ // eslint-disable-next-line no-await-in-loop
+ const item = await cursor.next();
+ found.add(item._key);
+ }
+ return Array.from(found);
};
module.delete = async function (key) {
diff --git a/src/database/postgres/main.js b/src/database/postgres/main.js
index 6c397b8879..163e97fb13 100644
--- a/src/database/postgres/main.js
+++ b/src/database/postgres/main.js
@@ -1,6 +1,7 @@
'use strict';
module.exports = function (module) {
+ const Cursor = require('pg-cursor');
const helpers = require('./helpers');
const dbHelpers = require('../helpers');
@@ -85,17 +86,43 @@ module.exports = function (module) {
};
module.scan = async function (params) {
- const regex = dbHelpers.globToRegex(params.match);
-
- const res = await module.pool.query({
- text: `
- SELECT o."_key"
- FROM "legacy_object_live" o
- WHERE o."_key" ~ $1`,
- values: [regex],
- });
+ const match = dbHelpers.globToRegex(params.match);
+ const batchSize = params.batch || 1000;
+ const found = new Set();
+
+ const client = await module.pool.connect();
+
+ const cursor = client.query(new Cursor(`
+ SELECT "_key"
+ FROM "legacy_object_live"
+ WHERE "_key" ~ $1
+ `, [match]));
+
+ try {
+ const fetchRows = () => {
+ return new Promise((resolve, reject) => {
+ cursor.read(batchSize, (err, rows) => {
+ if (err) return reject(err);
+ resolve(rows);
+ });
+ });
+ };
+
+ let rows;
+ do {
+ // eslint-disable-next-line no-await-in-loop
+ rows = await fetchRows();
+ for (const row of rows) {
+ found.add(row._key);
+ }
+ } while (rows.length > 0);
+ } finally {
+ cursor.close(() => {
+ client.release();
+ });
+ }
- return res.rows.map(r => r._key);
+ return Array.from(found);
};
module.delete = async function (key) {