Node.js based forum software built for the modern web
'use strict';
const os = require('os');
const path = require('path');
const winston = require('winston');
const workerpool = require('workerpool');
const db = require('../database');
const SendPool = {
_activityPub: null,
};
SendPool._pool = workerpool.pool(
path.join(__dirname, 'sendWorker.js'),
{
minWorkers: 2,
maxWorkers: Math.max(2, Math.min(os.availableParallelism(), 64)),
workerType: 'thread',
forkOpts: { silent: true },
},
);
Object.defineProperty(SendPool, 'pool', {
get() {
return this._pool?.stats()?.totalWorkers || 0;
},
});
SendPool.init = function (activityPub) {
if (activityPub) {
SendPool._activityPub = activityPub;
}
};
SendPool.handleResult = async function (queueId, result) {
if (result.success) {
// Success, fire analytics and remove from Redis
try {
await SendPool._activityPub.analytics.send({
type: result.payloadType,
target: result.uri,
});
} catch (e) {
winston.warn(`[activitypub/send] Analytics send failed: ${e.message}`);
}
db.delete(`ap:retry:queue:${queueId}`);
db.sortedSetRemove('ap:retry:queue', queueId);
} else {
try {
await SendPool._activityPub.analytics.sendError({
payload: JSON.parse(result.payload),
uri: result.uri,
error: new Error(result.error),
});
} catch (e) {
winston.warn(`[activitypub/send] Task failed: ${result.error}`);
}
}
};
SendPool.requeueTask = function (task) {
const oneMinute = 1000 * 60;
const maxDelay = 60 * 60 * 1000; // 1 hour
const attempts = task.attempts || 1;
const backoffMs = Math.min(
oneMinute * Math.pow(2, attempts - 1),
maxDelay,
);
task.attempts = attempts + 1;
task.timestamp = Date.now() + backoffMs;
const nextTryOn = task.timestamp;
const retryQueueAdd = [];
const retryQueuedSet = [];
retryQueueAdd.push(['ap:retry:queue', nextTryOn, task.queueId]);
retryQueuedSet.push([`ap:retry:queue:${task.queueId}`, {
queueId: task.queueId,
uri: task.uri,
id: task.id,
type: task.payloadType,
attempts: task.attempts,
timestamp: nextTryOn,
digest: task.digest,
payload: task.payload,
}]);
db.sortedSetAddBulk(retryQueueAdd);
db.setObjectBulk(retryQueuedSet);
};
SendPool.dispatch = function (task) {
// workerpool handles queuing automatically when all workers are busy.
// The 30s timeout on exec kills the worker and rejects the promise,
// which is caught by drainLoop and triggers re-queuing.
return SendPool._pool.exec('send', [{
id: task.id,
uri: task.uri,
payload: task.payload,
digest: task.digest,
key: task.key,
keyId: task.keyId,
}], { timeout: 30000 })
.then(result => result)
.catch(e => ({ success: false, error: e.message }));
};
SendPool.drainLoop = async function () {
if (SendPool._draining) {
return;
}
SendPool._draining = true;
SendPool._inFlight = SendPool._inFlight || new Set();
const MAX_PER_BATCH = 50;
while (SendPool._draining) {
try {
// Get due tasks from Redis sorted set
// eslint-disable-next-line no-await-in-loop
const dueTasks = await db.getSortedSetRangeByScore(
'ap:retry:queue',
0,
MAX_PER_BATCH,
'-inf',
Date.now(),
);
if (dueTasks.length === 0) {
// No tasks due - switch to idle mode
SendPool._draining = false;
setTimeout(() => {
if (SendPool.pool > 0) {
SendPool.drainLoop();
}
}, 10000); // 10-second idle timeout
return;
}
// eslint-disable-next-line no-await-in-loop
const taskDataList = await Promise.all(
dueTasks.map(queueId => db.getObject(`ap:retry:queue:${queueId}`)),
);
// Pair each queueId with its own task data and skip in-flight or
// missing tasks
const validTaskData = [];
for (let i = 0; i < dueTasks.length; i++) {
const queueId = dueTasks[i];
const taskData = taskDataList[i];
if (taskData && !SendPool._inFlight.has(queueId)) {
validTaskData.push({ queueId, taskData });
}
}
// Fetch all key data in parallel
const keyPromises = validTaskData.map(({ taskData }) => (
SendPool._activityPub.getPrivateKey(taskData.type, taskData.id)
));
// eslint-disable-next-line no-await-in-loop
const keyResults = await Promise.all(keyPromises);
// Dispatch each task to workerpool (auto-queues if busy)
for (let i = 0; i < validTaskData.length; i++) {
const { queueId, taskData } = validTaskData[i];
const keyData = keyResults[i];
const task = {
queueId,
uri: taskData.uri,
id: taskData.id,
payloadType: taskData.type,
payload: taskData.payload,
digest: taskData.digest,
key: keyData.key,
keyId: keyData.keyId,
attempts: taskData.attempts || 1,
};
SendPool._inFlight.add(task.queueId);
SendPool.dispatch(task)
.then(async (result) => {
SendPool._inFlight.delete(task.queueId);
if (result.fallback) {
winston.warn(`[activitypub/send] RFC 9421 request failed (${result.fallback}); retrying with draft signature`);
}
if (result.success) {
await SendPool.handleResult(queueId, {
...task,
success: true,
});
} else {
await SendPool.handleResult(queueId, {
...task,
success: false,
error: result.error,
});
SendPool.requeueTask(task);
}
})
.catch(async (e) => {
SendPool._inFlight.delete(task.queueId);
await SendPool.handleResult(queueId, {
...task,
success: false,
error: e.message || 'unknown error',
});
SendPool.requeueTask(task);
});
}
// Yield to event loop between batches to prevent starvation
if (dueTasks.length >= MAX_PER_BATCH) {
setImmediate(() => {
if (SendPool.pool > 0) {
SendPool.drainLoop();
}
});
}
} catch (e) {
winston.error(`[activitypub/send] drainLoop error: ${e.message}`);
SendPool._draining = false;
// Schedule retry after a delay
setTimeout(() => {
if (SendPool.pool > 0) {
SendPool.drainLoop();
}
}, 10000);
}
}
};
SendPool.shutdown = function () {
SendPool._draining = false;
// Graceful shutdown, let workers finish current tasks
SendPool._pool.terminate(false, 10000).catch(() => {
winston.warn('[activitypub/send] Workers did not exit gracefully');
});
// Force kill workers if they haven't exited within the timeout
const forceKillTimer = setTimeout(() => {
SendPool._pool.terminate(true, 10000).catch(() => {
winston.warn('[activitypub/send] Force shutdown failed');
});
}, 12000);
forceKillTimer.unref();
};
module.exports = SendPool;