Project NOMAD is an offline-first knowledge and education server. Wikipedia, thousands of books, courses, maps, and optional local AI, all running on hardware you own with no internet required.
TypeScript
/**
* Per-queue BullMQ stall-recovery options.
*
* The pure, dependency-free core of `getStallOptionsForQueue` in
* `commands/queue/work.ts` (mirrors `decideScanAction` in
* `kb_ingest_decision.ts`).
*
* Two questions decide a queue's entry, and they pull in opposite directions:
*
* 1. **How long does one job legitimately hold the lock?** If that can exceed
* `lockDuration`, BullMQ calls a perfectly healthy job stalled.
* 2. **Is re-running a job safe?** On a stall BullMQ increments the job's `stc`
* counter and re-queues it, failing it only once the counter passes
* `maxStalledCount` (`moveStalledJobsToWait`). It does **not** check whether
* the original is still running. For an idempotent job that recovery is what
* we want; for a job that writes non-idempotent data it is worse than the
* failure it is preventing.
*
* Note that `maxStalledCount: 0` is meaningful and is not the same as leaving
* it unset: the comparison is `stalledCount > maxStalledCount` after an
* increment, so 0 fails the job on the FIRST stall rather than re-queueing it,
* while the BullMQ default of 1 re-queues once.
*/
export const DEFAULT_LOCK_DURATION = 300_000
export const LONG_LOCK_DURATION = 1_800_000
export interface StallOptions {
lockDuration: number
/** Left undefined to keep BullMQ's default of 1. */
maxStalledCount?: number
}
export interface StallQueueNames {
/** ZIM/document embedding — a self-continuing batch chain. */
embedFile: string
/** Content downloads — resumable. */
download: string
drugDownload: string
drugIngest: string
}
export function stallOptionsForQueue(queueName: string, queues: StallQueueNames): StallOptions {
// Long single streams (a ~150 MB resumable HTTP pull, then an unzip +
// JSON-stream ingest at concurrency 1). Re-running a part is safe: the
// download resumes and the ingest upserts by key.
if (queueName === queues.drugDownload || queueName === queues.drugIngest) {
return { lockDuration: LONG_LOCK_DURATION, maxStalledCount: 3 }
}
// NEVER auto-recover a stalled job on this queue (#1269). Embedding is a
// self-continuing chain, so re-queueing while the original is alive produces
// two chains walking the same file, both writing to Qdrant with freshly
// minted point ids. A failed chain is visible and re-indexable; a forked one
// is silent and corrupts the index.
if (queueName === queues.embedFile) {
return { lockDuration: LONG_LOCK_DURATION, maxStalledCount: 0 }
}
// The opposite call to embeddings (#1203). A download is resumable and
// re-running one is harmless, but the default maxStalledCount of 1 fails the
// job on a second stall and bypasses its own `attempts: 10` retry policy.
if (queueName === queues.download) {
return { lockDuration: LONG_LOCK_DURATION, maxStalledCount: 3 }
}
return { lockDuration: DEFAULT_LOCK_DURATION }
}