Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(nuq): rebuild queue + concurrency limiting on FoundationDB (core engine) (#3758)

Gergő Móriczb9eb2faecee6
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.yml
index 2002b926b..6fd812c34 100644
--- a/.github/workflows/test-server.yml
+++ b/.github/workflows/test-server.yml
@@ -19,6 +19,15 @@ jobs:
proxy: ['proxy', 'no-proxy'] # proxy / no-proxy run different tests, keep both
search: ['searxng'] # disabled google for now, should be fine with just Searxng
ai: ['openai'] # AI only should be fine, as it simply adds tests, if non-AI fails, AI will fail.
+ nuq: ['postgres']
+ include:
+ # Exercise the self-hosted API/harness path through the FDB NuQ router
+ # without doubling the expensive Playwright/proxy matrix.
+ - engine: fetch
+ proxy: no-proxy
+ search: searxng
+ ai: openai
+ nuq: fdb
# legacy matrix:
# engine: ["playwright", "fetch"]
@@ -48,6 +57,9 @@ jobs:
PROXY_PASSWORD: ${{ matrix.proxy == 'proxy' && secrets.PROXY_PASSWORD || '' }}
NUQ_DATABASE_URL: postgres://postgres:postgres@localhost:5432/postgres
NUQ_RABBITMQ_URL: amqp://localhost:5672
+ NUQ_BACKEND: ${{ matrix.nuq == 'fdb' && 'fdb' || '' }}
+ FDB_CLUSTER_FILE: ${{ matrix.nuq == 'fdb' && '/tmp/fdb.cluster' || '' }}
+ FDB_VERSION: 7.3.63
USE_GO_MARKDOWN_PARSER: true
ALLOW_LOCAL_WEBHOOKS: true
FIRECRAWL_LOG_TO_FILE: true
@@ -215,10 +227,29 @@ jobs:
PORT: 3003
- name: Run Docker Postgres
+ if: matrix.nuq == 'postgres'
run: |
docker build -t firecrawl/nuq-postgres:latest ./apps/nuq-postgres
docker run -d -p 5432:5432 -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=postgres --name postgres firecrawl/nuq-postgres:latest
+ - name: Install FoundationDB client library
+ if: matrix.nuq == 'fdb'
+ run: |
+ curl -fsSL -o /tmp/fdb-clients.deb \
+ "https://github.com/apple/foundationdb/releases/download/${FDB_VERSION}/foundationdb-clients_${FDB_VERSION}-1_amd64.deb"
+ sudo dpkg -i /tmp/fdb-clients.deb
+
+ - name: Start FoundationDB
+ if: matrix.nuq == 'fdb'
+ run: |
+ docker run -d --name fdb -p 4500:4500 -e FDB_NETWORKING_MODE=host foundationdb/foundationdb:${FDB_VERSION}
+ for i in $(seq 1 30); do
+ if docker exec fdb fdbcli --exec "status minimal" 2>/dev/null; then break; fi
+ sleep 1
+ done
+ docker exec fdb fdbcli --exec "configure new single memory"
+ echo "docker:[email protected]:4500" > /tmp/fdb.cluster
+
- name: Run tests
run: pnpm harness pnpm test:snips
working-directory: apps/api
@@ -230,7 +261,7 @@ jobs:
shell: python3 {0}
env:
JUNIT: apps/api/test-results/junit.xml
- MATRIX_LABEL: "${{ matrix.engine }}, ${{ matrix.proxy }}, ${{ matrix.search }}, ${{ matrix.ai }}"
+ MATRIX_LABEL: "${{ matrix.engine }}, ${{ matrix.proxy }}, ${{ matrix.search }}, ${{ matrix.ai }}, ${{ matrix.nuq }}"
run: |
import os, sys, xml.etree.ElementTree as ET
@@ -367,7 +398,7 @@ jobs:
- uses: actions/upload-artifact@v4
if: always()
with:
- name: Logs (kubernetes, ${{ matrix.ai }}, ${{ matrix.search }}, ${{ matrix.engine }}, ${{ matrix.proxy }})
+ name: Logs (kubernetes, ${{ matrix.ai }}, ${{ matrix.search }}, ${{ matrix.engine }}, ${{ matrix.proxy }}, ${{ matrix.nuq }})
path: logs/logs.zip
# temp disabled
@@ -536,3 +567,52 @@ jobs:
# name: Encrypted Logs
# path: logs/logs.zip.gpg
# retention-days: 5
+
+ nuq-fdb-core:
+ name: NuQ FoundationDB core tests
+ runs-on: blacksmith-4vcpu-ubuntu-2404
+ env:
+ FDB_VERSION: 7.3.63
+ steps:
+ - uses: actions/checkout@v5
+
+ - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4
+ with:
+ version: 10
+
+ - run: pnpm config set store-dir ~/.pnpm-store
+ - run: mkdir -p ~/.pnpm-store
+
+ - uses: actions/setup-node@v6
+ with:
+ node-version: 22
+ cache: pnpm
+ cache-dependency-path: apps/api/pnpm-lock.yaml
+
+ - name: Install API dependencies
+ run: pnpm install --frozen-lockfile --ignore-scripts
+ working-directory: apps/api
+ env:
+ npm_config_ignore_scripts: 'true'
+
+ - name: Install FoundationDB client library
+ run: |
+ curl -fsSL -o /tmp/fdb-clients.deb \
+ "https://github.com/apple/foundationdb/releases/download/${FDB_VERSION}/foundationdb-clients_${FDB_VERSION}-1_amd64.deb"
+ sudo dpkg -i /tmp/fdb-clients.deb
+
+ - name: Start FoundationDB
+ run: |
+ docker run -d --name fdb -p 4500:4500 -e FDB_NETWORKING_MODE=host foundationdb/foundationdb:${FDB_VERSION}
+ for i in $(seq 1 30); do
+ if docker exec fdb fdbcli --exec "status minimal" 2>/dev/null; then break; fi
+ sleep 1
+ done
+ docker exec fdb fdbcli --exec "configure new single memory"
+ echo "docker:[email protected]:4500" > /tmp/fdb.cluster
+
+ - name: Run NuQ FDB core tests
+ run: npx jest src/__tests__/nuq-fdb --reporters=default
+ working-directory: apps/api
+ env:
+ FDB_CLUSTER_FILE: /tmp/fdb.cluster
diff --git a/apps/api/Dockerfile b/apps/api/Dockerfile
index ec85c8900..061b1c381 100644
--- a/apps/api/Dockerfile
+++ b/apps/api/Dockerfile
@@ -27,6 +27,15 @@ RUN apt-get update && apt-get install -y \
python3 \
&& rm -rf /var/lib/apt/lists/*
+# FoundationDB client library (libfdb_c + headers), needed by the
+# `foundationdb` npm package at install time on arches without prebuilds
+ARG FDB_VERSION=7.3.63
+RUN ARCH=$(dpkg --print-architecture) && \
+ case "$ARCH" in arm64) FDB_ARCH=aarch64 ;; *) FDB_ARCH=$ARCH ;; esac && \
+ curl -fsSL -o /tmp/fdb-clients.deb \
+ "https://github.com/apple/foundationdb/releases/download/${FDB_VERSION}/foundationdb-clients_${FDB_VERSION}-1_${FDB_ARCH}.deb" && \
+ dpkg -i /tmp/fdb-clients.deb && rm /tmp/fdb-clients.deb
+
# Install Rust
ENV RUSTUP_HOME=/usr/local/rustup \
CARGO_HOME=/usr/local/cargo \
@@ -58,8 +67,17 @@ FROM base AS runtime
RUN apt-get update && apt-get install -y \
git \
procps \
+ curl \
&& rm -rf /var/lib/apt/lists/*
+# FoundationDB client library (libfdb_c + fdbcli)
+ARG FDB_VERSION=7.3.63
+RUN ARCH=$(dpkg --print-architecture) && \
+ case "$ARCH" in arm64) FDB_ARCH=aarch64 ;; *) FDB_ARCH=$ARCH ;; esac && \
+ curl -fsSL -o /tmp/fdb-clients.deb \
+ "https://github.com/apple/foundationdb/releases/download/${FDB_VERSION}/foundationdb-clients_${FDB_VERSION}-1_${FDB_ARCH}.deb" && \
+ dpkg -i /tmp/fdb-clients.deb && rm /tmp/fdb-clients.deb
+
EXPOSE 8080
WORKDIR /app
diff --git a/apps/api/package.json b/apps/api/package.json
index b8cd62c33..3af53eef9 100644
--- a/apps/api/package.json
+++ b/apps/api/package.json
@@ -115,6 +115,7 @@
"escape-html": "^1.0.3",
"express": "4.22.0",
"express-ws": "^5.0.2",
+ "foundationdb": "^2.0.1",
"geoip-country": "^5.0.202510312342",
"git-diff": "^2.0.6",
"he": "^1.2.0",
diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yaml
index c16588556..c41f6554d 100644
--- a/apps/api/pnpm-lock.yaml
+++ b/apps/api/pnpm-lock.yaml
@@ -160,6 +160,9 @@ importers:
express-ws:
specifier: ^5.0.2
+ foundationdb:
+ specifier: ^2.0.1
+ version: 2.0.1
geoip-country:
specifier: ^5.0.202510312342
version: 5.0.202510312342
@@ -3789,6 +3792,9 @@ packages:
resolution: {integrity: sha512-jKmm9YtsNXN789RS/0mSzOC1NUq9mkVd65vbSSVsKdjGvYXBuE4oWe2QOEoFeRmJg+lPuZxpmrfFclNhoRMneQ==}
+ resolution: {integrity: sha512-8jSvKPCYCgTpi9Pt87qlfTk6griyMx4Gk3Xv31Dp72Qp8b6XgIyFsMm8KzPmFJ9iJ8K4pGvRxvOS8D0XGnrkjw==}
+
resolution: {integrity: sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw==}
@@ -3861,6 +3867,10 @@ packages:
resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==}
engines: {node: '>= 0.6'}
+ resolution: {integrity: sha512-ZMmpAoEI88H8d4DWk6MzXf4Kw9pGaQrxNygVdz/7g+qwlMrjHvmxUQsQz1zG4176+B0YNY+ufgmcT40kEv/TnQ==}
+ engines: {node: '>=14.0.0'}
+
resolution: {integrity: sha512-hJOBnKk2e89/FCSXU0UN7B2CpMnqtN6RrIKzoG3qWSjLUXGCTEU86aTdNEio/1Eu2NyExIAVJUyHuZvmEA9iuw==}
engines: {node: '>= 10.0.0'}
@@ -9411,6 +9421,8 @@ snapshots:
dependencies:
walk-up-path: 4.0.0
+
@@ -9495,6 +9507,11 @@ snapshots:
+ dependencies:
+ fdb-tuple: 1.0.0
+ node-gyp-build: 4.8.4
+
dependencies:
bindings: 1.5.0
@@ -10623,8 +10640,7 @@ snapshots:
detect-libc: 2.1.2
optional: true
- optional: true
diff --git a/apps/api/src/__tests__/nuq-fdb/core.test.ts b/apps/api/src/__tests__/nuq-fdb/core.test.ts
new file mode 100644
index 000000000..b43dd5da0
--- /dev/null
+++ b/apps/api/src/__tests__/nuq-fdb/core.test.ts
@@ -0,0 +1,700 @@
+import { randomUUID } from "crypto";
+import { config } from "../../config";
+import {
+ NuQFdbQueue,
+ NuQFdbJobGroup,
+ NuqFdbSweeper,
+} from "../../services/worker/nuq-fdb";
+import {
+ getNuqFdbDatabase,
+ getFdb,
+} from "../../services/worker/nuq-fdb/client";
+import { encodeJson } from "../../services/worker/nuq-fdb/keyspace";
+
+// These tests exercise the FDB queue core directly against a real FoundationDB
+// cluster (no API server needed). They are skipped when FDB is not configured.
+const describeIf = config.FDB_CLUSTER_FILE ? describe : describe.skip;
+
+// inline returnvalues only exist on the self-host path; cloud stores to GCS
+const expectInlineReturnvalue = !config.GCS_BUCKET_NAME;
+
+const RUN = randomUUID().slice(0, 8);
+const TEST_LEASE_MS = 1500;
+
+const createdQueueNames: string[] = [];
+
+type Ctx = {
+ queue: NuQFdbQueue;
+ finishedQueue: NuQFdbQueue;
+ group: NuQFdbJobGroup;
+ sweeper: NuqFdbSweeper;
+};
+
+// Each test gets its own queue keyspace so leaked jobs (some tests leave them
+// behind on purpose) can never bleed into other tests' takes.
+async function makeCtx(name: string): Promise<Ctx> {
+ const scrapeName = `t-${RUN}-${name}`;
+ const finishedName = `t-${RUN}-${name}-fin`;
+ createdQueueNames.push(scrapeName, finishedName);
+ const queue = new NuQFdbQueue(scrapeName, {
+ hasGroups: true,
+ finishedQueueName: finishedName,
+ leaseMs: TEST_LEASE_MS,
+ });
+ const finishedQueue = new NuQFdbQueue(finishedName, { hasGroups: false });
+ const group = new NuQFdbJobGroup(queue.ks, queue.groupOps!);
+ const sweeper = new NuqFdbSweeper([queue, finishedQueue]);
+ return { queue, finishedQueue, group, sweeper };
+}
+
+async function takeAll(
+ queue: NuQFdbQueue,
+ maxJobs: number = 50,
+): Promise<any[]> {
+ const out: any[] = [];
+ while (out.length < maxJobs) {
+ const job = await queue.getJobToProcess();
+ if (job === null) break;
+ out.push(job);
+ }
+ return out;
+}
+
+function freshOwner(): string {
+ return randomUUID();
+}
+
+function scrapeData(extra: Record<string, any> = {}): any {
+ return { mode: "single_urls", url: "https://example.com", ...extra };
+}
+
+const UNLIMITED = { teamLimit: null, queueCap: 1_000_000 };
+const gate = (limit: number, cap: number = 1_000_000) => ({
+ teamLimit: limit,
+ queueCap: cap,
+});
+
+describeIf("NuQ FDB core", () => {
+ afterAll(async () => {
+ const fdb = getFdb();
+ const db = getNuqFdbDatabase();
+ for (const name of createdQueueNames) {
+ const r = fdb.tuple.range(["nuq", name]);
+ await db.doTn(async tn =>
+ tn.clearRange(r.begin as Buffer, r.end as Buffer),
+ );
+ }
+ });
+
+ test("enqueue -> take -> finish roundtrip (ungated)", async () => {
+ const { queue } = await makeCtx("roundtrip");
+ const id = randomUUID();
+ await queue.addJob(id, scrapeData(), { ownerId: freshOwner() }, UNLIMITED);
+
+ const taken = await takeAll(queue, 1);
+ expect(taken.length).toBe(1);
+ expect(taken[0].id).toBe(id);
+ expect(taken[0].lock).toBeDefined();
+ expect(taken[0].data.url).toBe("https://example.com");
+
+ const ok = await queue.jobFinish(id, taken[0].lock!, { result: "yay" });
+ expect(ok).toBe(true);
+
+ const job = await queue.getJob(id);
+ expect(job?.status).toBe("completed");
+ if (expectInlineReturnvalue) {
+ expect(job?.returnvalue).toEqual({ result: "yay" });
+ } else {
+ expect(job?.returnvalue).toBeNull();
+ }
+ });
+
+ test("finish with wrong lock is rejected; double finish is idempotent", async () => {
+ const { queue } = await makeCtx("locks");
+ const id = randomUUID();
+ await queue.addJob(id, scrapeData(), { ownerId: freshOwner() }, UNLIMITED);
+ const [job] = await takeAll(queue, 1);
+ expect(job.id).toBe(id);
+
+ expect(await queue.jobFinish(id, randomUUID(), null)).toBe(false);
+ expect(await queue.jobFinish(id, job.lock!, { ok: 1 })).toBe(true);
+ expect(await queue.jobFinish(id, job.lock!, { ok: 1 })).toBe(true);
+ expect(await queue.jobFail(id, job.lock!, "nope")).toBe(false);
+ });
+
+ test("team concurrency gate: limit 2 admits 2, backlogs the rest, promotes on finish", async () => {
+ const { queue } = await makeCtx("teamgate");
+ const owner = freshOwner();
+ const ids = Array.from({ length: 5 }, () => randomUUID());
+ const jobs = await queue.addJobs(
+ ids.map(id => ({
+ id,
+ data: scrapeData(),
+ options: { ownerId: owner, timesOutAt: new Date(Date.now() + 60_000) },
+ })),
+ gate(2),
+ );
+
+ expect(jobs.filter(j => j.status === "queued").length).toBe(2);
+ expect(jobs.filter(j => j.status === "backlog").length).toBe(3);
+ expect(await queue.getTeamActiveCount(owner)).toBe(2);
+ expect(await queue.getTeamPendingCount(owner)).toBe(3);
+
+ const taken = await takeAll(queue, 5);
+ expect(taken.length).toBe(2);
+
+ // finishing one job promotes exactly one backlogged job
+ await queue.jobFinish(taken[0].id, taken[0].lock!, null);
+ expect(await queue.getTeamActiveCount(owner)).toBe(2);
+ expect(await queue.getTeamPendingCount(owner)).toBe(2);
+
+ const next = await takeAll(queue, 5);
+ expect(next.length).toBe(1);
+
+ await queue.jobFinish(taken[1].id, taken[1].lock!, null);
+ await queue.jobFinish(next[0].id, next[0].lock!, null);
+ const rest = await takeAll(queue, 5);
+ expect(rest.length).toBe(2);
+ for (const j of rest) await queue.jobFinish(j.id, j.lock!, null);
+ expect(await queue.getTeamActiveCount(owner)).toBe(0);
+ expect(await queue.getTeamPendingCount(owner)).toBe(0);
+ });
+
+ test("promotion respects priority order", async () => {
+ const { queue } = await makeCtx("priority");
+ const owner = freshOwner();
+ const blocker = randomUUID();
+ await queue.addJob(
+ blocker,
+ scrapeData(),
+ { ownerId: owner, priority: 0 },
+ gate(1),
+ );
+ const lowPrio = randomUUID();
+ const highPrio = randomUUID();
+ await queue.addJob(
+ lowPrio,
+ scrapeData(),
+ {
+ ownerId: owner,
+ priority: 50,
+ timesOutAt: new Date(Date.now() + 60_000),
+ },
+ gate(1),
+ );
+ await queue.addJob(
+ highPrio,
+ scrapeData(),
+ {
+ ownerId: owner,
+ priority: 1,
+ timesOutAt: new Date(Date.now() + 60_000),
+ },
+ gate(1),
+ );
+
+ const [b] = await takeAll(queue, 1);
+ expect(b.id).toBe(blocker);
+ await queue.jobFinish(b.id, b.lock!, null);
+
+ const [promoted] = await takeAll(queue, 1);
+ expect(promoted.id).toBe(highPrio);
+ });
+
+ test("kickoff jobs bypass the gate and hold no slot", async () => {
+ const { queue } = await makeCtx("kickoff");
+ const owner = freshOwner();
+ const kickoff = randomUUID();
+ await queue.addJob(
+ kickoff,
+ { mode: "kickoff", crawl_id: randomUUID() },
+ { ownerId: owner, bypassGate: true },
+ gate(1),
+ );
+ expect(await queue.getTeamActiveCount(owner)).toBe(0);
+
+ const [k] = await takeAll(queue, 1);
+ expect(k.id).toBe(kickoff);
+ expect(await queue.getTeamActiveCount(owner)).toBe(0);
+ await queue.jobFinish(k.id, k.lock!, null);
+ expect(await queue.getTeamActiveCount(owner)).toBe(0);
+ });
+
+ test("QueueFullError when the backlog cap is exceeded", async () => {
+ const { queue } = await makeCtx("qfull");
+ const owner = freshOwner();
+ const blocker = randomUUID();
+ await queue.addJob(blocker, scrapeData(), { ownerId: owner }, gate(1, 2));
+ // 1 active, cap 2: two backlogged jobs fit, the third addJobs blows up
+ await queue.addJobs(
+ [randomUUID(), randomUUID()].map(id => ({
+ id,
+ data: scrapeData(),
+ options: { ownerId: owner, timesOutAt: new Date(Date.now() + 60_000) },
+ })),
+ gate(1, 2),
+ );
+ await expect(
+ queue.addJob(
+ randomUUID(),
+ scrapeData(),
+ { ownerId: owner, timesOutAt: new Date(Date.now() + 60_000) },
+ gate(1, 2),
+ ),
+ ).rejects.toThrow(/queue limit reached/i);
+ });
+
+ test("group lifecycle: numeric stats, finish detection, crawl_finished emission", async () => {
+ const { queue, finishedQueue, group } = await makeCtx("glife");
+ const owner = freshOwner();
+ const gid = randomUUID();
+ await group.addGroup(gid, owner);
+
+ const ids = Array.from({ length: 3 }, () => randomUUID());
+ await queue.addJobs(
+ ids.map(id => ({
+ id,
+ data: scrapeData(),
+ options: { ownerId: owner, groupId: gid },
+ })),
+ gate(10),
+ );
+
+ let stats = await queue.getGroupNumericStats(gid);
+ expect(stats.queued).toBe(3);
+
+ const taken = await takeAll(queue, 3);
+ expect(taken.length).toBe(3);
+ stats = await queue.getGroupNumericStats(gid);
+ expect(stats.active).toBe(3);
+
+ for (const j of taken.slice(0, 2))
+ await queue.jobFinish(j.id, j.lock!, null);
+ await queue.jobFail(taken[2].id, taken[2].lock!, "boom");
+
+ stats = await queue.getGroupNumericStats(gid);
+ expect(stats.completed).toBe(2);
+ expect(stats.failed).toBe(1);
+ expect(stats.active).toBe(0);
+
+ // the last terminal transition completes the group inline
+ const g = await group.getGroup(gid);
+ expect(g?.status).toBe("completed");
+ expect(g?.expiresAt).toBeDefined();
+
+ // and emits exactly one crawl_finished job
+ const finishedJob = await finishedQueue.getJobToProcess();
+ expect(finishedJob).not.toBeNull();
+ expect(finishedJob!.groupId).toBe(gid);
+ expect(await finishedQueue.getJobToProcess()).toBeNull();
+ await finishedQueue.jobFinish(finishedJob!.id, finishedJob!.lock!, null);
+
+ const ongoing = await group.getOngoingByOwner(owner);
+ expect(ongoing.find(o => o.id === gid)).toBeUndefined();
+ });
+
+ test("crawl maxConcurrency gates within the team limit", async () => {
+ const { queue, group } = await makeCtx("crawlmax");
+ const owner = freshOwner();
+ const gid = randomUUID();
+ await group.addGroup(gid, owner, undefined, { maxConcurrency: 1 });
+
+ const ids = Array.from({ length: 3 }, () => randomUUID());
+ await queue.addJobs(
+ ids.map(id => ({
+ id,
+ data: scrapeData(),
+ options: {
+ ownerId: owner,
+ groupId: gid,
+ timesOutAt: new Date(Date.now() + 60_000),
+ },
+ })),
+ gate(10),
+ );
+
+ // only 1 crawl slot: one job ready, two crawl-pending
+ expect(await queue.getTeamActiveCount(owner)).toBe(1);
+ const taken1 = await takeAll(queue, 3);
+ expect(taken1.length).toBe(1);
+
+ await queue.jobFinish(taken1[0].id, taken1[0].lock!, null);
+ const taken2 = await takeAll(queue, 3);
+ expect(taken2.length).toBe(1);
+
+ await queue.jobFinish(taken2[0].id, taken2[0].lock!, null);
+ const taken3 = await takeAll(queue, 3);
+ expect(taken3.length).toBe(1);
+ await queue.jobFinish(taken3[0].id, taken3[0].lock!, null);
+
+ const g = await group.getGroup(gid);
+ expect(g?.status).toBe("completed");
+ });
+
+ test("crawl delay: next job is parked until the not-before time", async () => {
+ const { queue, group, sweeper } = await makeCtx("delay");
+ const owner = freshOwner();
+ const gid = randomUUID();
+ await group.addGroup(gid, owner, undefined, { delaySeconds: 1 });
+
+ const ids = [randomUUID(), randomUUID()];
+ await queue.addJobs(
+ ids.map(id => ({
+ id,
+ data: scrapeData(),
+ options: {
+ ownerId: owner,
+ groupId: gid,
+ timesOutAt: new Date(Date.now() + 60_000),
+ },
+ })),
+ gate(10),
+ );
+
+ const [first] = await takeAll(queue, 2);
+ expect(first).toBeDefined();
+ await queue.jobFinish(first.id, first.lock!, null);
+
+ // second job is in the delay index, not takeable yet
+ await sweeper.sweepOnce();
+ expect(await queue.getJobToProcess()).toBeNull();
+
+ await new Promise(resolve => setTimeout(resolve, 1200));
+ await sweeper.sweepOnce();
+ const [second] = await takeAll(queue, 1);
+ expect(second).toBeDefined();
+ expect(second.id).toBe(ids.find(i => i !== first.id));
+ await queue.jobFinish(second.id, second.lock!, null);
+ });
+
+ test("lease expiry: sweeper requeues a stalled job, then fails it after MAX_STALLS", async () => {
+ const { queue, sweeper } = await makeCtx("stalls");
+ const owner = freshOwner();
+ const id = randomUUID();
+ await queue.addJob(id, scrapeData(), { ownerId: owner }, gate(5));
+
+ const [job] = await takeAll(queue, 1);
+ expect(job.id).toBe(id);
+
+ // let the lease expire without renewal
+ await new Promise(resolve => setTimeout(resolve, TEST_LEASE_MS + 200));
+ await sweeper.sweepOnce();
+
+ let j = await queue.getJob(id);
+ expect(j?.status).toBe("queued");
+
+ // worker that lost its lease can no longer finish
+ expect(await queue.jobFinish(id, job.lock!, null)).toBe(false);
+
+ // stall it to death
+ for (let i = 0; i < 9; i++) {
+ const [again] = await takeAll(queue, 1);
+ expect(again.id).toBe(id);
+ await new Promise(resolve => setTimeout(resolve, TEST_LEASE_MS + 200));
+ await sweeper.sweepOnce();
+ }
+ j = await queue.getJob(id);
+ expect(j?.status).toBe("failed");
+ expect(j?.failedReason).toMatch(/stalled/i);
+ expect(await queue.getTeamActiveCount(owner)).toBe(0);
+ }, 60_000);
+
+ test("renewLock keeps the lease alive", async () => {
+ const { queue, sweeper } = await makeCtx("renew");
+ const owner = freshOwner();
+ const id = randomUUID();
+ await queue.addJob(id, scrapeData(), { ownerId: owner }, gate(5));
+ const [job] = await takeAll(queue, 1);
+
+ for (let i = 0; i < 3; i++) {
+ await new Promise(resolve => setTimeout(resolve, TEST_LEASE_MS / 2));
+ expect(await queue.renewLock(id, job.lock!)).toBe(true);
+ }
+ await sweeper.sweepOnce();
+ const j = await queue.getJob(id);
+ expect(j?.status).toBe("active");
+ expect(await queue.jobFinish(id, job.lock!, null)).toBe(true);
+ }, 30_000);
+
+ test("backlog timeout: pending jobs are silently dropped at their deadline", async () => {
+ const { queue, sweeper } = await makeCtx("bto");
+ const owner = freshOwner();
+ const blocker = randomUUID();
+ await queue.addJob(blocker, scrapeData(), { ownerId: owner }, gate(1));
+ const doomed = randomUUID();
+ await queue.addJob(
+ doomed,
+ scrapeData(),
+ { ownerId: owner, timesOutAt: new Date(Date.now() - 1000) },
+ gate(1),
+ );
+
+ expect(await queue.getTeamPendingCount(owner)).toBe(1);
+ await sweeper.sweepOnce();
+ expect(await queue.getTeamPendingCount(owner)).toBe(0);
+ expect(await queue.getJob(doomed)).toBeNull();
+
+ const [b] = await takeAll(queue, 1);
+ await queue.jobFinish(b.id, b.lock!, null);
+ });
+
+ test("group cancellation: pending dropped, ready diverted, group still completes", async () => {
+ const { queue, finishedQueue, group, sweeper } = await makeCtx("cancel");
+ const owner = freshOwner();
+ const gid = randomUUID();
+ await group.addGroup(gid, owner);
+
+ const ids = Array.from({ length: 4 }, () => randomUUID());
+ await queue.addJobs(
+ ids.map(id => ({
+ id,
+ data: scrapeData(),
+ options: {
+ ownerId: owner,
+ groupId: gid,
+ timesOutAt: new Date(Date.now() + 60_000),
+ },
+ })),
+ gate(2),
+ );
+ // 2 ready, 2 team-pending
+ const [active] = await takeAll(queue, 1);
+
+ expect(await group.cancelGroup(gid)).toBe(true);
+ await sweeper.sweepOnce(); // cleans pending members
+
+ // the remaining ready job is diverted at take time
+ const after = await takeAll(queue, 4);
+ expect(after.length).toBe(0);
+
+ // active job finishes normally; that drains the group
+ await queue.jobFinish(active.id, active.lock!, null);
+ await sweeper.sweepOnce();
+
+ const g = await group.getGroup(gid);
+ expect(g?.status).toBe("completed");
+ expect(await queue.getTeamActiveCount(owner)).toBe(0);
+
+ // cancelled crawls still emit their crawl_finished job
+ const fin = await finishedQueue.getJobToProcess();
+ expect(fin).not.toBeNull();
+ expect(fin!.groupId).toBe(gid);
+ await finishedQueue.jobFinish(fin!.id, fin!.lock!, null);
+ });
+
+ test("group cancellation scans past stale group-index rows before clearing task", async () => {
+ const { queue, group, sweeper } = await makeCtx("cancel-stale-index");
+ const db = getNuqFdbDatabase();
+ const owner = freshOwner();
+ const gid = randomUUID();
+ const pendingId = "zzzz-real-pending";
+ await group.addGroup(gid, owner);
+
+ await db.doTn(async tn => {
+ for (let i = 0; i < 600; i++) {
+ tn.set(
+ queue.ks.groupJob(gid, `0000-stale-${String(i).padStart(4, "0")}`),
+ encodeJson({ m: 1, s: "pending" }),
+ );
+ }
+ });
+
+ await queue.addJob(
+ pendingId,
+ scrapeData(),
+ {
+ ownerId: owner,
+ groupId: gid,
+ timesOutAt: new Date(Date.now() + 60_000),
+ },
+ gate(0),
+ );
+ expect(await queue.getTeamPendingCount(owner)).toBe(1);
+
+ expect(await group.cancelGroup(gid)).toBe(true);
+ await sweeper.sweepOnce();
+
+ expect(await queue.getJob(pendingId)).toBeNull();
+ expect(await queue.getTeamPendingCount(owner)).toBe(0);
+ await db.doTn(async tn => {
+ expect(await tn.get(queue.ks.taskGroupCancel(gid))).toBeFalsy();
+ });
+ });
+
+ test("waitForJob resolves on completion and rejects on failure", async () => {
+ const { queue } = await makeCtx("wait");
+ const owner = freshOwner();
+ const id1 = randomUUID();
+ await queue.addJob(id1, scrapeData(), { ownerId: owner }, UNLIMITED);
+
+ const wait1 = queue.waitForJob(id1, 15_000);
+ const [j1] = await takeAll(queue, 1);
+ await queue.jobFinish(j1.id, j1.lock!, { doc: "ok" });
+ if (expectInlineReturnvalue) {
+ await expect(wait1).resolves.toEqual({ doc: "ok" });
+ } else {
+ await expect(wait1).resolves.toBeNull();
+ }
+
+ const id2 = randomUUID();
+ await queue.addJob(id2, scrapeData(), { ownerId: owner }, UNLIMITED);
+ const wait2 = queue.waitForJob(id2, 15_000);
+ const [j2] = await takeAll(queue, 1);
+ await queue.jobFail(j2.id, j2.lock!, "scrape exploded");
+ await expect(wait2).rejects.toThrow("scrape exploded");
+
+ const id3 = randomUUID();
+ await queue.addJob(id3, scrapeData(), { ownerId: owner }, UNLIMITED);
+ await expect(queue.waitForJob(id3, 500)).rejects.toThrow(/timed out/i);
+ const [j3] = await takeAll(queue, 1);
+ await queue.jobFinish(j3.id, j3.lock!, null);
+ }, 30_000);
+
+ test("large returnvalue is chunked and reassembled (self-host path)", async () => {
+ if (!expectInlineReturnvalue) return;
+ const { queue } = await makeCtx("chunks");
+ const owner = freshOwner();
+ const id = randomUUID();
+ await queue.addJob(id, scrapeData(), { ownerId: owner }, UNLIMITED);
+ const [job] = await takeAll(queue, 1);
+
+ const big = { blob: "x".repeat(300 * 1024) };
+ await queue.jobFinish(id, job.lock!, big);
+
+ const j = await queue.getJob(id);
+ expect(j?.status).toBe("completed");
+ expect(j?.returnvalue?.blob?.length).toBe(300 * 1024);
+ });
+
+ test("getCrawlJobsForListing paginates completed jobs in finish order", async () => {
+ const { queue, group } = await makeCtx("listing");
+ const owner = freshOwner();
+ const gid = randomUUID();
+ await group.addGroup(gid, owner);
+
+ const ids = Array.from({ length: 5 }, () => randomUUID());
+ await queue.addJobs(
+ ids.map(id => ({
+ id,
+ data: scrapeData(),
+ options: { ownerId: owner, groupId: gid },
+ })),
+ gate(10),
+ );
+ const taken = await takeAll(queue, 5);
+ expect(taken.length).toBe(5);
+ const finishOrder: string[] = [];
+ for (const j of taken) {
+ await queue.jobFinish(j.id, j.lock!, null);
+ finishOrder.push(j.id);
+ }
+
+ const page1 = await queue.getCrawlJobsForListing(gid, 3, 0);
+ const page2 = await queue.getCrawlJobsForListing(gid, 3, 3);
+ expect(page1.map(j => j.id)).toEqual(finishOrder.slice(0, 3));
+ expect(page2.map(j => j.id)).toEqual(finishOrder.slice(3));
+ });
+
+ test("getGroupAnyJob returns a single_urls member and checks ownership", async () => {
+ const { queue, group } = await makeCtx("anyjob");
+ const owner = freshOwner();
+ const gid = randomUUID();
+ await group.addGroup(gid, owner);
+ const id = randomUUID();
+ await queue.addJob(
+ id,
+ scrapeData(),
+ { ownerId: owner, groupId: gid },
+ gate(10),
+ );
+
+ const any = await queue.getGroupAnyJob(gid, owner);
+ expect(any?.id).toBe(id);
+ expect(await queue.getGroupAnyJob(gid, randomUUID())).toBeNull();
+
+ const [j] = await takeAll(queue, 1);
+ await queue.jobFinish(j.id, j.lock!, null);
+ });
+
+ test("removeJob releases slots and promotes backlog", async () => {
+ const { queue } = await makeCtx("remove");
+ const owner = freshOwner();
+ const a = randomUUID();
+ const b = randomUUID();
+ await queue.addJob(a, scrapeData(), { ownerId: owner }, gate(1));
+ await queue.addJob(
+ b,
+ scrapeData(),
+ { ownerId: owner, timesOutAt: new Date(Date.now() + 60_000) },
+ gate(1),
+ );
+ expect(await queue.getTeamPendingCount(owner)).toBe(1);
+
+ // removing the slot-holding job promotes the backlogged one
+ expect(await queue.removeJob(a)).toBe(true);
+ expect(await queue.getTeamPendingCount(owner)).toBe(0);
+ expect(await queue.getJob(a)).toBeNull();
+
+ const [j] = await takeAll(queue, 1);
+ expect(j.id).toBe(b);
+ await queue.jobFinish(j.id, j.lock!, null);
+ });
+
+ test("limit raise promotes backlogged jobs via the sweeper", async () => {
+ const { queue, sweeper } = await makeCtx("raise");
+ const owner = freshOwner();
+ const ids = Array.from({ length: 4 }, () => randomUUID());
+ await queue.addJobs(
+ ids.map(id => ({
+ id,
+ data: scrapeData(),
+ options: { ownerId: owner, timesOutAt: new Date(Date.now() + 60_000) },
+ })),
+ gate(1),
+ );
+ expect(await queue.getTeamActiveCount(owner)).toBe(1);
+ expect(await queue.getTeamPendingCount(owner)).toBe(3);
+
+ // a later enqueue arrives with a raised limit (ACUC change)
+ await queue.addJob(
+ randomUUID(),
+ scrapeData(),
+ { ownerId: owner, timesOutAt: new Date(Date.now() + 60_000) },
+ gate(4),
+ );
+ await sweeper.sweepOnce();
+
+ expect(await queue.getTeamActiveCount(owner)).toBe(4);
+ expect(await queue.getTeamPendingCount(owner)).toBe(1);
+
+ const taken = await takeAll(queue, 4);
+ expect(taken.length).toBe(4);
+ for (const j of taken) await queue.jobFinish(j.id, j.lock!, null);
+ const [last] = await takeAll(queue, 1);
+ await queue.jobFinish(last.id, last.lock!, null);
+ });
+
+ test("group TTL cleanup removes job records", async () => {
+ const { queue, group, sweeper } = await makeCtx("gttl");
+ const owner = freshOwner();
+ const gid = randomUUID();
+ await group.addGroup(gid, owner, 1000); // 1s TTL
+
+ const id = randomUUID();
+ await queue.addJob(
+ id,
+ scrapeData(),
+ { ownerId: owner, groupId: gid },
+ gate(10),
+ );
+ const [j] = await takeAll(queue, 1);
+ await queue.jobFinish(j.id, j.lock!, null);
+
+ expect((await group.getGroup(gid))?.status).toBe("completed");
+ await new Promise(resolve => setTimeout(resolve, 1100));
+ await sweeper.sweepOnce();
+
+ expect(await group.getGroup(gid)).toBeNull();
+ expect(await queue.getJob(id)).toBeNull();
+ });
+});
diff --git a/apps/api/src/__tests__/nuq-fdb/router.test.ts b/apps/api/src/__tests__/nuq-fdb/router.test.ts
new file mode 100644
index 000000000..5c5a9fdb7
--- /dev/null
+++ b/apps/api/src/__tests__/nuq-fdb/router.test.ts
@@ -0,0 +1,187 @@
+import { randomUUID } from "crypto";
+import { config } from "../../config";
+import {
+ fdbQueueEnabled,
+ isFdbTeam,
+ resolveJobBackend,
+ resolveNewGroupBackend,
+ fdbEnqueueScrapeJobs,
+ scrapeQueue,
+ crawlGroup,
+ crawlFinishedQueue,
+ mirrorExternalSlotAcquire,
+ mirrorExternalSlotRelease,
+} from "../../services/worker/nuq-router";
+import { scrapeQueueFdb } from "../../services/worker/nuq-fdb";
+import {
+ getNuqFdbDatabase,
+ getFdb,
+} from "../../services/worker/nuq-fdb/client";
+
+// Exercises the dual-backend router in forced-FDB mode (NUQ_BACKEND=fdb,
+// self-hosted), which needs neither ACUC nor a PG nuq database: everything
+// must route to FDB and never touch the PG pool. Requires a live FDB cluster.
+const describeIf = config.FDB_CLUSTER_FILE ? describe : describe.skip;
+
+const prevNuqBackend = config.NUQ_BACKEND;
+const prevDbAuth = config.USE_DB_AUTHENTICATION;
+
+describeIf("NuQ router (forced FDB mode)", () => {
+ beforeAll(() => {
+ config.NUQ_BACKEND = "fdb";
+ config.USE_DB_AUTHENTICATION = false; // self-hosted: unlimited concurrency
+ });
+
+ afterAll(async () => {
+ config.NUQ_BACKEND = prevNuqBackend;
+ config.USE_DB_AUTHENTICATION = prevDbAuth;
+ // forced mode writes into the real "scrape"/"crawl_finished" queue
+ // namespaces; wipe them so reruns and other suites start clean
+ const fdb = getFdb();
+ const db = getNuqFdbDatabase();
+ for (const name of ["scrape", "crawl_finished"]) {
+ const r = fdb.tuple.range(["nuq", name]);
+ await db.doTn(async tn =>
+ tn.clearRange(r.begin as Buffer, r.end as Buffer),
+ );
+ }
+ });
+
+ test("routing decisions resolve to fdb when forced", async () => {
+ expect(fdbQueueEnabled()).toBe(true);
+ expect(await isFdbTeam(randomUUID())).toBe(true);
+ expect(await resolveNewGroupBackend(randomUUID())).toBe("fdb");
+ expect(
+ await resolveJobBackend({
+ mode: "single_urls",
+ url: "https://example.com",
+ team_id: randomUUID(),
+ } as any),
+ ).toBe("fdb");
+ });
+
+ test("enqueue -> routed take -> routed finish -> routed waitForJob, all on FDB", async () => {
+ const teamId = randomUUID();
+ const jobId = randomUUID();
+ const { jobs, backloggedCount } = await fdbEnqueueScrapeJobs(
+ [
+ {
+ jobId,
+ data: {
+ mode: "single_urls",
+ url: "https://example.com",
+ team_id: teamId,
+ } as any,
+ priority: 0,
+ backlogTimeoutMs: 60_000,
+ },
+ ],
+ teamId,
+ );
+ expect(jobs[0].backend).toBe("fdb");
+ expect(jobs[0].status).toBe("queued"); // self-hosted: no gate
+ expect(backloggedCount).toBe(0);
+
+ const wait = scrapeQueue.waitForJob(jobId, 15_000);
+
+ // routed take must find the FDB job without ever polling PG (no PG here)
+ let taken: any = null;
+ for (let i = 0; i < 10 && !taken; i++) {
+ try {
+ taken = await scrapeQueue.getJobToProcess();
+ } catch {
+ // PG fallback poll can throw without a database; FDB must still win
+ }
+ }
+ expect(taken).not.toBeNull();
+ expect(taken.id).toBe(jobId);
+ expect((taken as any).backend).toBe("fdb");
+
+ expect(await scrapeQueue.renewLock(jobId, taken.lock!)).toBe(true);
+ expect(await scrapeQueue.jobFinish(jobId, taken.lock!, { ok: true })).toBe(
+ true,
+ );
+
+ await expect(wait).resolves.toBeDefined();
+
+ const job = await scrapeQueue.getJob(jobId);
+ expect(job?.status).toBe("completed");
+ const jobsRead = await scrapeQueue.getJobs([jobId]);
+ expect(jobsRead.length).toBe(1);
+ });
+
+ test("routed group lifecycle incl. crawl_finished consumption and cancel", async () => {
+ const teamId = randomUUID();
+ const gid = randomUUID();
+ const group = await crawlGroup.addGroup(gid, teamId, 60_000, {
+ backend: "fdb",
+ });
+ expect(group.status).toBe("active");
+ expect((await crawlGroup.getGroup(gid))?.id).toBe(gid);
+
+ const jobId = randomUUID();
+ await fdbEnqueueScrapeJobs(
+ [
+ {
+ jobId,
+ data: {
+ mode: "single_urls",
+ url: "https://example.com",
+ team_id: teamId,
+ crawl_id: gid,
+ } as any,
+ priority: 0,
+ backlogTimeoutMs: 60_000,
+ },
+ ],
+ teamId,
+ );
+
+ let taken: any = null;
+ for (let i = 0; i < 10 && !taken; i++) {
+ try {
+ taken = await scrapeQueue.getJobToProcess();
+ } catch {}
+ }
+ expect(taken?.id).toBe(jobId);
+ await scrapeQueue.jobFinish(jobId, taken.lock!, null);
+
+ expect((await crawlGroup.getGroup(gid))?.status).toBe("completed");
+ expect(await scrapeQueue.getGroupNumericStats(gid)).toMatchObject({
+ completed: 1,
+ });
+ const listing = await scrapeQueue.getCrawlJobsForListing(gid, 10, 0);
+ expect(listing.map(j => j.id)).toEqual([jobId]);
+
+ // the emitted crawl_finished job is consumable through the router
+ let fin: any = null;
+ for (let i = 0; i < 10 && !fin; i++) {
+ try {
+ fin = await crawlFinishedQueue.getJobToProcess();
+ } catch {}
+ }
+ expect(fin).not.toBeNull();
+ expect(fin.groupId).toBe(gid);
+ expect(await crawlFinishedQueue.jobFinish(fin.id, fin.lock!, null)).toBe(
+ true,
+ );
+
+ // cancel on an already-completed group is a no-op
+ expect(await crawlGroup.cancelGroup(gid)).toBe(false);
+ });
+
+ test("external slot mirror consumes and releases FDB capacity", async () => {
+ const teamId = randomUUID();
+ const holder = randomUUID();
+ await mirrorExternalSlotAcquire(teamId, holder, 30_000);
+ expect(await scrapeQueueFdb.getTeamActiveCount(teamId)).toBe(1);
+ // re-acquire (heartbeat) must not double-count
+ await mirrorExternalSlotAcquire(teamId, holder, 30_000);
+ expect(await scrapeQueueFdb.getTeamActiveCount(teamId)).toBe(1);
+ await mirrorExternalSlotRelease(teamId, holder);
+ expect(await scrapeQueueFdb.getTeamActiveCount(teamId)).toBe(0);
+ // double release is a no-op
+ await mirrorExternalSlotRelease(teamId, holder);
+ expect(await scrapeQueueFdb.getTeamActiveCount(teamId)).toBe(0);
+ });
+});
diff --git a/apps/api/src/__tests__/nuq-fdb/stress.test.ts b/apps/api/src/__tests__/nuq-fdb/stress.test.ts
new file mode 100644
index 000000000..db6daeb56
--- /dev/null
+++ b/apps/api/src/__tests__/nuq-fdb/stress.test.ts
@@ -0,0 +1,176 @@
+import { randomUUID } from "crypto";
+import { config } from "../../config";
+import { NuQFdbQueue, NuQFdbJobGroup } from "../../services/worker/nuq-fdb";
+import {
+ getNuqFdbDatabase,
+ getFdb,
+} from "../../services/worker/nuq-fdb/client";
+
+const describeIf = config.FDB_CLUSTER_FILE ? describe : describe.skip;
+
+const RUN = randomUUID().slice(0, 8);
+
+describeIf("NuQ FDB concurrency stress", () => {
+ const queueNames: string[] = [];
+
+ afterAll(async () => {
+ const fdb = getFdb();
+ const db = getNuqFdbDatabase();
+ for (const name of queueNames) {
+ const r = fdb.tuple.range(["nuq", name]);
+ await db.doTn(async tn =>
+ tn.clearRange(r.begin as Buffer, r.end as Buffer),
+ );
+ }
+ });
+
+ test("parallel workers never exceed the team limit and drain everything", async () => {
+ const name = `t-${RUN}-stress`;
+ queueNames.push(name);
+ const queue: NuQFdbQueue = new NuQFdbQueue(name, { hasGroups: true });
+
+ const owner = randomUUID();
+ const LIMIT = 5;
+ const JOBS = 60;
+ const WORKERS = 12;
+ const gate = { teamLimit: LIMIT, queueCap: 1_000_000 };
+
+ await queue.addJobs(
+ Array.from({ length: JOBS }, () => ({
+ id: randomUUID(),
+ data: { mode: "single_urls", url: "https://example.com" },
+ options: {
+ ownerId: owner,
+ timesOutAt: new Date(Date.now() + 120_000),
+ },
+ })),
+ gate,
+ );
+
+ expect(await queue.getTeamActiveCount(owner)).toBe(LIMIT);
+ expect(await queue.getTeamPendingCount(owner)).toBe(JOBS - LIMIT);
+
+ let inFlight = 0;
+ let maxInFlight = 0;
+ let processed = 0;
+ const seen = new Set<string>();
+
+ // simulated workers: take, hold briefly, finish
+ const worker = async () => {
+ let idleStrikes = 0;
+ while (idleStrikes < 8) {
+ const job = await queue.getJobToProcess();
+ if (!job) {
+ idleStrikes++;
+ await new Promise(resolve => setTimeout(resolve, 50));
+ continue;
+ }
+ idleStrikes = 0;
+ expect(seen.has(job.id)).toBe(false);
+ seen.add(job.id);
+ inFlight++;
+ maxInFlight = Math.max(maxInFlight, inFlight);
+ await new Promise(resolve =>
+ setTimeout(resolve, 10 + Math.random() * 30),
+ );
+ inFlight--;
+ const ok = await queue.jobFinish(job.id, job.lock!, null);
+ expect(ok).toBe(true);
+ processed++;
+ }
+ };
+
+ await Promise.all(Array.from({ length: WORKERS }, () => worker()));
+
+ expect(processed).toBe(JOBS);
+ expect(maxInFlight).toBeLessThanOrEqual(LIMIT);
+ expect(await queue.getTeamActiveCount(owner)).toBe(0);
+ expect(await queue.getTeamPendingCount(owner)).toBe(0);
+ }, 120_000);
+
+ test("parallel enqueuers + workers with a crawl gate stay within both limits", async () => {
+ const name = `t-${RUN}-stress2`;
+ queueNames.push(name);
+ const queue: NuQFdbQueue = new NuQFdbQueue(name, {
+ hasGroups: true,
+ finishedQueueName: `${name}-fin`,
+ });
+ queueNames.push(`${name}-fin`);
+ const finishedQueue: NuQFdbQueue = new NuQFdbQueue(`${name}-fin`, {
+ hasGroups: false,
+ });
+ const group = new NuQFdbJobGroup(queue.ks, queue.groupOps!);
+
+ const owner = randomUUID();
+ const gid = randomUUID();
+ const CRAWL_LIMIT = 3;
+ const TEAM_LIMIT = 10;
+ const JOBS = 40;
+ const gate = { teamLimit: TEAM_LIMIT, queueCap: 1_000_000 };
+ await group.addGroup(gid, owner, undefined, {
+ maxConcurrency: CRAWL_LIMIT,
+ });
+
+ // enqueue from 4 parallel producers while workers consume
+ const producers = Array.from({ length: 4 }, (_, p) =>
+ queue.addJobs(
+ Array.from({ length: JOBS / 4 }, () => ({
+ id: randomUUID(),
+ data: { mode: "single_urls", url: "https://example.com" },
+ options: {
+ ownerId: owner,
+ groupId: gid,
+ timesOutAt: new Date(Date.now() + 120_000),
+ },
+ })),
+ gate,
+ ),
+ );
+
+ let inFlight = 0;
+ let maxInFlight = 0;
+ let processed = 0;
+ const worker = async () => {
+ let idleStrikes = 0;
+ while (idleStrikes < 10) {
+ const job = await queue.getJobToProcess();
+ if (!job) {
+ idleStrikes++;
+ await new Promise(resolve => setTimeout(resolve, 50));
+ continue;
+ }
+ idleStrikes = 0;
+ inFlight++;
+ maxInFlight = Math.max(maxInFlight, inFlight);
+ await new Promise(resolve =>
+ setTimeout(resolve, 5 + Math.random() * 20),
+ );
+ inFlight--;
+ await queue.jobFinish(job.id, job.lock!, null);
+ processed++;
+ }
+ };
+
+ const [results] = await Promise.all([
+ Promise.all(producers),
+ ...Array.from({ length: 6 }, () => worker()),
+ ]);
+ expect(results.flat().length).toBe(JOBS);
+ expect(processed).toBe(JOBS);
+ expect(maxInFlight).toBeLessThanOrEqual(CRAWL_LIMIT);
+
+ // the group drained, so it must have completed and emitted exactly one
+ // crawl_finished job even under concurrent finishers
+ const g = await group.getGroup(gid);
+ expect(g?.status).toBe("completed");
+ const fin = await finishedQueue.getJobToProcess();
+ expect(fin).not.toBeNull();
+ expect(await finishedQueue.getJobToProcess()).toBeNull();
+
+ const stats = await queue.getGroupNumericStats(gid);
+ expect(stats.completed).toBe(JOBS);
+ expect(stats.active).toBe(0);
+ expect(stats.queued).toBe(0);
+ expect(stats.backlog).toBe(0);
+ }, 120_000);
+});
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index ac6c4bacc..ca83f0f68 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -9,6 +9,12 @@ const delimitedList = (separator = ",") => {
});
};
+const emptyStringAsUndefined = <T extends z.ZodTypeAny>(schema: T) =>
+ z.preprocess(value => (value === "" ? undefined : value), schema.optional());
+
+const emptyStringAsDefault = <T extends z.ZodTypeAny>(schema: T) =>
+ z.preprocess(value => (value === "" ? undefined : value), schema);
+
// Ethereum address schema: validates 0x followed by 40 hex characters
const ethereumAddress = z
.string()
@@ -96,6 +102,19 @@ const configSchema = z.object({
NUQ_DATABASE_URL: z.string().optional(),
NUQ_DATABASE_URL_LISTEN: z.string().optional(),
NUQ_RABBITMQ_URL: z.string().optional(),
+ FDB_CLUSTER_FILE: emptyStringAsUndefined(z.string()),
+ NUQ_BACKEND: emptyStringAsUndefined(z.enum(["pg", "fdb"])),
+ NUQ_FDB_READY_SHARDS: emptyStringAsDefault(
+ z.coerce.number().int().positive().default(2048),
+ ),
+ // 1 = strict (priority, FIFO) promotion order per team; raise for teams with
+ // extreme finish rates at the cost of approximate cross-shard ordering
+ NUQ_FDB_TEAM_PENDING_SHARDS: emptyStringAsDefault(
+ z.coerce.number().int().positive().default(1),
+ ),
+ NUQ_FDB_TIME_BUCKETS: emptyStringAsDefault(
+ z.coerce.number().int().positive().default(16),
+ ),
// Google Cloud Storage
GCS_BUCKET_NAME: z.string().optional(),
diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.ts
index 86d116c29..0e43c324b 100644
--- a/apps/api/src/controllers/auth.ts
+++ b/apps/api/src/controllers/auth.ts
@@ -1,6 +1,6 @@
import * as crypto from "crypto";
import { RateLimiterRedis } from "rate-limiter-flexible";
-import { validate } from "uuid";
+import { isValidUuid } from "../lib/owner-id";
import { config } from "../config";
import { logger } from "../lib/logger";
import { parseApi } from "../lib/parseApi";
@@ -23,7 +23,7 @@ import { AuthCreditUsageChunk, AuthCreditUsageChunkFromTeam } from "./v1/types";
function normalizedApiIsUuid(potentialUuid: string): boolean {
// Check if the string is a valid UUID
- return validate(potentialUuid);
+ return isValidUuid(potentialUuid);
}
export async function setCachedACUC(
diff --git a/apps/api/src/controllers/v0/admin/cclog.ts b/apps/api/src/controllers/v0/admin/cclog.ts
index d77159a34..a2303bb51 100644
--- a/apps/api/src/controllers/v0/admin/cclog.ts
+++ b/apps/api/src/controllers/v0/admin/cclog.ts
@@ -3,15 +3,19 @@ import { db } from "../../../db/connection";
import * as schema from "../../../db/schema";
import { logger as _logger } from "../../../lib/logger";
import { Request, Response } from "express";
+import { scrapeQueueFdb } from "../../../services/worker/nuq-fdb";
+import { fdbQueueEnabled } from "../../../services/worker/nuq-router";
async function cclog() {
const logger = _logger.child({
module: "cclog",
});
+ const concurrencyByTeam = new Map<string, number>();
+ const redis = getRedisConnection();
let cursor = 0;
do {
- const result = await getRedisConnection().scan(
+ const result = await redis.scan(
cursor,
"MATCH",
"concurrency-limiter:*",
@@ -24,41 +28,47 @@ async function cclog() {
logger.info("Stepped", { cursor, usable: usable.length });
if (usable.length > 0) {
- const entries: {
- team_id: string;
- concurrency: number;
- created_at: Date;
- }[] = [];
-
for (const x of usable) {
- const at = new Date();
- const concurrency = await getRedisConnection().zrangebyscore(
- x,
- Date.now(),
- Infinity,
+ const concurrency = await redis.zrangebyscore(x, Date.now(), Infinity);
+ const teamId = x.split(":")[1];
+ concurrencyByTeam.set(
+ teamId,
+ (concurrencyByTeam.get(teamId) ?? 0) + concurrency.length,
);
- if (concurrency) {
- entries.push({
- team_id: x.split(":")[1],
- concurrency: concurrency.length,
- created_at: at,
- });
- }
}
+ }
+ } while (cursor != 0);
- try {
- await db.insert(schema.concurrency_log).values(
- entries.map(e => ({
- team_id: e.team_id,
- concurrency: e.concurrency,
- created_at: e.created_at.toISOString(),
- })),
+ if (fdbQueueEnabled()) {
+ try {
+ const fdbCounts = await scrapeQueueFdb.getTeamActiveCounts();
+ for (const [teamId, concurrency] of fdbCounts) {
+ concurrencyByTeam.set(
+ teamId,
+ (concurrencyByTeam.get(teamId) ?? 0) + concurrency,
);
- } catch (e) {
- logger.error("Error inserting", { error: e });
}
+ } catch (e) {
+ logger.warn("Error reading FDB concurrency", { error: e });
}
- } while (cursor != 0);
+ }
+
+ const at = new Date();
+ const entries = Array.from(concurrencyByTeam.entries()).map(
+ ([team_id, concurrency]) => ({
+ team_id,
+ concurrency,
+ created_at: at.toISOString(),
+ }),
+ );
+
+ if (entries.length === 0) return;
+
+ try {
+ await db.insert(schema.concurrency_log).values(entries);
+ } catch (e) {
+ logger.error("Error inserting", { error: e });
+ }
}
export async function cclogController(req: Request, res: Response) {
diff --git a/apps/api/src/controllers/v0/admin/crawl-monitor.ts b/apps/api/src/controllers/v0/admin/crawl-monitor.ts
index c2a6d6e28..a1a9c4682 100644
--- a/apps/api/src/controllers/v0/admin/crawl-monitor.ts
+++ b/apps/api/src/controllers/v0/admin/crawl-monitor.ts
@@ -15,7 +15,10 @@ import {
StoredCrawl,
} from "../../../lib/crawl-redis";
import { config } from "../../../config";
-import { crawlGroup } from "../../../services/worker/nuq";
+import {
+ crawlGroup,
+ resolveNewGroupBackend,
+} from "../../../services/worker/nuq-router";
import { _addScrapeJobToBullMQ } from "../../../services/queue-jobs";
type ResponseType = {
@@ -77,10 +80,16 @@ export async function crawlMonitorController(
});
}
+ sc.queueBackend = await resolveNewGroupBackend(sc.team_id);
await crawlGroup.addGroup(
id,
sc.team_id,
(req.acuc?.flags?.crawlTtlHours ?? 24) * 60 * 60 * 1000,
+ {
+ backend: sc.queueBackend,
+ maxConcurrency: sc.maxConcurrency,
+ delaySeconds: sc.crawlerOptions?.delay,
+ },
);
await saveCrawl(id, sc);
diff --git a/apps/api/src/controllers/v0/admin/metrics.ts b/apps/api/src/controllers/v0/admin/metrics.ts
index b64a839f0..3866a6a5c 100644
--- a/apps/api/src/controllers/v0/admin/metrics.ts
+++ b/apps/api/src/controllers/v0/admin/metrics.ts
@@ -1,6 +1,7 @@
import type { Request, Response } from "express";
import { getRedisConnection } from "../../../services/queue-service";
-import { nuqGetLocalMetrics, scrapeQueue } from "../../../services/worker/nuq";
+import { nuqGetLocalMetrics } from "../../../services/worker/nuq";
+import { scrapeQueue } from "../../../services/worker/nuq-router";
import { teamConcurrencySemaphore } from "../../../services/worker/team-semaphore";
export async function metricsController(_: Request, res: Response) {
diff --git a/apps/api/src/controllers/v0/crawl-cancel.ts b/apps/api/src/controllers/v0/crawl-cancel.ts
index b091009f1..49617bbf6 100644
--- a/apps/api/src/controllers/v0/crawl-cancel.ts
+++ b/apps/api/src/controllers/v0/crawl-cancel.ts
@@ -6,7 +6,7 @@ import { getCrawl, saveCrawl } from "../../../src/lib/crawl-redis";
import * as Sentry from "@sentry/node";
import { configDotenv } from "dotenv";
import { redisEvictConnection } from "../../../src/services/redis";
-import { crawlGroup } from "../../services/worker/nuq";
+import { crawlGroup } from "../../services/worker/nuq-router";
import { getScrapeZDR } from "../../lib/zdr-helpers";
import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";
configDotenv();
@@ -73,6 +73,10 @@ export async function crawlCancelController(req: Request, res: Response) {
logger.error(error);
}
+ if (sc.queueBackend === "fdb") {
+ await crawlGroup.cancelGroup(req.params.jobId);
+ }
+
res.json({
status: "cancelled",
});
diff --git a/apps/api/src/controllers/v0/crawl-status.ts b/apps/api/src/controllers/v0/crawl-status.ts
index 0ce22a4af..f63525bf2 100644
--- a/apps/api/src/controllers/v0/crawl-status.ts
+++ b/apps/api/src/controllers/v0/crawl-status.ts
@@ -11,7 +11,7 @@ import { configDotenv } from "dotenv";
import { toLegacyDocument } from "../v1/types";
import type { DBScrape, PseudoJob } from "../v1/crawl-status";
import { getJobFromGCS } from "../../lib/gcs-jobs";
-import { scrapeQueue, NuQJob } from "../../services/worker/nuq";
+import { scrapeQueue, NuQJob } from "../../services/worker/nuq-router";
import { includesFormat } from "../../lib/format-utils";
import { getScrapeZDR } from "../../lib/zdr-helpers";
import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";
diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.ts
index f532c135b..c75a3ac1c 100644
--- a/apps/api/src/controllers/v0/crawl.ts
+++ b/apps/api/src/controllers/v0/crawl.ts
@@ -33,7 +33,10 @@ import { ZodError } from "zod";
import { UNSUPPORTED_SITE_MESSAGE } from "../../lib/strings";
import { fromV0ScrapeOptions } from "../v2/types";
import { isSelfHosted } from "../../lib/deployment";
-import { crawlGroup } from "../../services/worker/nuq";
+import {
+ crawlGroup,
+ resolveNewGroupBackend,
+} from "../../services/worker/nuq-router";
import { logRequest } from "../../services/logging/log_job";
import { getScrapeZDR } from "../../lib/zdr-helpers";
import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";
@@ -225,10 +228,16 @@ export async function crawlController(req: Request, res: Response) {
sc.robots = await crawler.getRobotsTxt();
} catch (_) {}
+ sc.queueBackend = await resolveNewGroupBackend(sc.team_id);
await crawlGroup.addGroup(
id,
sc.team_id,
(chunk?.flags?.crawlTtlHours ?? 24) * 60 * 60 * 1000,
+ {
+ backend: sc.queueBackend,
+ maxConcurrency: sc.maxConcurrency,
+ delaySeconds: sc.crawlerOptions?.delay,
+ },
);
await saveCrawl(id, sc);
diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.ts
index e4fd44656..cb0b88a09 100644
--- a/apps/api/src/controllers/v0/scrape.ts
+++ b/apps/api/src/controllers/v0/scrape.ts
@@ -22,7 +22,7 @@ import { Document as V0Document } from "./../../lib/entities";
import { UNSUPPORTED_SITE_MESSAGE } from "../../lib/strings";
import { fromV0Combo } from "../v2/types";
import { ScrapeJobTimeoutError } from "../../lib/error";
-import { scrapeQueue } from "../../services/worker/nuq";
+import { scrapeQueue } from "../../services/worker/nuq-router";
import { getErrorContactMessage } from "../../lib/deployment";
import { logRequest } from "../../services/logging/log_job";
import { getScrapeZDR } from "../../lib/zdr-helpers";
diff --git a/apps/api/src/controllers/v0/search.ts b/apps/api/src/controllers/v0/search.ts
index f0b995dbc..105dafe77 100644
--- a/apps/api/src/controllers/v0/search.ts
+++ b/apps/api/src/controllers/v0/search.ts
@@ -22,7 +22,7 @@ import {
} from "../v1/types";
import { fromV0Combo } from "../v2/types";
import { ScrapeJobTimeoutError } from "../../lib/error";
-import { scrapeQueue } from "../../services/worker/nuq";
+import { scrapeQueue } from "../../services/worker/nuq-router";
import { defaultOrigin } from "../../lib/default-values";
import { getSearchZDR } from "../../lib/zdr-helpers";
import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";
diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.ts
index d5bf86d37..39b95f717 100644
--- a/apps/api/src/controllers/v1/batch-scrape.ts
+++ b/apps/api/src/controllers/v1/batch-scrape.ts
@@ -26,7 +26,10 @@ import { UNSUPPORTED_SITE_MESSAGE } from "../../lib/strings";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
import { fromV1ScrapeOptions } from "../v2/types";
import { checkPermissions } from "../../lib/permissions";
-import { crawlGroup } from "../../services/worker/nuq";
+import {
+ crawlGroup,
+ resolveNewGroupBackend,
+} from "../../services/worker/nuq-router";
import { logRequest } from "../../services/logging/log_job";
import { getScrapeZDR } from "../../lib/zdr-helpers";
@@ -163,10 +166,16 @@ export async function batchScrapeController(
};
if (!req.body.appendToId) {
+ sc.queueBackend = await resolveNewGroupBackend(sc.team_id);
await crawlGroup.addGroup(
id,
sc.team_id,
(req.acuc?.flags?.crawlTtlHours ?? 24) * 60 * 60 * 1000,
+ {
+ backend: sc.queueBackend,
+ maxConcurrency: sc.maxConcurrency,
+ delaySeconds: sc.crawlerOptions?.delay,
+ },
);
await saveCrawl(id, sc);
await markCrawlActive(id);
diff --git a/apps/api/src/controllers/v1/concurrency-check.ts b/apps/api/src/controllers/v1/concurrency-check.ts
index b25fedbc9..0824f9e8d 100644
--- a/apps/api/src/controllers/v1/concurrency-check.ts
+++ b/apps/api/src/controllers/v1/concurrency-check.ts
@@ -4,24 +4,18 @@ import {
RequestWithAuth,
} from "./types";
import { Response } from "express";
-import { getRedisConnection } from "../../../src/services/queue-service";
+import { getCombinedTeamActiveCount } from "../../services/worker/nuq-router";
// Basically just middleware and error wrapping
export async function concurrencyCheckController(
req: RequestWithAuth<ConcurrencyCheckParams, undefined, undefined>,
res: Response<ConcurrencyCheckResponse>,
) {
- const concurrencyLimiterKey = "concurrency-limiter:" + req.auth.team_id;
- const now = Date.now();
- const activeJobsOfTeam = await getRedisConnection().zrangebyscore(
- concurrencyLimiterKey,
- now,
- Infinity,
- );
+ const activeJobsOfTeam = await getCombinedTeamActiveCount(req.auth.team_id);
return res.status(200).json({
success: true,
- concurrency: activeJobsOfTeam.length,
+ concurrency: activeJobsOfTeam,
maxConcurrency: req.acuc?.concurrency ?? 0,
});
}
diff --git a/apps/api/src/controllers/v1/crawl-cancel.ts b/apps/api/src/controllers/v1/crawl-cancel.ts
index cd783f824..2e1d80c82 100644
--- a/apps/api/src/controllers/v1/crawl-cancel.ts
+++ b/apps/api/src/controllers/v1/crawl-cancel.ts
@@ -4,7 +4,7 @@ import { getCrawl, getCrawlJobs, saveCrawl } from "../../lib/crawl-redis";
import * as Sentry from "@sentry/node";
import { configDotenv } from "dotenv";
import { RequestWithAuth } from "./types";
-import { crawlGroup } from "../../services/worker/nuq";
+import { crawlGroup } from "../../services/worker/nuq-router";
import { removeConcurrencyLimitedJobs } from "../../lib/concurrency-limit";
configDotenv();
@@ -38,8 +38,12 @@ export async function crawlCancelController(
logger.error(error);
}
- const jobIds = await getCrawlJobs(req.params.jobId);
- await removeConcurrencyLimitedJobs(sc.team_id, jobIds);
+ if (sc.queueBackend === "fdb") {
+ await crawlGroup.cancelGroup(req.params.jobId);
+ } else {
+ const jobIds = await getCrawlJobs(req.params.jobId);
+ await removeConcurrencyLimitedJobs(sc.team_id, jobIds);
+ }
res.json({
status: "cancelled",
diff --git a/apps/api/src/controllers/v1/crawl-errors.ts b/apps/api/src/controllers/v1/crawl-errors.ts
index d89fe4ce0..b8ad56762 100644
--- a/apps/api/src/controllers/v1/crawl-errors.ts
+++ b/apps/api/src/controllers/v1/crawl-errors.ts
@@ -14,7 +14,7 @@ import * as schema from "../../db/schema";
import { logger as _logger } from "../../lib/logger";
import { deserializeTransportableError } from "../../lib/error-serde";
import { TransportableError } from "../../lib/error";
-import { scrapeQueue } from "../../services/worker/nuq";
+import { scrapeQueue } from "../../services/worker/nuq-router";
configDotenv();
export async function crawlErrorsController(
diff --git a/apps/api/src/controllers/v1/crawl-ongoing.ts b/apps/api/src/controllers/v1/crawl-ongoing.ts
index 74f7143be..833ac3160 100644
--- a/apps/api/src/controllers/v1/crawl-ongoing.ts
+++ b/apps/api/src/controllers/v1/crawl-ongoing.ts
@@ -6,7 +6,7 @@ import {
} from "./types";
import { getCrawl } from "../../lib/crawl-redis";
import { configDotenv } from "dotenv";
-import { crawlGroup } from "../../services/worker/nuq";
+import { crawlGroup } from "../../services/worker/nuq-router";
configDotenv();
export async function ongoingCrawlsController(
diff --git a/apps/api/src/controllers/v1/crawl-status-ws.ts b/apps/api/src/controllers/v1/crawl-status-ws.ts
index ffe8439cf..762c25213 100644
--- a/apps/api/src/controllers/v1/crawl-status-ws.ts
+++ b/apps/api/src/controllers/v1/crawl-status-ws.ts
@@ -19,7 +19,7 @@ import {
import { getJobs, PseudoJob } from "./crawl-status";
import * as Sentry from "@sentry/node";
import { getConcurrencyLimitedJobs } from "../../lib/concurrency-limit";
-import { scrapeQueue, NuQJobStatus } from "../../services/worker/nuq";
+import { scrapeQueue, NuQJobStatus } from "../../services/worker/nuq-router";
import { getErrorContactMessage } from "../../lib/deployment";
type ErrorMessage = {
diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.ts
index 9bb4cbe58..bf4fcd82c 100644
--- a/apps/api/src/controllers/v1/crawl-status.ts
+++ b/apps/api/src/controllers/v1/crawl-status.ts
@@ -28,7 +28,7 @@ import {
NuQJob,
NuQJobStatus,
crawlGroup,
-} from "../../services/worker/nuq";
+} from "../../services/worker/nuq-router";
import { ScrapeJobSingleUrls } from "../../types";
configDotenv();
diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.ts
index a2aea48b2..92320a234 100644
--- a/apps/api/src/controllers/v1/crawl.ts
+++ b/apps/api/src/controllers/v1/crawl.ts
@@ -18,7 +18,10 @@ import { _addScrapeJobToBullMQ } from "../../services/queue-jobs";
import { logger as _logger } from "../../lib/logger";
import { fromV1ScrapeOptions } from "../v2/types";
import { checkPermissions } from "../../lib/permissions";
-import { crawlGroup } from "../../services/worker/nuq";
+import {
+ crawlGroup,
+ resolveNewGroupBackend,
+} from "../../services/worker/nuq-router";
import { logRequest } from "../../services/logging/log_job";
import { getScrapeZDR } from "../../lib/zdr-helpers";
@@ -156,10 +159,16 @@ export async function crawlController(
});
}
+ sc.queueBackend = await resolveNewGroupBackend(sc.team_id);
await crawlGroup.addGroup(
id,
sc.team_id,
(req.acuc?.flags?.crawlTtlHours ?? 24) * 60 * 60 * 1000,
+ {
+ backend: sc.queueBackend,
+ maxConcurrency: sc.maxConcurrency,
+ delaySeconds: sc.crawlerOptions?.delay,
+ },
);
await saveCrawl(id, sc);
diff --git a/apps/api/src/controllers/v1/queue-status.ts b/apps/api/src/controllers/v1/queue-status.ts
index dcec60b99..e86f3202c 100644
--- a/apps/api/src/controllers/v1/queue-status.ts
+++ b/apps/api/src/controllers/v1/queue-status.ts
@@ -3,6 +3,9 @@ import { getACUCTeam } from "../auth";
import { AuthCreditUsageChunkFromTeam, RequestWithAuth } from "./types";
import { Response } from "express";
import { getRedisConnection } from "../../services/queue-service";
+import { fdbQueueEnabled } from "../../services/worker/nuq-router";
+import { scrapeQueueFdb } from "../../services/worker/nuq-fdb";
+import { logger } from "../../lib/logger";
import {
cleanOldConcurrencyLimitedJobs,
cleanOldConcurrencyLimitEntries,
@@ -41,11 +44,29 @@ export async function queueStatusController(
}
await cleanOldConcurrencyLimitEntries(req.auth.team_id);
- const activeJobsOfTeam = await getConcurrencyLimitActiveJobsCount(
+ let activeJobsOfTeam = await getConcurrencyLimitActiveJobsCount(
req.auth.team_id,
);
await cleanOldConcurrencyLimitedJobs(req.auth.team_id);
- const queuedJobsOfTeam = await getConcurrencyQueueJobsCount(req.auth.team_id);
+ let queuedJobsOfTeam = await getConcurrencyQueueJobsCount(req.auth.team_id);
+
+ // during the FDB migration a team can have load on both ledgers
+ if (fdbQueueEnabled()) {
+ try {
+ const [fdbActive, fdbPending] = await Promise.all([
+ scrapeQueueFdb.getTeamActiveCount(req.auth.team_id),
+ scrapeQueueFdb.getTeamPendingCount(req.auth.team_id),
+ ]);
+ activeJobsOfTeam += fdbActive;
+ queuedJobsOfTeam += fdbPending;
+ } catch (error) {
+ logger.warn("Failed to read FDB queue counts, falling back to Redis", {
+ module: "queue-status",
+ version: "v1",
+ error,
+ });
+ }
+ }
const mostRecentSuccess = await getRedisConnection().get(
"most-recent-success:" + req.auth.team_id,
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index ff1de3ac8..4b4bd5c9a 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -1311,6 +1311,8 @@ export type TeamFlags = {
searchFeedbackOptOut?: boolean;
researchBeta?: boolean;
highlightsBeta?: boolean;
+ // routes the team's new queue work to the FoundationDB backend
+ nuqFdb?: boolean;
} | null;
export type AuthCreditUsageChunkFromTeam = Omit<
diff --git a/apps/api/src/controllers/v1/x402-search.ts b/apps/api/src/controllers/v1/x402-search.ts
index 606f2254f..cd8d6db79 100644
--- a/apps/api/src/controllers/v1/x402-search.ts
+++ b/apps/api/src/controllers/v1/x402-search.ts
@@ -23,7 +23,7 @@ import { db } from "../../db/connection";
import * as schema from "../../db/schema";
import { fromV1ScrapeOptions } from "../v2/types";
import { ScrapeJobTimeoutError } from "../../lib/error";
-import { scrapeQueue } from "../../services/worker/nuq";
+import { scrapeQueue } from "../../services/worker/nuq-router";
import {
applyZdrScope,
captureExceptionWithZdrCheck,
diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.ts
index 79351421f..440054e45 100644
--- a/apps/api/src/controllers/v2/batch-scrape.ts
+++ b/apps/api/src/controllers/v2/batch-scrape.ts
@@ -26,7 +26,10 @@ import { logger as _logger } from "../../lib/logger";
import { UNSUPPORTED_SITE_MESSAGE } from "../../lib/strings";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
import { checkPermissions } from "../../lib/permissions";
-import { crawlGroup } from "../../services/worker/nuq";
+import {
+ crawlGroup,
+ resolveNewGroupBackend,
+} from "../../services/worker/nuq-router";
import { logRequest } from "../../services/logging/log_job";
import type { BillingMetadata } from "../../services/billing/types";
import { getScrapeZDR } from "../../lib/zdr-helpers";
@@ -51,7 +54,8 @@ export async function batchScrapeController(
}
const zeroDataRetention =
- getScrapeZDR(req.acuc?.flags) === "forced" || (req.body.zeroDataRetention ?? false);
+ getScrapeZDR(req.acuc?.flags) === "forced" ||
+ (req.body.zeroDataRetention ?? false);
if (
req.body.__agentInterop &&
@@ -186,10 +190,16 @@ export async function batchScrapeController(
}
if (!req.body.appendToId) {
+ sc.queueBackend = await resolveNewGroupBackend(sc.team_id);
await crawlGroup.addGroup(
id,
sc.team_id,
(req.acuc?.flags?.crawlTtlHours ?? 24) * 60 * 60 * 1000,
+ {
+ backend: sc.queueBackend,
+ maxConcurrency: sc.maxConcurrency,
+ delaySeconds: sc.crawlerOptions?.delay,
+ },
);
await saveCrawl(id, sc);
await markCrawlActive(id);
diff --git a/apps/api/src/controllers/v2/browser.ts b/apps/api/src/controllers/v2/browser.ts
index 256ebf22f..ae15531f5 100644
--- a/apps/api/src/controllers/v2/browser.ts
+++ b/apps/api/src/controllers/v2/browser.ts
@@ -18,10 +18,10 @@ import {
clearBrowserSessionPromptFlag,
} from "../../lib/browser-sessions";
import {
- getConcurrencyLimitActiveJobsCount,
- pushConcurrencyLimitActiveJob,
- removeConcurrencyLimitActiveJob,
-} from "../../lib/concurrency-limit";
+ getCombinedTeamActiveCount,
+ mirrorExternalSlotAcquire,
+ mirrorExternalSlotRelease,
+} from "../../services/worker/nuq-router";
import { RequestWithAuth } from "./types";
import { billTeam } from "../../services/billing/credit_billing";
import { enqueueBrowserSessionActivity } from "../../lib/browser-session-activity";
@@ -245,9 +245,7 @@ export async function browserCreateController(
// 0b. Enforce concurrency limit (shared pool with scrape/crawl/interact)
const concurrencyLimit = req.acuc?.concurrency ?? 2;
- const activeCount = await getConcurrencyLimitActiveJobsCount(
- req.auth.team_id,
- );
+ const activeCount = await getCombinedTeamActiveCount(req.auth.team_id);
if (activeCount >= concurrencyLimit) {
logger.warn("Concurrency limit reached for browser session", {
activeCount,
@@ -374,7 +372,7 @@ export async function browserCreateController(
// Register in the shared concurrency limiter so this session counts
// against the team's concurrent job limit while it's active.
- pushConcurrencyLimitActiveJob(req.auth.team_id, sessionId, ttl * 1000).catch(
+ mirrorExternalSlotAcquire(req.auth.team_id, sessionId, ttl * 1000).catch(
() => {},
);
@@ -554,7 +552,7 @@ export async function browserDeleteController(
// Invalidate cached count so next check reflects the destroyed session
invalidateActiveBrowserSessionCount(session.team_id).catch(() => {});
- removeConcurrencyLimitActiveJob(session.team_id, session.id).catch(error => {
+ mirrorExternalSlotRelease(session.team_id, session.id).catch(error => {
logger.error(
"Failed to remove concurrency limiter entry for browser session",
{
@@ -701,7 +699,7 @@ export async function browserWebhookDestroyedController(
const claimed = await claimBrowserSessionDestroyed(session.id);
invalidateActiveBrowserSessionCount(session.team_id).catch(() => {});
- removeConcurrencyLimitActiveJob(session.team_id, session.id).catch(error => {
+ mirrorExternalSlotRelease(session.team_id, session.id).catch(error => {
logger.error(
"Failed to remove concurrency limiter entry for browser session via webhook",
{
diff --git a/apps/api/src/controllers/v2/concurrency-check.ts b/apps/api/src/controllers/v2/concurrency-check.ts
index 4431ac41f..36017e8e9 100644
--- a/apps/api/src/controllers/v2/concurrency-check.ts
+++ b/apps/api/src/controllers/v2/concurrency-check.ts
@@ -5,9 +5,9 @@ import {
} from "./types";
import { AuthCreditUsageChunkFromTeam } from "../v1/types";
import { Response } from "express";
-import { getRedisConnection } from "../../../src/services/queue-service";
import { getACUCTeam } from "../auth";
import { RateLimiterMode } from "../../types";
+import { getCombinedTeamActiveCount } from "../../services/worker/nuq-router";
// Basically just middleware and error wrapping
export async function concurrencyCheckController(
@@ -38,17 +38,11 @@ export async function concurrencyCheckController(
);
}
- const concurrencyLimiterKey = "concurrency-limiter:" + req.auth.team_id;
- const now = Date.now();
- const activeJobsOfTeam = await getRedisConnection().zrangebyscore(
- concurrencyLimiterKey,
- now,
- Infinity,
- );
+ const activeJobsOfTeam = await getCombinedTeamActiveCount(req.auth.team_id);
return res.status(200).json({
success: true,
- concurrency: activeJobsOfTeam.length,
+ concurrency: activeJobsOfTeam,
maxConcurrency: Math.max(req.acuc.concurrency, otherACUC?.concurrency ?? 0),
});
}
diff --git a/apps/api/src/controllers/v2/crawl-cancel.ts b/apps/api/src/controllers/v2/crawl-cancel.ts
index 86dfc5123..f3cfe561a 100644
--- a/apps/api/src/controllers/v2/crawl-cancel.ts
+++ b/apps/api/src/controllers/v2/crawl-cancel.ts
@@ -9,7 +9,8 @@ import {
import * as Sentry from "@sentry/node";
import { configDotenv } from "dotenv";
import { RequestWithAuth, scrapeOptions } from "./types";
-import { crawlGroup, normalizeOwnerId } from "../../services/worker/nuq";
+import { crawlGroup } from "../../services/worker/nuq-router";
+import { normalizeOwnerId } from "../../lib/owner-id";
import { removeConcurrencyLimitedJobs } from "../../lib/concurrency-limit";
configDotenv();
@@ -50,8 +51,12 @@ export async function crawlCancelController(
logger.error(error);
}
- const jobIds = await getCrawlJobs(req.params.jobId);
- await removeConcurrencyLimitedJobs(sc.team_id, jobIds);
+ if (sc.queueBackend === "fdb") {
+ await crawlGroup.cancelGroup(req.params.jobId);
+ } else {
+ const jobIds = await getCrawlJobs(req.params.jobId);
+ await removeConcurrencyLimitedJobs(sc.team_id, jobIds);
+ }
res.json({
status: "cancelled",
diff --git a/apps/api/src/controllers/v2/crawl-errors.ts b/apps/api/src/controllers/v2/crawl-errors.ts
index d89fe4ce0..b8ad56762 100644
--- a/apps/api/src/controllers/v2/crawl-errors.ts
+++ b/apps/api/src/controllers/v2/crawl-errors.ts
@@ -14,7 +14,7 @@ import * as schema from "../../db/schema";
import { logger as _logger } from "../../lib/logger";
import { deserializeTransportableError } from "../../lib/error-serde";
import { TransportableError } from "../../lib/error";
-import { scrapeQueue } from "../../services/worker/nuq";
+import { scrapeQueue } from "../../services/worker/nuq-router";
configDotenv();
export async function crawlErrorsController(
diff --git a/apps/api/src/controllers/v2/crawl-ongoing.ts b/apps/api/src/controllers/v2/crawl-ongoing.ts
index a27771da3..d22e45ecd 100644
--- a/apps/api/src/controllers/v2/crawl-ongoing.ts
+++ b/apps/api/src/controllers/v2/crawl-ongoing.ts
@@ -6,7 +6,7 @@ import {
} from "./types";
import { getCrawl } from "../../lib/crawl-redis";
import { configDotenv } from "dotenv";
-import { crawlGroup } from "../../services/worker/nuq";
+import { crawlGroup } from "../../services/worker/nuq-router";
configDotenv();
export async function ongoingCrawlsController(
diff --git a/apps/api/src/controllers/v2/crawl-status-ws.ts b/apps/api/src/controllers/v2/crawl-status-ws.ts
index f5c4113d6..b44137223 100644
--- a/apps/api/src/controllers/v2/crawl-status-ws.ts
+++ b/apps/api/src/controllers/v2/crawl-status-ws.ts
@@ -20,7 +20,7 @@ import {
import { getJobs, PseudoJob } from "./crawl-status";
import * as Sentry from "@sentry/node";
import { getConcurrencyLimitedJobs } from "../../lib/concurrency-limit";
-import { scrapeQueue, NuQJobStatus } from "../../services/worker/nuq";
+import { scrapeQueue, NuQJobStatus } from "../../services/worker/nuq-router";
import { getErrorContactMessage } from "../../lib/deployment";
type ErrorMessage = {
diff --git a/apps/api/src/controllers/v2/crawl-status.ts b/apps/api/src/controllers/v2/crawl-status.ts
index ba8b6f677..cae21d208 100644
--- a/apps/api/src/controllers/v2/crawl-status.ts
+++ b/apps/api/src/controllers/v2/crawl-status.ts
@@ -29,7 +29,7 @@ import {
NuQJob,
NuQJobStatus,
crawlGroup,
-} from "../../services/worker/nuq";
+} from "../../services/worker/nuq-router";
import { ScrapeJobSingleUrls } from "../../types";
import { redisEvictConnection } from "../../../src/services/redis";
import { isBaseDomain, extractBaseDomain } from "../../lib/url-utils";
diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.ts
index 91de703ed..79d5c1497 100644
--- a/apps/api/src/controllers/v2/crawl.ts
+++ b/apps/api/src/controllers/v2/crawl.ts
@@ -20,7 +20,10 @@ import { generateCrawlerOptionsFromPrompt } from "../../scraper/scrapeURL/transf
import { CostTracking } from "../../lib/cost-tracking";
import { checkPermissions } from "../../lib/permissions";
import { buildPromptWithWebsiteStructure } from "../../lib/map-utils";
-import { crawlGroup } from "../../services/worker/nuq";
+import {
+ crawlGroup,
+ resolveNewGroupBackend,
+} from "../../services/worker/nuq-router";
import { logRequest } from "../../services/logging/log_job";
import { getScrapeZDR } from "../../lib/zdr-helpers";
@@ -215,10 +218,16 @@ export async function crawlController(
});
}
+ sc.queueBackend = await resolveNewGroupBackend(sc.team_id);
await crawlGroup.addGroup(
id,
sc.team_id,
(req.acuc?.flags?.crawlTtlHours ?? 24) * 60 * 60 * 1000,
+ {
+ backend: sc.queueBackend,
+ maxConcurrency: sc.maxConcurrency,
+ delaySeconds: sc.crawlerOptions?.delay,
+ },
);
await saveCrawl(id, sc);
diff --git a/apps/api/src/controllers/v2/queue-status.ts b/apps/api/src/controllers/v2/queue-status.ts
index 1322e544d..ef9d383ec 100644
--- a/apps/api/src/controllers/v2/queue-status.ts
+++ b/apps/api/src/controllers/v2/queue-status.ts
@@ -4,6 +4,9 @@ import { RequestWithAuth } from "./types";
import { AuthCreditUsageChunkFromTeam } from "../v1/types";
import { Response } from "express";
import { getRedisConnection } from "../../services/queue-service";
+import { fdbQueueEnabled } from "../../services/worker/nuq-router";
+import { scrapeQueueFdb } from "../../services/worker/nuq-fdb";
+import { logger } from "../../lib/logger";
import {
cleanOldConcurrencyLimitedJobs,
cleanOldConcurrencyLimitEntries,
@@ -42,11 +45,29 @@ export async function queueStatusController(
}
await cleanOldConcurrencyLimitEntries(req.auth.team_id);
- const activeJobsOfTeam = await getConcurrencyLimitActiveJobsCount(
+ let activeJobsOfTeam = await getConcurrencyLimitActiveJobsCount(
req.auth.team_id,
);
await cleanOldConcurrencyLimitedJobs(req.auth.team_id);
- const queuedJobsOfTeam = await getConcurrencyQueueJobsCount(req.auth.team_id);
+ let queuedJobsOfTeam = await getConcurrencyQueueJobsCount(req.auth.team_id);
+
+ // during the FDB migration a team can have load on both ledgers
+ if (fdbQueueEnabled()) {
+ try {
+ const [fdbActive, fdbPending] = await Promise.all([
+ scrapeQueueFdb.getTeamActiveCount(req.auth.team_id),
+ scrapeQueueFdb.getTeamPendingCount(req.auth.team_id),
+ ]);
+ activeJobsOfTeam += fdbActive;
+ queuedJobsOfTeam += fdbPending;
+ } catch (error) {
+ logger.warn("Failed to read FDB queue counts, falling back to Redis", {
+ module: "queue-status",
+ version: "v2",
+ error,
+ });
+ }
+ }
const mostRecentSuccess = await getRedisConnection().get(
"most-recent-success:" + req.auth.team_id,
diff --git a/apps/api/src/controllers/v2/scrape-browser.ts b/apps/api/src/controllers/v2/scrape-browser.ts
index fc8e11cfc..08f46feb4 100644
--- a/apps/api/src/controllers/v2/scrape-browser.ts
+++ b/apps/api/src/controllers/v2/scrape-browser.ts
@@ -17,11 +17,12 @@ import {
didBrowserSessionUsePrompt,
clearBrowserSessionPromptFlag,
} from "../../lib/browser-sessions";
+import {} from "../../lib/concurrency-limit";
import {
- getConcurrencyLimitActiveJobsCount,
- pushConcurrencyLimitActiveJob,
- removeConcurrencyLimitActiveJob,
-} from "../../lib/concurrency-limit";
+ getCombinedTeamActiveCount,
+ mirrorExternalSlotAcquire,
+ mirrorExternalSlotRelease,
+} from "../../services/worker/nuq-router";
import {
browserServiceRequest,
BrowserServiceError,
@@ -402,7 +403,7 @@ export async function scrapeStopInteractiveBrowserController(
const claimed = await claimBrowserSessionDestroyed(session.id);
invalidateActiveBrowserSessionCount(session.team_id).catch(() => {});
- removeConcurrencyLimitActiveJob(session.team_id, session.id).catch(error => {
+ mirrorExternalSlotRelease(session.team_id, session.id).catch(error => {
logger.error(
"Failed to remove concurrency limiter entry for browser session",
{
@@ -533,9 +534,7 @@ async function createSessionForScrape(
// Active session limit — uses the same concurrency pool as scrape/crawl
const concurrencyLimit = req.acuc?.concurrency ?? 2;
- const activeCount = await getConcurrencyLimitActiveJobsCount(
- req.auth.team_id,
- );
+ const activeCount = await getCombinedTeamActiveCount(req.auth.team_id);
if (activeCount >= concurrencyLimit) {
return {
status: 429,
@@ -738,11 +737,9 @@ async function createSessionForScrape(
// Register in the shared concurrency limiter so this session counts
// against the team's concurrent job limit while it's active.
- pushConcurrencyLimitActiveJob(
- req.auth.team_id,
- sessionId,
- ttl * 1000,
- ).catch(() => {});
+ mirrorExternalSlotAcquire(req.auth.team_id, sessionId, ttl * 1000).catch(
+ () => {},
+ );
return { session };
} catch (err) {
diff --git a/apps/api/src/controllers/v2/x402-search.ts b/apps/api/src/controllers/v2/x402-search.ts
index 1f6114373..493ee9736 100644
--- a/apps/api/src/controllers/v2/x402-search.ts
+++ b/apps/api/src/controllers/v2/x402-search.ts
@@ -23,7 +23,7 @@ import { db } from "../../db/connection";
import * as schema from "../../db/schema";
import { SearchV2Response } from "../../lib/entities";
import { ScrapeJobTimeoutError } from "../../lib/error";
-import { scrapeQueue } from "../../services/worker/nuq";
+import { scrapeQueue } from "../../services/worker/nuq-router";
import { z } from "zod";
import {
buildSearchQuery,
diff --git a/apps/api/src/harness.ts b/apps/api/src/harness.ts
index 51e5c1822..f5a778f29 100644
--- a/apps/api/src/harness.ts
+++ b/apps/api/src/harness.ts
@@ -19,6 +19,10 @@ let nuqRabbitMQContainer: {
containerName: string;
containerRuntime: string;
} | null = null;
+let fdbContainer: {
+ containerName: string;
+ containerRuntime: string;
+} | null = null;
// Get the monorepo root for both tsx source execution and compiled dist execution.
// __dirname is available in CommonJS (which this compiles to)
@@ -53,6 +57,10 @@ interface Services {
containerName: string;
containerRuntime: string;
};
+ fdb?: {
+ containerName: string;
+ containerRuntime: string;
+ };
}
const colors = {
@@ -675,6 +683,96 @@ async function setupNuqRabbitMQ(): Promise<Services["nuqRabbitMQ"]> {
return containerInfo;
}
+const FDB_IMAGE = "foundationdb/foundationdb:7.3.63";
+
+async function waitForFdb(
+ runtime: string,
+ containerName: string,
+ timeoutMs: number = config.HARNESS_STARTUP_TIMEOUT_MS,
+): Promise<void> {
+ logger.info("Waiting for FoundationDB to be ready...");
+ const start = Date.now();
+
+ while (Date.now() - start < timeoutMs) {
+ try {
+ const check = execForward(
+ `${runtime}@fdb-status`,
+ `${runtime} exec ${containerName} fdbcli --exec "status minimal"`,
+ );
+ await check.promise;
+ logger.success("FoundationDB is ready");
+ return;
+ } catch (e) {
+ await new Promise(resolve => setTimeout(resolve, 500));
+ }
+ }
+
+ throw new Error(`FoundationDB did not become ready within ${timeoutMs}ms`);
+}
+
+async function setupFdb(): Promise<Services["fdb"]> {
+ // FDB is only needed when the FDB queue backend is in play
+ if (config.NUQ_BACKEND !== "fdb" && !process.env.NUQ_FDB_TESTS) {
+ return undefined;
+ }
+
+ // If FDB_CLUSTER_FILE is already set, respect it (user's explicit choice)
+ if (config.FDB_CLUSTER_FILE) {
+ logger.info("FDB_CLUSTER_FILE is set, skipping container management");
+ return undefined;
+ }
+
+ logger.section("Setting up FoundationDB container");
+
+ const runtime = await detectContainerRuntime();
+ if (!runtime) {
+ throw new Error(
+ "Neither Docker nor Podman found. Please install Docker/Podman or set FDB_CLUSTER_FILE manually.",
+ );
+ }
+
+ logger.success(`Using container runtime: ${runtime}`);
+
+ const containerName = "firecrawl-fdb";
+ await stopAndRemoveContainer(runtime, containerName);
+
+ const start = execForward(
+ `${runtime}@start`,
+ `${runtime} run -d --name ${containerName} -p 4500:4500 -e FDB_NETWORKING_MODE=host ${FDB_IMAGE}`,
+ );
+ await start.promise;
+
+ await waitForFdb(runtime, containerName);
+
+ // first boot: create the database; only suppress the known "already
+ // configured" failure and surface any real configure error.
+ const configure = execForward(
+ `${runtime}@fdb-configure`,
+ `${runtime} exec ${containerName} sh -c 'out=$(fdbcli --exec "configure new single memory" 2>&1); status=$?; printf "%s\\n" "$out"; if [ "$status" -eq 0 ]; then exit 0; fi; printf "%s\\n" "$out" | grep -Eiq "already.*configured|database.*configured"'`,
+ );
+ await configure.promise;
+
+ // the host-side client needs a cluster file pointing at the published port
+ const { writeFileSync, mkdirSync } = await import("fs");
+ const { tmpdir } = await import("os");
+ const clusterDir = join(tmpdir(), "firecrawl-fdb");
+ mkdirSync(clusterDir, { recursive: true });
+ const clusterFile = join(clusterDir, "fdb.cluster");
+ writeFileSync(clusterFile, "docker:[email protected]:4500");
+
+ config.FDB_CLUSTER_FILE = clusterFile;
+ process.env.FDB_CLUSTER_FILE = clusterFile;
+
+ logger.success("FoundationDB container is ready");
+
+ const containerInfo = {
+ containerName,
+ containerRuntime: runtime,
+ };
+ fdbContainer = containerInfo;
+ return containerInfo;
+}
+
async function installDependencies() {
logger.section("Installing dependencies");
@@ -724,6 +822,9 @@ async function startServices(command?: string[]): Promise<Services> {
// Setup NUQ RabbitMQ container if needed
const nuqRabbitMQ = await setupNuqRabbitMQ();
+ // Setup FoundationDB container if the FDB queue backend is enabled
+ const fdb = await setupFdb();
+
logger.section("Starting services");
const api = execForward(
@@ -775,30 +876,36 @@ async function startServices(command?: string[]): Promise<Services> {
),
);
- const nuqPrefetchWorker = execForward(
- "nuq-prefetch-worker",
- process.argv[2] === "--start-docker"
- ? "node dist/src/services/worker/nuq-prefetch-worker.js"
- : "pnpm nuq-prefetch-worker:production",
- {
- NUQ_PREFETCH_WORKER_PORT: String(NUQ_PREFETCH_WORKER_PORT),
- NUQ_REDUCE_NOISE: "true",
- NUQ_POD_NAME: "nuq-prefetch-worker-0",
- NUQ_PREFETCH_REPLICAS: String(1),
- },
- );
-
- const nuqReconcilerWorker = execForward(
- "nuq-reconciler",
- process.argv[2] === "--start-docker"
- ? "node dist/src/services/worker/nuq-reconciler-worker.js"
- : "pnpm nuq-reconciler-worker:production",
- {
- NUQ_RECONCILER_WORKER_PORT: String(NUQ_RECONCILER_WORKER_PORT),
- NUQ_REDUCE_NOISE: "true",
- NUQ_POD_NAME: "nuq-reconciler-worker-0",
- },
- );
+ const nuqPrefetchWorker =
+ config.NUQ_BACKEND === "fdb"
+ ? undefined
+ : execForward(
+ "nuq-prefetch-worker",
+ process.argv[2] === "--start-docker"
+ ? "node dist/src/services/worker/nuq-prefetch-worker.js"
+ : "pnpm nuq-prefetch-worker:production",
+ {
+ NUQ_PREFETCH_WORKER_PORT: String(NUQ_PREFETCH_WORKER_PORT),
+ NUQ_REDUCE_NOISE: "true",
+ NUQ_POD_NAME: "nuq-prefetch-worker-0",
+ NUQ_PREFETCH_REPLICAS: String(1),
+ },
+ );
+
+ const nuqReconcilerWorker =
+ config.NUQ_BACKEND === "fdb"
+ ? undefined
+ : execForward(
+ "nuq-reconciler",
+ process.argv[2] === "--start-docker"
+ ? "node dist/src/services/worker/nuq-reconciler-worker.js"
+ : "pnpm nuq-reconciler-worker:production",
+ {
+ NUQ_RECONCILER_WORKER_PORT: String(NUQ_RECONCILER_WORKER_PORT),
+ NUQ_REDUCE_NOISE: "true",
+ NUQ_POD_NAME: "nuq-reconciler-worker-0",
+ },
+ );
const indexWorker = config.USE_DB_AUTHENTICATION
? execForward(
@@ -840,6 +947,7 @@ async function startServices(command?: string[]): Promise<Services> {
command: commandProcess,
nuqPostgres,
nuqRabbitMQ,
+ fdb,
};
}
@@ -1060,6 +1168,17 @@ async function gracefulShutdown() {
nuqRabbitMQContainer = null;
}
+ // Stop and remove FoundationDB container if it was started by harness
+ if (fdbContainer) {
+ logger.info("Stopping FoundationDB container");
+ await stopAndRemoveContainer(
+ fdbContainer.containerRuntime,
+ fdbContainer.containerName,
+ );
+ logger.success("FoundationDB container stopped");
+ fdbContainer = null;
+ }
+
logger.success("All processes terminated");
}
diff --git a/apps/api/src/lib/concurrency-limit.ts b/apps/api/src/lib/concurrency-limit.ts
index 34922fadf..dd332b478 100644
--- a/apps/api/src/lib/concurrency-limit.ts
+++ b/apps/api/src/lib/concurrency-limit.ts
@@ -6,20 +6,22 @@ import { logger } from "./logger";
import { abTestJob } from "../services/ab-test";
import { scrapeQueue, type NuQJob } from "../services/worker/nuq";
export { QueueFullError } from "./queue-full-error";
-
-// min 50k, max 2M, 2000 per concurrent browser
-export function getTeamQueueLimit(concurrencyLimit: number): number {
- return Math.min(Math.max(concurrencyLimit * 2000, 50_000), 2_000_000);
-}
-
-// Upper bound for how long a job may sit in the concurrency-limit backlog.
-// This bounds both the Redis ZSET score and the Postgres `times_out_at`
-// column on `nuq.queue_scrape_backlog`, so the reaper can always evict
-// stale rows. A backlogged crawl job that outlives this window is
-// unrecoverable anyway — its StoredCrawl in Redis (24h TTL) is gone.
-export const MAX_BACKLOG_TIMEOUT_MS = 172800000; // 48h
-
-const constructKey = (team_id: string) => "concurrency-limiter:" + team_id;
+export {
+ getTeamQueueLimit,
+ MAX_BACKLOG_TIMEOUT_MS,
+ getConcurrencyLimitActiveJobsCount,
+ pushConcurrencyLimitActiveJob,
+ removeConcurrencyLimitActiveJob,
+} from "./concurrency-redis";
+import {
+ getTeamQueueLimit,
+ MAX_BACKLOG_TIMEOUT_MS,
+ constructConcurrencyLimitKey,
+ pushConcurrencyLimitActiveJob,
+ removeConcurrencyLimitActiveJob,
+} from "./concurrency-redis";
+
+const constructKey = constructConcurrencyLimitKey;
const constructQueueKey = (team_id: string) =>
"concurrency-limit-queue:" + team_id;
@@ -39,16 +41,6 @@ export async function cleanOldConcurrencyLimitEntries(
);
}
-export async function getConcurrencyLimitActiveJobsCount(
- team_id: string,
-): Promise<number> {
- return await getRedisConnection().zcount(
- constructKey(team_id),
- Date.now(),
- Infinity,
- );
-}
-
export async function getConcurrencyLimitActiveJobs(
team_id: string,
now: number = Date.now(),
@@ -60,22 +52,6 @@ export async function getConcurrencyLimitActiveJobs(
);
}
-export async function pushConcurrencyLimitActiveJob(
- team_id: string,
- id: string,
- timeout: number,
- now: number = Date.now(),
-) {
- await getRedisConnection().zadd(constructKey(team_id), now + timeout, id);
-}
-
-export async function removeConcurrencyLimitActiveJob(
- team_id: string,
- id: string,
-) {
- await getRedisConnection().zrem(constructKey(team_id), id);
-}
-
export async function removeConcurrencyLimitedJobs(
team_id: string,
job_ids: string[],
diff --git a/apps/api/src/lib/concurrency-redis.ts b/apps/api/src/lib/concurrency-redis.ts
new file mode 100644
index 000000000..8b9a419eb
--- /dev/null
+++ b/apps/api/src/lib/concurrency-redis.ts
@@ -0,0 +1,50 @@
+import { getRedisConnection } from "../services/queue-service";
+
+// Redis primitives of the concurrency limiter, split out of
+// concurrency-limit.ts so light consumers (the NuQ dual-backend router) can
+// use them without dragging in the scraper tree via crawl-redis.
+
+// min 50k, max 2M, 2000 per concurrent browser
+export function getTeamQueueLimit(concurrencyLimit: number): number {
+ return Math.min(Math.max(concurrencyLimit * 2000, 50_000), 2_000_000);
+}
+
+// Upper bound for how long a job may sit in the concurrency-limit backlog.
+// This bounds both the Redis ZSET score and the Postgres `times_out_at`
+// column on `nuq.queue_scrape_backlog`, so the reaper can always evict
+// stale rows. A backlogged crawl job that outlives this window is
+// unrecoverable anyway — its StoredCrawl in Redis (24h TTL) is gone.
+export const MAX_BACKLOG_TIMEOUT_MS = 172800000; // 48h
+
+export const constructConcurrencyLimitKey = (team_id: string) =>
+ "concurrency-limiter:" + team_id;
+
+export async function getConcurrencyLimitActiveJobsCount(
+ team_id: string,
+): Promise<number> {
+ return await getRedisConnection().zcount(
+ constructConcurrencyLimitKey(team_id),
+ Date.now(),
+ Infinity,
+ );
+}
+
+export async function pushConcurrencyLimitActiveJob(
+ team_id: string,
+ id: string,
+ timeout: number,
+ now: number = Date.now(),
+) {
+ await getRedisConnection().zadd(
+ constructConcurrencyLimitKey(team_id),
+ now + timeout,
+ id,
+ );
+}
+
+export async function removeConcurrencyLimitActiveJob(
+ team_id: string,
+ id: string,
+) {
+ await getRedisConnection().zrem(constructConcurrencyLimitKey(team_id), id);
+}
diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.ts
index eddbb4e59..0780a7034 100644
--- a/apps/api/src/lib/crawl-redis.ts
+++ b/apps/api/src/lib/crawl-redis.ts
@@ -19,6 +19,9 @@ export type StoredCrawl = {
createdAt: number;
maxConcurrency?: number;
zeroDataRetention?: boolean;
+ // which queue backend this crawl's group + jobs live on; a crawl never
+ // spans backends. Absent on crawls created before the FDB rollout (= pg).
+ queueBackend?: "pg" | "fdb";
};
export async function saveCrawl(id: string, crawl: StoredCrawl) {
diff --git a/apps/api/src/lib/extract/document-scraper.ts b/apps/api/src/lib/extract/document-scraper.ts
index dcd053ea8..1da8e2740 100644
--- a/apps/api/src/lib/extract/document-scraper.ts
+++ b/apps/api/src/lib/extract/document-scraper.ts
@@ -12,7 +12,7 @@ import { addScrapeJob } from "../../services/queue-jobs";
import { getJobPriority } from "../job-priority";
import type { Logger } from "winston";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
-import { scrapeQueue } from "../../services/worker/nuq";
+import { scrapeQueue } from "../../services/worker/nuq-router";
interface ScrapeDocumentOptions {
url: string;
diff --git a/apps/api/src/lib/extract/extraction-service.ts b/apps/api/src/lib/extract/extraction-service.ts
index fab347bba..79261ede3 100644
--- a/apps/api/src/lib/extract/extraction-service.ts
+++ b/apps/api/src/lib/extract/extraction-service.ts
@@ -155,13 +155,11 @@ async function performExtraction(
apiKeyId,
{ endpoint: "extract", jobId: extractId },
logger,
- ).catch(
- error => {
- logger.error(
- `Failed to bill team ${teamId} for ${creditsToBill} credits: ${error}`,
- );
- },
- );
+ ).catch(error => {
+ logger.error(
+ `Failed to bill team ${teamId} for ${creditsToBill} credits: ${error}`,
+ );
+ });
return {
success: false,
@@ -673,13 +671,11 @@ async function performExtraction(
apiKeyId,
{ endpoint: "extract", jobId: extractId },
logger,
- ).catch(
- error => {
- logger.error(
- `Failed to bill team ${teamId} for ${creditsToBill} credits: ${error}`,
- );
- },
- );
+ ).catch(error => {
+ logger.error(
+ `Failed to bill team ${teamId} for ${creditsToBill} credits: ${error}`,
+ );
+ });
return {
success: false,
error: getErrorContactMessage(),
@@ -787,13 +783,11 @@ async function performExtraction(
apiKeyId,
{ endpoint: "extract", jobId: extractId },
logger,
- ).catch(
- error => {
- logger.error(
- `Failed to bill team ${teamId} for thinking tokens: ${error}`,
- );
- },
- );
+ ).catch(error => {
+ logger.error(
+ `Failed to bill team ${teamId} for thinking tokens: ${error}`,
+ );
+ });
return {
success: false,
error: error.message,
@@ -817,13 +811,11 @@ async function performExtraction(
apiKeyId,
{ endpoint: "extract", jobId: extractId },
logger,
- ).catch(
- error => {
- logger.error(
- `Failed to bill team ${teamId} for thinking tokens: ${error}`,
- );
- },
- );
+ ).catch(error => {
+ logger.error(
+ `Failed to bill team ${teamId} for thinking tokens: ${error}`,
+ );
+ });
await logExtract({
id: extractId,
request_id: extractId,
@@ -1011,13 +1003,11 @@ async function performExtraction(
apiKeyId,
{ endpoint: "extract" },
logger,
- ).catch(
- error => {
- logger.error(
- `Failed to bill team ${teamId} for ${creditsToBill} credits: ${error}`,
- );
- },
- );
+ ).catch(error => {
+ logger.error(
+ `Failed to bill team ${teamId} for ${creditsToBill} credits: ${error}`,
+ );
+ });
// Log job with token usage and sources
logger.debug("Logging extract to database", {
@@ -1081,13 +1071,11 @@ async function performExtraction(
apiKeyId,
{ endpoint: "extract" },
logger,
- ).catch(
- error => {
- logger.error(
- `Failed to bill team ${teamId} for ${creditsToBill} credits: ${error}`,
- );
- },
- );
+ ).catch(error => {
+ logger.error(
+ `Failed to bill team ${teamId} for ${creditsToBill} credits: ${error}`,
+ );
+ });
await logExtract({
id: extractId,
request_id: extractId,
diff --git a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts b/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts
index a888aecf1..0b884f62a 100644
--- a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts
+++ b/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts
@@ -11,7 +11,7 @@ import { addScrapeJob } from "../../../services/queue-jobs";
import { getJobPriority } from "../../job-priority";
import type { Logger } from "winston";
import { isUrlBlocked } from "../../../scraper/WebScraper/utils/blocklist";
-import { scrapeQueue } from "../../../services/worker/nuq";
+import { scrapeQueue } from "../../../services/worker/nuq-router";
interface ScrapeDocumentOptions {
url: string;
diff --git a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts b/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts
index 64dc894eb..1ac0ddefe 100644
--- a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts
+++ b/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts
@@ -852,7 +852,14 @@ export async function performExtraction_F0(
const creditsToBill = Math.ceil(tokensToBill / 15);
// Bill team for usage
- billTeam(teamId, subId, creditsToBill, apiKeyId, { endpoint: "extract", jobId: extractId }, logger).catch(error => {
+ billTeam(
+ teamId,
+ subId,
+ creditsToBill,
+ apiKeyId,
+ { endpoint: "extract", jobId: extractId },
+ logger,
+ ).catch(error => {
logger.error(
`Failed to bill team ${teamId} for ${creditsToBill} credits: ${error}`,
);
diff --git a/apps/api/src/lib/fire-privacy-chunker.ts b/apps/api/src/lib/fire-privacy-chunker.ts
index 2dbdf8524..a1ae85768 100644
--- a/apps/api/src/lib/fire-privacy-chunker.ts
+++ b/apps/api/src/lib/fire-privacy-chunker.ts
@@ -75,10 +75,14 @@ export function chunkMarkdown(text: string, opts?: ChunkOptions): Chunk[] {
// Guard against caller bugs that would otherwise leave the cursor parked
// at 0 (maxChars ≤ 0 → tentativeEnd ≤ cursor) and spin forever.
if (maxChars <= 0) {
- throw new RangeError(`chunkMarkdown: maxChars must be > 0 (got ${maxChars})`);
+ throw new RangeError(
+ `chunkMarkdown: maxChars must be > 0 (got ${maxChars})`,
+ );
}
if (maxBytes <= 0) {
- throw new RangeError(`chunkMarkdown: maxBytes must be > 0 (got ${maxBytes})`);
+ throw new RangeError(
+ `chunkMarkdown: maxBytes must be > 0 (got ${maxBytes})`,
+ );
}
const chunks: Chunk[] = [];
diff --git a/apps/api/src/lib/fire-privacy-client.test.ts b/apps/api/src/lib/fire-privacy-client.test.ts
index 951a19753..44227a577 100644
--- a/apps/api/src/lib/fire-privacy-client.test.ts
+++ b/apps/api/src/lib/fire-privacy-client.test.ts
@@ -272,7 +272,12 @@ describe("redactText", () => {
handler = withBody({
redacted_text: "x",
spans: [
- { start: 0, end: 5, kind: "PRIVATE_PERSON", source: "openai-privacy-filter" },
+ {
+ start: 0,
+ end: 5,
+ kind: "PRIVATE_PERSON",
+ source: "openai-privacy-filter",
+ },
{ start: 6, end: 10, kind: "EMAIL_ADDRESS", source: "EmailRecognizer" },
{ start: 11, end: 15, kind: "ORGANIZATION", source: "SpacyRecognizer" },
],
diff --git a/apps/api/src/lib/owner-id.ts b/apps/api/src/lib/owner-id.ts
new file mode 100644
index 000000000..769a60cfc
--- /dev/null
+++ b/apps/api/src/lib/owner-id.ts
@@ -0,0 +1,33 @@
+import { createHash } from "crypto";
+
+// mirrors the uuid package's validate(): RFC 9562 plus the nil/max specials
+const UUID_RE =
+ /^(?:[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$/i;
+
+// RFC 4122 v5 (namespaced SHA-1), matching the uuid package's v5()
+function uuidv5(name: string, namespace: string): string {
+ const ns = Buffer.from(namespace.replace(/-/g, ""), "hex");
+ const hash = createHash("sha1")
+ .update(ns)
+ .update(Buffer.from(name, "utf8"))
+ .digest();
+ const bytes = hash.subarray(0, 16);
+ bytes[6] = (bytes[6] & 0x0f) | 0x50;
+ bytes[8] = (bytes[8] & 0x3f) | 0x80;
+ const hex = bytes.toString("hex");
+ return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
+}
+
+export function isValidUuid(value: string): boolean {
+ return UUID_RE.test(value);
+}
+
+// owner IDs can sometimes be non-UUID, so normalize them to avoid breakage
+const normalizedUUIDNamespace = "0f38e00e-d7ee-4b77-8a7a-a787a3537ca2";
+export function normalizeOwnerId(
+ ownerId: string | undefined | null,
+): string | null {
+ if (typeof ownerId !== "string") return null;
+ if (UUID_RE.test(ownerId)) return ownerId;
+ return uuidv5(ownerId, normalizedUUIDNamespace);
+}
diff --git a/apps/api/src/scraper/WebScraper/__tests__/crawler.test.ts b/apps/api/src/scraper/WebScraper/__tests__/crawler.test.ts
index 0f8a110c2..302396a7e 100644
--- a/apps/api/src/scraper/WebScraper/__tests__/crawler.test.ts
+++ b/apps/api/src/scraper/WebScraper/__tests__/crawler.test.ts
@@ -152,4 +152,32 @@ describe("WebCrawler", () => {
filteredLinks.denialReasons.has("https://sub.example.com/blog"),
).toBe(true);
});
+
+ it("can validate the source URL without applying maxDiscoveryDepth", async () => {
+ const initialUrl = "https://example.com/blog";
+
+ crawler = new WebCrawler({
+ jobId: "TEST",
+ initialUrl,
+ includes: ["^/blog"],
+ excludes: [],
+ limit: 10,
+ maxCrawledDepth: 10,
+ maxDiscoveryDepth: 0,
+ currentDiscoveryDepth: 0,
+ });
+
+ const discoveryResult = await crawler["filterLinks"]([initialUrl], 1, 10);
+ expect(discoveryResult.links).toEqual([]);
+
+ const sourceValidationResult = await crawler["filterLinks"](
+ [initialUrl],
+ 1,
+ 10,
+ false,
+ false,
+ true,
+ );
+ expect(sourceValidationResult.links).toEqual([initialUrl]);
+ });
});
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts
index 28105c72e..3eb1b7b89 100644
--- a/apps/api/src/scraper/WebScraper/crawler.ts
+++ b/apps/api/src/scraper/WebScraper/crawler.ts
@@ -160,10 +160,14 @@ export class WebCrawler {
maxDepth: number,
fromMap: boolean = false,
skipRobots: boolean = false,
+ ignoreDiscoveryDepth: boolean = false,
): Promise<FilterLinksResult> {
const denialReasons = new Map<string, string>();
- if (this.currentDiscoveryDepth === this.maxDiscoveryDepth) {
+ if (
+ !ignoreDiscoveryDepth &&
+ this.currentDiscoveryDepth === this.maxDiscoveryDepth
+ ) {
this.logger.debug("Max discovery depth hit, filtering off all links", {
currentDiscoveryDepth: this.currentDiscoveryDepth,
maxDiscoveryDepth: this.maxDiscoveryDepth,
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 6589f6737..796fe4c87 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -416,19 +416,21 @@ async function buildMetaObject(
}
}
+ const normalizedOptions = {
+ ...options,
+ skipTlsVerification:
+ options.skipTlsVerification ??
+ ((options.headers && Object.keys(options.headers).length > 0) ||
+ (options.actions && options.actions.length > 0)
+ ? false
+ : true),
+ };
+
return {
id,
url,
rewrittenUrl: rewriteUrl(url),
- options: {
- ...options,
- skipTlsVerification:
- options.skipTlsVerification ??
- ((options.headers && Object.keys(options.headers).length > 0) ||
- (options.actions && options.actions.length > 0)
- ? false
- : true),
- },
+ options: normalizedOptions,
internalOptions,
logger,
abortHandle,
@@ -445,7 +447,7 @@ async function buildMetaObject(
}
: undefined,
),
- featureFlags: buildFeatureFlags(url, options, internalOptions),
+ featureFlags: buildFeatureFlags(url, normalizedOptions, internalOptions),
mock:
options.useMock !== undefined
? await loadMock(options.useMock, _logger)
diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.ts
index 3072069d7..19c321d4c 100644
--- a/apps/api/src/services/indexing/index-worker.ts
+++ b/apps/api/src/services/indexing/index-worker.ts
@@ -38,7 +38,7 @@ import {
import { StoredCrawl, crawlToCrawler, saveCrawl } from "../../lib/crawl-redis";
import { _addScrapeJobToBullMQ } from "../queue-jobs";
import { withSpan, setSpanAttributes } from "../../lib/otel-tracer";
-import { crawlGroup } from "../worker/nuq";
+import { crawlGroup, resolveNewGroupBackend } from "../worker/nuq-router";
import { getACUCTeam } from "../../controllers/auth";
import { processEngpickerJob } from "../../lib/engpicker";
import { logRequest } from "../logging/log_job";
@@ -507,6 +507,7 @@ const processPrecrawlJob = async (token: string, job: Job) => {
zeroDataRetention: false,
};
+ sc.queueBackend = await resolveNewGroupBackend(sc.team_id);
await crawlGroup.addGroup(
crawlId,
sc.team_id,
@@ -514,6 +515,11 @@ const processPrecrawlJob = async (token: string, job: Job) => {
60 *
60 *
1000,
+ {
+ backend: sc.queueBackend,
+ maxConcurrency: sc.maxConcurrency,
+ delaySeconds: sc.crawlerOptions?.delay,
+ },
);
await saveCrawl(crawlId, sc);
diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.ts
index d4ba63523..a7e0b2fe6 100644
--- a/apps/api/src/services/monitoring/runner.ts
+++ b/apps/api/src/services/monitoring/runner.ts
@@ -4,7 +4,12 @@ import { logger as _logger } from "../../lib/logger";
import { logRequest } from "../logging/log_job";
import { getMonitorDiffArtifact } from "../../lib/gcs-monitoring";
import { processJobInternal } from "../worker/scrape-worker";
-import { NuQJob, crawlGroup, scrapeQueue } from "../worker/nuq";
+import {
+ NuQJob,
+ crawlGroup,
+ scrapeQueue,
+ resolveNewGroupBackend,
+} from "../worker/nuq-router";
import { ScrapeJobData } from "../../types";
import { getJobFromGCS } from "../../lib/gcs-jobs";
import { includesFormat } from "../../lib/format-utils";
@@ -505,7 +510,12 @@ async function runCrawlTarget(params: {
// Crawls tolerate robots fetch failures in the public controller too.
}
- await crawlGroup.addGroup(crawlId, sc.team_id, 24 * 60 * 60 * 1000);
+ sc.queueBackend = await resolveNewGroupBackend(sc.team_id);
+ await crawlGroup.addGroup(crawlId, sc.team_id, 24 * 60 * 60 * 1000, {
+ backend: sc.queueBackend,
+ maxConcurrency: sc.maxConcurrency,
+ delaySeconds: sc.crawlerOptions?.delay,
+ });
await saveCrawl(crawlId, sc);
await markCrawlActive(crawlId);
@@ -891,7 +901,12 @@ async function enqueueMonitorCrawlTarget(params: {
// Non-fatal, same as the public crawl controller.
}
- await crawlGroup.addGroup(crawlId, sc.team_id, 24 * 60 * 60 * 1000);
+ sc.queueBackend = await resolveNewGroupBackend(sc.team_id);
+ await crawlGroup.addGroup(crawlId, sc.team_id, 24 * 60 * 60 * 1000, {
+ backend: sc.queueBackend,
+ maxConcurrency: sc.maxConcurrency,
+ delaySeconds: sc.crawlerOptions?.delay,
+ });
await saveCrawl(crawlId, sc);
await markCrawlActive(crawlId);
diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.ts
index d933a021d..0e56ae531 100644
--- a/apps/api/src/services/queue-jobs.ts
+++ b/apps/api/src/services/queue-jobs.ts
@@ -25,6 +25,12 @@ import { ScrapeJobTimeoutError, TransportableError } from "../lib/error";
import { deserializeTransportableError } from "../lib/error-serde";
import { abTestJob } from "./ab-test";
import { NuQJob, scrapeQueue } from "./worker/nuq";
+import {
+ fdbEnqueueScrapeJobs,
+ resolveJobBackend,
+ scrapeQueue as routedScrapeQueue,
+} from "./worker/nuq-router";
+import { scrapeQueueFdb } from "./worker/nuq-fdb";
import { serializeTraceContext } from "../lib/otel-tracer";
import { isSelfHosted } from "../lib/deployment";
import { MONITOR_CHECK_STALE_TIMEOUT_MS } from "./monitoring/stale";
@@ -145,6 +151,41 @@ export async function _addScrapeJobToBullMQ(
jobId: string,
priority: number = 0,
listenable: boolean = false,
+): Promise<NuQJob<ScrapeJobData>> {
+ // direct adds bypass the gates; on the FDB backend that's a slotless enqueue
+ if ((await resolveJobBackend(webScraperOptions)) === "fdb") {
+ if (webScraperOptions.mode === "single_urls") {
+ abTestJob(webScraperOptions);
+ }
+ const { jobs } = await fdbEnqueueScrapeJobs(
+ [
+ {
+ jobId,
+ data: webScraperOptions,
+ priority,
+ listenable,
+ backlogTimeoutMs: backlogTimeoutMs(webScraperOptions),
+ },
+ ],
+ webScraperOptions.team_id,
+ { bypassGate: true },
+ );
+ return jobs[0];
+ }
+
+ return _addScrapeJobToBullMQPg(
+ webScraperOptions,
+ jobId,
+ priority,
+ listenable,
+ );
+}
+
+async function _addScrapeJobToBullMQPg(
+ webScraperOptions: ScrapeJobData,
+ jobId: string,
+ priority: number = 0,
+ listenable: boolean = false,
): Promise<NuQJob<ScrapeJobData>> {
if (webScraperOptions.mode === "single_urls") {
abTestJob(webScraperOptions);
@@ -224,6 +265,76 @@ async function _addScrapeJobsToBullMQ(
);
}
+async function addScrapeJobFdb(
+ webScraperOptions: ScrapeJobData,
+ jobId: string,
+ priority: number,
+ directToBullMQ: boolean,
+ listenable: boolean,
+): Promise<NuQJob<ScrapeJobData> | null> {
+ if (webScraperOptions.mode === "single_urls") {
+ abTestJob(webScraperOptions);
+ }
+
+ const { jobs, backloggedCount, teamLimit } = await fdbEnqueueScrapeJobs(
+ [
+ {
+ jobId,
+ data: webScraperOptions,
+ priority,
+ listenable,
+ backlogTimeoutMs: backlogTimeoutMs(webScraperOptions),
+ },
+ ],
+ webScraperOptions.team_id,
+ { bypassGate: directToBullMQ },
+ );
+
+ if (backloggedCount > 0) {
+ await maybeSendConcurrencyNotificationFdb(
+ webScraperOptions.team_id,
+ teamLimit,
+ isCrawlOrBatchScrape(webScraperOptions),
+ );
+ // matches the PG contract: null = job waiting in the concurrency queue
+ return null;
+ }
+ return jobs[0];
+}
+
+// parity with the PG path: notify when the backlog exceeds the team limit
+async function maybeSendConcurrencyNotificationFdb(
+ teamId: string,
+ teamLimit: number | null,
+ crawlOrBatch: boolean,
+) {
+ if (teamLimit === null || crawlOrBatch) return;
+ try {
+ const pending = await scrapeQueueFdb.getTeamPendingCount(teamId);
+ if (pending <= teamLimit) return;
+ const shouldSendNotification =
+ await shouldSendConcurrencyLimitNotification(teamId);
+ if (shouldSendNotification) {
+ sendNotificationWithCustomDays(
+ teamId,
+ NotificationType.CONCURRENCY_LIMIT_REACHED,
+ 15,
+ false,
+ true,
+ ).catch(error => {
+ _logger.error(
+ "Error sending notification (concurrency limit reached)",
+ {
+ error,
+ },
+ );
+ });
+ }
+ } catch (error) {
+ _logger.warn("Failed to check FDB concurrency notification", { error });
+ }
+}
+
async function addScrapeJobRaw(
webScraperOptions: ScrapeJobData,
jobId: string,
@@ -231,6 +342,16 @@ async function addScrapeJobRaw(
directToBullMQ: boolean = false,
listenable: boolean = false,
): Promise<NuQJob<ScrapeJobData> | null> {
+ if ((await resolveJobBackend(webScraperOptions)) === "fdb") {
+ return addScrapeJobFdb(
+ webScraperOptions,
+ jobId,
+ priority,
+ directToBullMQ,
+ listenable,
+ );
+ }
+
let concurrencyLimited: "yes" | "yes-crawl" | "no" | null = null;
let currentActiveConcurrency: number | null = null;
let maxConcurrency = 0;
@@ -357,7 +478,7 @@ async function addScrapeJobRaw(
);
return null;
} else {
- return await _addScrapeJobToBullMQ(
+ return await _addScrapeJobToBullMQPg(
webScraperOptions,
jobId,
priority,
@@ -419,7 +540,49 @@ export async function addScrapeJobs(
jobsByTeam.get(job.data.team_id)!.push(job);
}
- for (const [teamId, teamJobs] of jobsByTeam) {
+ for (const [teamId, allTeamJobs] of jobsByTeam) {
+ // jobs can split across backends mid-migration (old crawls drain on PG
+ // while the team's new crawls run on FDB); partition by job backend
+ const backendByJob = new Map<string, "pg" | "fdb">();
+ const backendByCrawl = new Map<string, "pg" | "fdb">();
+ for (const job of allTeamJobs) {
+ const crawlId = job.data.crawl_id;
+ if (crawlId && backendByCrawl.has(crawlId)) {
+ backendByJob.set(job.jobId, backendByCrawl.get(crawlId)!);
+ continue;
+ }
+ const backend = await resolveJobBackend(job.data);
+ backendByJob.set(job.jobId, backend);
+ if (crawlId) backendByCrawl.set(crawlId, backend);
+ }
+
+ const fdbJobs = allTeamJobs.filter(
+ j => backendByJob.get(j.jobId) === "fdb",
+ );
+ if (fdbJobs.length > 0) {
+ const { backloggedCount, teamLimit } = await fdbEnqueueScrapeJobs(
+ fdbJobs.map(job => ({
+ jobId: job.jobId,
+ data: { ...job.data, traceContext },
+ priority: job.priority,
+ listenable: job.listenable,
+ backlogTimeoutMs: backlogTimeoutMs(job.data),
+ })),
+ teamId,
+ );
+ if (backloggedCount > 0) {
+ await maybeSendConcurrencyNotificationFdb(
+ teamId,
+ teamLimit,
+ isCrawlOrBatchScrape(fdbJobs[0].data),
+ );
+ }
+ }
+
+ const teamJobs = allTeamJobs.filter(
+ j => backendByJob.get(j.jobId) === "pg",
+ );
+ if (teamJobs.length === 0) continue;
// == Buckets for jobs ==
let jobsForcedToCQ: {
data: ScrapeJobData;
@@ -650,7 +813,7 @@ export async function waitForJob(
try {
doc = await Promise.race(
[
- scrapeQueue.waitForJob(
+ routedScrapeQueue.waitForJob<Document>(
jobId,
timeout !== null ? timeout + 100 : null,
logger,
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index a96a70961..cdce794cc 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -21,7 +21,7 @@ import Express from "express";
import { robustFetch } from "../scraper/scrapeURL/lib/fetch";
import { initializeBlocklist } from "../scraper/WebScraper/utils/blocklist";
import { initializeEngineForcing } from "../scraper/WebScraper/utils/engine-forcing";
-import { crawlFinishedQueue, NuQJob, scrapeQueue } from "./worker/nuq";
+import { crawlFinishedQueue, NuQJob, scrapeQueue } from "./worker/nuq-router";
import { finishCrawlSuper } from "./worker/crawl-logic";
import { getCrawl } from "../lib/crawl-redis";
import { TransportableError } from "../lib/error";
diff --git a/apps/api/src/services/worker/nuq-fdb/client.ts b/apps/api/src/services/worker/nuq-fdb/client.ts
new file mode 100644
index 000000000..0335147eb
--- /dev/null
+++ b/apps/api/src/services/worker/nuq-fdb/client.ts
@@ -0,0 +1,41 @@
+import type * as FDBModule from "foundationdb";
+import type { Database } from "foundationdb";
+import { config } from "../../../config";
+
+// The foundationdb package loads libfdb_c at require-time. Keep the require
+// lazy so processes that never touch the FDB backend (PG-only deploys, unit
+// tests, environments without the client library) can still boot.
+let fdbModule: typeof FDBModule | null = null;
+
+export function getFdb(): typeof FDBModule {
+ if (fdbModule === null) {
+ // eslint-disable-next-line @typescript-eslint/no-var-requires
+ fdbModule = require("foundationdb") as typeof FDBModule;
+ fdbModule.setAPIVersion(720);
+ }
+ return fdbModule;
+}
+
+let db: Database | null = null;
+
+export function getNuqFdbDatabase(): Database {
+ if (db === null) {
+ db = getFdb().open(config.FDB_CLUSTER_FILE);
+ }
+ return db;
+}
+
+export function isFdbConfigured(): boolean {
+ return !!config.FDB_CLUSTER_FILE || config.NUQ_BACKEND === "fdb";
+}
+
+export async function nuqFdbHealthCheck(): Promise<boolean> {
+ try {
+ await getNuqFdbDatabase().doTn(async tn => {
+ await tn.getReadVersion();
+ });
+ return true;
+ } catch {
+ return false;
+ }
+}
diff --git a/apps/api/src/services/worker/nuq-fdb/groups.ts b/apps/api/src/services/worker/nuq-fdb/groups.ts
new file mode 100644
index 000000000..2b4cb4ca0
--- /dev/null
+++ b/apps/api/src/services/worker/nuq-fdb/groups.ts
@@ -0,0 +1,237 @@
+import { randomUUID } from "crypto";
+import type { Transaction } from "foundationdb";
+import { Logger } from "winston";
+import { logger as _logger } from "../../../lib/logger";
+import { getNuqFdbDatabase } from "./client";
+import {
+ NuqFdbKeyspace,
+ GroupMeta,
+ JobMeta,
+ QueueEntry,
+ encodeJson,
+ decodeJson,
+ decodeI64,
+ normalizeOwnerId,
+} from "./keyspace";
+import {
+ ONE,
+ MINUS_ONE,
+ EMPTY,
+ TxContext,
+ newTxContext,
+ uvSuffix,
+ pushReady,
+ setStatusQueued,
+ setGroupJobIndex,
+ bumpGroupStatusCount,
+} from "./ops";
+
+export type NuQFdbGroupStatus = "active" | "completed" | "cancelled";
+
+export type NuQFdbJobGroupInstance = {
+ id: string;
+ status: NuQFdbGroupStatus;
+ createdAt: Date;
+ ownerId: string;
+ ttl: number;
+ expiresAt?: Date;
+ maxConcurrency?: number;
+ delaySeconds?: number;
+};
+
+const DEFAULT_GROUP_TTL_MS = 86400000;
+
+export class NuqFdbGroupOps {
+ constructor(
+ public readonly ks: NuqFdbKeyspace,
+ public readonly finishedKs: NuqFdbKeyspace | null,
+ ) {}
+
+ // Group accounting for a job reaching a terminal state. Must run in the same
+ // transaction as the status transition. The blind task-key set is the
+ // race-free backstop for finish detection; the inline completion attempt
+ // covers the common small-crawl case instantly.
+ public async terminalAccounting(
+ tn: Transaction,
+ gid: string,
+ id: string,
+ prevStatus: string,
+ outcome: "completed" | "failed",
+ countable: boolean,
+ now: number,
+ txc: TxContext,
+ ): Promise<void> {
+ setGroupJobIndex(tn, this.ks, gid, id, countable, outcome);
+ if (countable) {
+ bumpGroupStatusCount(tn, this.ks, gid, prevStatus, -1);
+ bumpGroupStatusCount(tn, this.ks, gid, outcome, 1);
+ if (outcome === "completed") {
+ tn.setVersionstampSuffixedKey(
+ this.ks.groupDonePrefix(gid),
+ Buffer.from(id, "utf8"),
+ uvSuffix(txc),
+ );
+ }
+ }
+
+ const remSnap = decodeI64(
+ await tn.snapshot().get(this.ks.groupRemaining(gid)),
+ );
+ tn.set(this.ks.taskGroupFinish(gid), EMPTY);
+ if (remSnap <= 5) {
+ // near the end: read for real (conflicts with concurrent finishers, but
+ // contention is bounded to the last few jobs of the group)
+ const remReal = decodeI64(await tn.get(this.ks.groupRemaining(gid)));
+ tn.add(this.ks.groupRemaining(gid), MINUS_ONE);
+ if (remReal - 1 <= 0) {
+ await this.tryCompleteGroup(tn, gid, now, txc);
+ }
+ } else {
+ tn.add(this.ks.groupRemaining(gid), MINUS_ONE);
+ }
+ }
+
+ // Completes a drained group: flips status, schedules TTL cleanup, and emits
+ // the crawl-finished job. The normal read of group meta serializes
+ // concurrent completers; exactly one transaction performs the emit.
+ public async tryCompleteGroup(
+ tn: Transaction,
+ gid: string,
+ now: number,
+ txc: TxContext,
+ ): Promise<boolean> {
+ const gMeta = decodeJson<GroupMeta>(await tn.get(this.ks.groupMeta(gid)));
+ if (!gMeta || gMeta.s === "completed") {
+ tn.clear(this.ks.taskGroupFinish(gid));
+ return false;
+ }
+ const expiresAt = now + gMeta.t;
+ const updated: GroupMeta = { ...gMeta, s: "completed", x: expiresAt };
+ tn.set(this.ks.groupMeta(gid), encodeJson(updated));
+ tn.set(this.ks.groupExpiry(expiresAt, gid), EMPTY);
+ tn.clear(this.ks.ongoingGroup(gMeta.o, gid));
+ tn.clear(this.ks.taskGroupFinish(gid));
+
+ if (this.finishedKs) {
+ const fid = randomUUID();
+ const meta: JobMeta = { c: now, p: 0, o: gMeta.o, g: gid, f: 0, dc: 1 };
+ tn.set(this.finishedKs.jobMeta(fid), encodeJson(meta));
+ tn.set(this.finishedKs.jobData(fid, 0), encodeJson({}));
+ const entry: QueueEntry = {
+ i: fid,
+ o: gMeta.o,
+ g: gid,
+ p: 0,
+ f: 0,
+ c: now,
+ };
+ pushReady(tn, this.finishedKs, entry, txc);
+ setStatusQueued(tn, this.finishedKs, fid);
+ // pointer for group TTL cleanup to find the finished job's records
+ tn.set(this.ks.groupFinishedJob(gid), Buffer.from(fid, "utf8"));
+ }
+ return true;
+ }
+}
+
+export class NuQFdbJobGroup {
+ constructor(
+ public readonly ks: NuqFdbKeyspace,
+ public readonly groupOps: NuqFdbGroupOps,
+ ) {}
+
+ private get db() {
+ return getNuqFdbDatabase();
+ }
+
+ private toInstance(id: string, g: GroupMeta): NuQFdbJobGroupInstance {
+ return {
+ id,
+ status: g.s,
+ createdAt: new Date(g.c),
+ ownerId: g.o,
+ ttl: g.t,
+ expiresAt: g.x !== undefined ? new Date(g.x) : undefined,
+ maxConcurrency: g.m,
+ delaySeconds: g.d,
+ };
+ }
+
+ public async addGroup(
+ id: string,
+ ownerId: string,
+ ttl?: number,
+ opts?: { maxConcurrency?: number; delaySeconds?: number },
+ logger: Logger = _logger,
+ ): Promise<NuQFdbJobGroupInstance> {
+ const owner = normalizeOwnerId(ownerId);
+ if (owner === null) throw new Error("Group owner is required");
+ return await this.db.doTn(async tn => {
+ const existingBuf = await tn.get(this.ks.groupMeta(id));
+ const existing = decodeJson<GroupMeta>(existingBuf);
+ if (existing) return this.toInstance(id, existing);
+ const now = Date.now();
+ const g: GroupMeta = {
+ o: owner,
+ c: now,
+ t: ttl ?? DEFAULT_GROUP_TTL_MS,
+ s: "active",
+ m: opts?.maxConcurrency,
+ d: opts?.delaySeconds,
+ };
+ tn.set(this.ks.groupMeta(id), encodeJson(g));
+ tn.set(this.ks.ongoingGroup(owner, id), encodeJson({ c: now }));
+ return this.toInstance(id, g);
+ });
+ }
+
+ public async getGroup(
+ id: string,
+ logger: Logger = _logger,
+ ): Promise<NuQFdbJobGroupInstance | null> {
+ return await this.db.doTn(async tn => {
+ const g = decodeJson<GroupMeta>(await tn.get(this.ks.groupMeta(id)));
+ return g ? this.toInstance(id, g) : null;
+ });
+ }
+
+ public async getOngoingByOwner(
+ ownerId: string,
+ logger: Logger = _logger,
+ ): Promise<NuQFdbJobGroupInstance[]> {
+ const owner = normalizeOwnerId(ownerId);
+ if (owner === null) return [];
+ return await this.db.doTn(async tn => {
+ const r = this.ks.ongoingGroupRange(owner);
+ const rows = await tn.snapshot().getRangeAll(r.begin, r.end);
+ const out: NuQFdbJobGroupInstance[] = [];
+ for (const [key] of rows) {
+ const gid = this.ks.unpackId(key as Buffer);
+ const g = decodeJson<GroupMeta>(
+ await tn.snapshot().get(this.ks.groupMeta(gid)),
+ );
+ if (g && g.s === "active") out.push(this.toInstance(gid, g));
+ }
+ return out;
+ });
+ }
+
+ // O(1) cancellation: flips the group status and leaves the heavy lifting to
+ // the sweeper (pending entries) and take-side diversion (ready entries).
+ public async cancelGroup(
+ id: string,
+ logger: Logger = _logger,
+ ): Promise<boolean> {
+ return await this.db.doTn(async tn => {
+ const g = decodeJson<GroupMeta>(await tn.get(this.ks.groupMeta(id)));
+ if (!g || g.s !== "active") return false;
+ tn.set(
+ this.ks.groupMeta(id),
+ encodeJson({ ...g, s: "cancelled" } satisfies GroupMeta),
+ );
+ tn.set(this.ks.taskGroupCancel(id), EMPTY);
+ tn.clear(this.ks.ongoingGroup(g.o, id));
+ return true;
+ });
+ }
+}
diff --git a/apps/api/src/services/worker/nuq-fdb/index.ts b/apps/api/src/services/worker/nuq-fdb/index.ts
new file mode 100644
index 000000000..c0ef3d84b
--- /dev/null
+++ b/apps/api/src/services/worker/nuq-fdb/index.ts
@@ -0,0 +1,50 @@
+import type { ScrapeJobData } from "../../../types";
+import { NuQFdbQueue, QueueFullError, normalizeOwnerId } from "./queue";
+import { NuQFdbJobGroup } from "./groups";
+import { NuqFdbSweeper } from "./sweeper";
+import { NuqFdbExternalSlots } from "./slots";
+import { isFdbConfigured, nuqFdbHealthCheck } from "./client";
+
+export { NuQFdbQueue, QueueFullError, normalizeOwnerId } from "./queue";
+export type {
+ NuQFdbJob,
+ NuQFdbJobOptions,
+ NuQFdbGate,
+ NuQJobStatusCompat,
+} from "./queue";
+export { NuQFdbJobGroup } from "./groups";
+export type { NuQFdbJobGroupInstance, NuQFdbGroupStatus } from "./groups";
+export { NuqFdbSweeper } from "./sweeper";
+export { NuqFdbExternalSlots } from "./slots";
+export { isFdbConfigured, nuqFdbHealthCheck } from "./client";
+
+export const scrapeQueueFdb = new NuQFdbQueue<ScrapeJobData, any>("scrape", {
+ hasGroups: true,
+ finishedQueueName: "crawl_finished",
+});
+
+export const crawlFinishedQueueFdb = new NuQFdbQueue<any, any>(
+ "crawl_finished",
+ {
+ hasGroups: false,
+ },
+);
+
+export const crawlGroupFdb = new NuQFdbJobGroup(
+ scrapeQueueFdb.ks,
+ scrapeQueueFdb.groupOps!,
+);
+
+export const externalSlotsFdb = new NuqFdbExternalSlots(scrapeQueueFdb.ks);
+
+let sweeper: NuqFdbSweeper | null = null;
+
+export function getNuqFdbSweeper(): NuqFdbSweeper {
+ if (!sweeper) {
+ sweeper = new NuqFdbSweeper(
+ [scrapeQueueFdb, crawlFinishedQueueFdb],
+ [externalSlotsFdb],
+ );
+ }
+ return sweeper;
+}
diff --git a/apps/api/src/services/worker/nuq-fdb/keyspace.ts b/apps/api/src/services/worker/nuq-fdb/keyspace.ts
new file mode 100644
index 000000000..65e61287a
--- /dev/null
+++ b/apps/api/src/services/worker/nuq-fdb/keyspace.ts
@@ -0,0 +1,345 @@
+import { config } from "../../../config";
+import { getFdb } from "./client";
+
+export { normalizeOwnerId } from "../../../lib/owner-id";
+
+// All keys for a queue live under the tuple prefix ("nuq", queueName).
+// Counters are 8-byte little-endian signed ints mutated with atomic ADD.
+
+export const READY_SHARDS = config.NUQ_FDB_READY_SHARDS;
+export const TEAM_PENDING_SHARDS = config.NUQ_FDB_TEAM_PENDING_SHARDS;
+export const TIME_BUCKETS = config.NUQ_FDB_TIME_BUCKETS;
+
+export type NuqFdbJobStatus =
+ | "pending" // waiting for a concurrency slot (externally: "backlog")
+ | "queued" // holds its slots, in a ready shard
+ | "active"
+ | "completed"
+ | "failed"
+ | "cancelled";
+
+// Job flags (bitmask on job meta and queue entries)
+export const F_GATED = 1; // holds a team slot while queued/active
+export const F_CRAWL_GATED = 2; // holds a crawl slot while team-pending/queued/active
+export const F_LISTENABLE = 4;
+export const F_ZDR = 8;
+export const F_COUNTABLE = 16; // mode === "single_urls": counted in group numeric stats
+export const F_GACC = 32; // participates in group remaining-count accounting
+
+// Where a pending job's queue entry lives, stored on its status record so
+// sweepers and removers can clear the exact key without scanning.
+export type PendingLoc =
+ | { k: "tq"; s: number; p: number; c: number } // team-pending: shard, priority, createdAtMs
+ | { k: "gq"; p: number; c: number } // crawl-pending: priority, createdAtMs
+ | { k: "dl"; at: number }; // delay index: notBeforeMs
+
+export type JobMeta = {
+ c: number; // createdAt ms
+ p: number; // priority
+ o: string; // ownerId (normalized uuid)
+ g?: string; // groupId
+ f: number; // flags
+ to?: number; // backlog timesOutAt ms
+ dc: number; // data chunk count
+};
+
+export type JobStatusRecord = {
+ s: NuqFdbJobStatus;
+ l?: string; // lock uuid while active
+ e?: number; // lease expiry ms while active
+ st: number; // stall count
+ fa?: number; // finishedAt ms
+ loc?: PendingLoc;
+};
+
+// Entry stored in ready shards, pending queues and the delay index.
+export type QueueEntry = {
+ i: string; // jobId
+ o: string; // ownerId
+ g?: string; // groupId
+ p: number; // priority
+ f: number; // flags
+ c: number; // createdAt ms
+ to?: number; // backlog timesOutAt ms
+};
+
+export type GroupMeta = {
+ o: string; // ownerId
+ c: number; // createdAt ms
+ t: number; // ttl ms
+ s: "active" | "completed" | "cancelled";
+ m?: number; // maxConcurrency
+ d?: number; // delay seconds between job starts
+ x?: number; // expiresAt ms
+};
+
+export function encodeI64(n: number): Buffer {
+ const buf = Buffer.alloc(8);
+ buf.writeBigInt64LE(BigInt(n));
+ return buf;
+}
+
+export function decodeI64(buf: Buffer | undefined | null): number {
+ if (!buf || buf.length < 8) return 0;
+ return Number(buf.readBigInt64LE());
+}
+
+export function encodeJson(v: any): Buffer {
+ return Buffer.from(JSON.stringify(v), "utf8");
+}
+
+export function decodeJson<T = any>(buf: Buffer | undefined | null): T | null {
+ if (!buf) return null;
+ return JSON.parse(buf.toString("utf8")) as T;
+}
+
+export function fnv1a(str: string): number {
+ let hash = 0x811c9dc5;
+ for (let i = 0; i < str.length; i++) {
+ hash ^= str.charCodeAt(i);
+ hash = Math.imul(hash, 0x01000193);
+ }
+ return hash >>> 0;
+}
+
+function prefixEnd(prefix: Buffer): Buffer {
+ const end = Buffer.from(prefix);
+ for (let i = end.length - 1; i >= 0; i--) {
+ if (end[i] !== 0xff) {
+ end[i]++;
+ return end.subarray(0, i + 1);
+ }
+ }
+ throw new Error("Unable to construct FDB prefix range end");
+}
+
+export function timeBucket(jobId: string): number {
+ return fnv1a(jobId) % TIME_BUCKETS;
+}
+
+export function teamPendingShard(jobId: string): number {
+ // different rotation than timeBucket so the two don't correlate
+ return fnv1a(jobId + "/tq") % TEAM_PENDING_SHARDS;
+}
+
+export class NuqFdbKeyspace {
+ constructor(public readonly queueName: string) {}
+
+ pack(parts: any[]): Buffer {
+ return getFdb().tuple.pack(["nuq", this.queueName, ...parts]) as Buffer;
+ }
+
+ private packRange(parts: any[]): { begin: Buffer; end: Buffer } {
+ const begin = this.pack(parts);
+ return { begin, end: prefixEnd(begin) };
+ }
+
+ unpack(key: Buffer): unknown[] {
+ return getFdb().tuple.unpack(key);
+ }
+
+ // === Job records
+ jobMeta(id: string): Buffer {
+ return this.pack(["j", id, "m"]);
+ }
+ jobStatus(id: string): Buffer {
+ return this.pack(["j", id, "s"]);
+ }
+ jobData(id: string, chunk: number): Buffer {
+ return this.pack(["j", id, "d", chunk]);
+ }
+ jobDataRange(id: string) {
+ return this.packRange(["j", id, "d"]);
+ }
+ jobReturnvalue(id: string, chunk: number): Buffer {
+ return this.pack(["j", id, "r", chunk]);
+ }
+ jobReturnvalueRange(id: string) {
+ return this.packRange(["j", id, "r"]);
+ }
+ jobFailedReason(id: string): Buffer {
+ return this.pack(["j", id, "f"]);
+ }
+ jobRange(id: string) {
+ return this.packRange(["j", id]);
+ }
+
+ // === Team gate
+ teamLimit(tid: string): Buffer {
+ return this.pack(["t", tid, "limit"]);
+ }
+ teamActive(tid: string): Buffer {
+ return this.pack(["t", tid, "active"]);
+ }
+ teamActiveIndex(tid: string): Buffer {
+ return this.pack(["ta", tid]);
+ }
+ teamActiveIndexRange() {
+ return this.packRange(["ta"]);
+ }
+ teamPendingCount(tid: string): Buffer {
+ return this.pack(["t", tid, "pend"]);
+ }
+ teamShardCount(tid: string, shard: number): Buffer {
+ return this.pack(["t", tid, "qn", shard]);
+ }
+ teamShardCountRange(tid: string) {
+ return this.packRange(["t", tid, "qn"]);
+ }
+ teamPendingKey(
+ tid: string,
+ shard: number,
+ priority: number,
+ createdAtMs: number,
+ id: string,
+ ): Buffer {
+ return this.pack(["t", tid, "q", shard, priority, createdAtMs, id]);
+ }
+ teamPendingShardRange(tid: string, shard: number) {
+ return this.packRange(["t", tid, "q", shard]);
+ }
+
+ // === Group (crawl) records
+ groupMeta(gid: string): Buffer {
+ return this.pack(["g", gid, "meta"]);
+ }
+ groupRemaining(gid: string): Buffer {
+ return this.pack(["g", gid, "rem"]);
+ }
+ groupCrawlActive(gid: string): Buffer {
+ return this.pack(["g", gid, "cact"]);
+ }
+ groupStatusCount(gid: string, status: string): Buffer {
+ return this.pack(["g", gid, "n", status]);
+ }
+ groupStatusCountRange(gid: string) {
+ return this.packRange(["g", gid, "n"]);
+ }
+ groupPendingCount(gid: string): Buffer {
+ return this.pack(["g", gid, "qn"]);
+ }
+ groupPendingKey(
+ gid: string,
+ priority: number,
+ createdAtMs: number,
+ id: string,
+ ): Buffer {
+ return this.pack(["g", gid, "q", priority, createdAtMs, id]);
+ }
+ groupPendingRange(gid: string) {
+ return this.packRange(["g", gid, "q"]);
+ }
+ groupJob(gid: string, id: string): Buffer {
+ return this.pack(["g", gid, "jobs", id]);
+ }
+ groupJobRange(gid: string) {
+ return this.packRange(["g", gid, "jobs"]);
+ }
+ groupDonePrefix(gid: string): Buffer {
+ return this.pack(["g", gid, "done"]);
+ }
+ groupDoneRange(gid: string) {
+ return this.packRange(["g", gid, "done"]);
+ }
+ groupFinishedJob(gid: string): Buffer {
+ return this.pack(["g", gid, "fjob"]);
+ }
+ groupRange(gid: string) {
+ return this.packRange(["g", gid]);
+ }
+ ongoingGroup(ownerId: string, gid: string): Buffer {
+ return this.pack(["go", ownerId, gid]);
+ }
+ ongoingGroupRange(ownerId: string) {
+ return this.packRange(["go", ownerId]);
+ }
+
+ // === Ready shards
+ readyPrefix(shard: number, priority: number): Buffer {
+ return this.pack(["r", shard, "q", priority]);
+ }
+ readyShardRange(shard: number) {
+ return this.packRange(["r", shard, "q"]);
+ }
+ readyShardCount(shard: number): Buffer {
+ return this.pack(["rn", shard]);
+ }
+ readyShardCountRange() {
+ return this.packRange(["rn"]);
+ }
+
+ // === Time-ordered indexes (sweeper-owned)
+ lease(bucket: number, expMs: number, id: string): Buffer {
+ return this.pack(["lease", bucket, expMs, id]);
+ }
+ leaseScanRange(bucket: number, untilMs: number) {
+ return {
+ begin: this.pack(["lease", bucket]),
+ end: this.pack(["lease", bucket, untilMs]),
+ };
+ }
+ backlogTimeout(bucket: number, atMs: number, id: string): Buffer {
+ return this.pack(["bto", bucket, atMs, id]);
+ }
+ backlogTimeoutScanRange(bucket: number, untilMs: number) {
+ return {
+ begin: this.pack(["bto", bucket]),
+ end: this.pack(["bto", bucket, untilMs]),
+ };
+ }
+ delayed(bucket: number, notBeforeMs: number, id: string): Buffer {
+ return this.pack(["delay", bucket, notBeforeMs, id]);
+ }
+ delayedScanRange(bucket: number, untilMs: number) {
+ return {
+ begin: this.pack(["delay", bucket]),
+ end: this.pack(["delay", bucket, untilMs]),
+ };
+ }
+ jobExpiry(bucket: number, atMs: number, id: string): Buffer {
+ return this.pack(["jexp", bucket, atMs, id]);
+ }
+ jobExpiryScanRange(bucket: number, untilMs: number) {
+ return {
+ begin: this.pack(["jexp", bucket]),
+ end: this.pack(["jexp", bucket, untilMs]),
+ };
+ }
+ groupExpiry(atMs: number, gid: string): Buffer {
+ return this.pack(["gexp", atMs, gid]);
+ }
+ groupExpiryScanRange(untilMs: number) {
+ return {
+ begin: this.pack(["gexp"]),
+ end: this.pack(["gexp", untilMs]),
+ };
+ }
+
+ // === Task keys (blind-set, sweeper-drained)
+ taskGroupFinish(gid: string): Buffer {
+ return this.pack(["task", "gfin", gid]);
+ }
+ taskGroupFinishRange() {
+ return this.packRange(["task", "gfin"]);
+ }
+ taskGroupCancel(gid: string): Buffer {
+ return this.pack(["task", "gcancel", gid]);
+ }
+ taskGroupCancelRange() {
+ return this.packRange(["task", "gcancel"]);
+ }
+ taskTeamRaise(tid: string): Buffer {
+ return this.pack(["task", "traise", tid]);
+ }
+ taskTeamRaiseRange() {
+ return this.packRange(["task", "traise"]);
+ }
+ sweeperLock(): Buffer {
+ return this.pack(["sweep", "lock"]);
+ }
+
+ unpackId(key: Buffer, indexFromEnd: number = 0): string {
+ const parts = getFdb().tuple.unpack(key);
+ return String(parts[parts.length - 1 - indexFromEnd]);
+ }
+}
diff --git a/apps/api/src/services/worker/nuq-fdb/ops.ts b/apps/api/src/services/worker/nuq-fdb/ops.ts
new file mode 100644
index 000000000..24526174d
--- /dev/null
+++ b/apps/api/src/services/worker/nuq-fdb/ops.ts
@@ -0,0 +1,433 @@
+import type { Transaction } from "foundationdb";
+import { logger as _logger } from "../../../lib/logger";
+import {
+ NuqFdbKeyspace,
+ QueueEntry,
+ JobStatusRecord,
+ GroupMeta,
+ PendingLoc,
+ encodeI64,
+ decodeI64,
+ encodeJson,
+ decodeJson,
+ timeBucket,
+ teamPendingShard,
+ READY_SHARDS,
+ TEAM_PENDING_SHARDS,
+ F_GATED,
+ F_CRAWL_GATED,
+ F_COUNTABLE,
+} from "./keyspace";
+
+export const ONE = encodeI64(1);
+export const MINUS_ONE = encodeI64(-1);
+export const EMPTY = Buffer.alloc(0);
+
+export const LEASE_MS = 90_000;
+export const MAX_STALLS = 9;
+export const COMPLETED_STANDALONE_RETENTION_MS = 60 * 60 * 1000;
+export const FAILED_STANDALONE_RETENTION_MS = 6 * 60 * 60 * 1000;
+
+export function bumpTeamActive(
+ tn: Transaction,
+ ks: NuqFdbKeyspace,
+ teamId: string,
+ delta: number,
+): void {
+ if (delta === 0) return;
+ const encoded = encodeI64(delta);
+ tn.add(ks.teamActive(teamId), encoded);
+ tn.add(ks.teamActiveIndex(teamId), encoded);
+}
+
+// Per-transaction-attempt context. uv makes versionstamp-suffixed keys unique
+// within one transaction; it resets naturally when doTn retries the closure.
+export type TxContext = { uv: number };
+
+export function newTxContext(): TxContext {
+ return { uv: 0 };
+}
+
+export function uvSuffix(txc: TxContext): Buffer {
+ const buf = Buffer.alloc(2);
+ buf.writeUInt16BE(txc.uv++);
+ return buf;
+}
+
+export function randomReadyShard(): number {
+ return Math.floor(Math.random() * READY_SHARDS);
+}
+
+// === Status record writers
+
+export function setStatusQueued(
+ tn: Transaction,
+ ks: NuqFdbKeyspace,
+ id: string,
+ stalls: number = 0,
+): void {
+ const rec: JobStatusRecord = { s: "queued", st: stalls };
+ tn.set(ks.jobStatus(id), encodeJson(rec));
+}
+
+export function setStatusPending(
+ tn: Transaction,
+ ks: NuqFdbKeyspace,
+ id: string,
+ loc: PendingLoc,
+): void {
+ const rec: JobStatusRecord = { s: "pending", st: 0, loc };
+ tn.set(ks.jobStatus(id), encodeJson(rec));
+}
+
+// === Queue placement
+
+export function pushReady(
+ tn: Transaction,
+ ks: NuqFdbKeyspace,
+ e: QueueEntry,
+ txc: TxContext,
+): void {
+ const shard = randomReadyShard();
+ tn.setVersionstampSuffixedKey(
+ ks.readyPrefix(shard, e.p),
+ encodeJson(e),
+ uvSuffix(txc),
+ );
+ tn.add(ks.readyShardCount(shard), ONE);
+}
+
+export function appendTeamPending(
+ tn: Transaction,
+ ks: NuqFdbKeyspace,
+ e: QueueEntry,
+): PendingLoc {
+ const shard = teamPendingShard(e.i);
+ tn.set(ks.teamPendingKey(e.o, shard, e.p, e.c, e.i), encodeJson(e));
+ tn.add(ks.teamShardCount(e.o, shard), ONE);
+ tn.add(ks.teamPendingCount(e.o), ONE);
+ if (e.to !== undefined) {
+ tn.set(ks.backlogTimeout(timeBucket(e.i), e.to, e.i), EMPTY);
+ }
+ return { k: "tq", s: shard, p: e.p, c: e.c };
+}
+
+export function appendCrawlPending(
+ tn: Transaction,
+ ks: NuqFdbKeyspace,
+ e: QueueEntry,
+): PendingLoc {
+ tn.set(ks.groupPendingKey(e.g!, e.p, e.c, e.i), encodeJson(e));
+ tn.add(ks.groupPendingCount(e.g!), ONE);
+ tn.add(ks.teamPendingCount(e.o), ONE);
+ if (e.to !== undefined) {
+ tn.set(ks.backlogTimeout(timeBucket(e.i), e.to, e.i), EMPTY);
+ }
+ return { k: "gq", p: e.p, c: e.c };
+}
+
+export function clearBacklogTimeout(
+ tn: Transaction,
+ ks: NuqFdbKeyspace,
+ e: { i: string; to?: number },
+): void {
+ if (e.to !== undefined) {
+ tn.clear(ks.backlogTimeout(timeBucket(e.i), e.to, e.i));
+ }
+}
+
+// Moves a pending entry into a ready shard, with group counter upkeep.
+// Callers are responsible for slot accounting.
+export function promoteEntryToReady(
+ tn: Transaction,
+ ks: NuqFdbKeyspace,
+ e: QueueEntry,
+ txc: TxContext,
+): void {
+ pushReady(tn, ks, e, txc);
+ setStatusQueued(tn, ks, e.i);
+ if (e.g && e.f & F_COUNTABLE) {
+ tn.add(ks.groupStatusCount(e.g, "pending"), MINUS_ONE);
+ tn.add(ks.groupStatusCount(e.g, "queued"), ONE);
+ }
+}
+
+// Reconstructs the exact pending key for a job from its status record.
+export function pendingKeyFromLoc(
+ ks: NuqFdbKeyspace,
+ id: string,
+ ownerId: string,
+ groupId: string | undefined,
+ loc: PendingLoc,
+): Buffer {
+ if (loc.k === "tq")
+ return ks.teamPendingKey(ownerId, loc.s, loc.p, loc.c, id);
+ if (loc.k === "gq") return ks.groupPendingKey(groupId!, loc.p, loc.c, id);
+ return ks.delayed(timeBucket(id), loc.at, id);
+}
+
+// Clears a pending job's queue entry + counters + backlog timeout. The caller
+// is responsible for slot accounting and the job's status/records.
+export function clearPendingPlacement(
+ tn: Transaction,
+ ks: NuqFdbKeyspace,
+ id: string,
+ ownerId: string,
+ groupId: string | undefined,
+ loc: PendingLoc,
+ timesOutAt: number | undefined,
+): void {
+ tn.clear(pendingKeyFromLoc(ks, id, ownerId, groupId, loc));
+ if (loc.k === "tq") {
+ tn.add(ks.teamShardCount(ownerId, loc.s), MINUS_ONE);
+ tn.add(ks.teamPendingCount(ownerId), MINUS_ONE);
+ } else if (loc.k === "gq") {
+ tn.add(ks.groupPendingCount(groupId!), MINUS_ONE);
+ tn.add(ks.teamPendingCount(ownerId), MINUS_ONE);
+ }
+ if (loc.k !== "dl") {
+ clearBacklogTimeout(tn, ks, { i: id, to: timesOutAt });
+ }
+}
+
+// === Pending queue pops
+
+// Pops the best entry from a team's pending shards. Snapshot-reads the shard
+// occupancy counters and probes up to 3 non-empty shards. Returns null if all
+// probed shards are empty.
+export async function popTeamPending(
+ tn: Transaction,
+ ks: NuqFdbKeyspace,
+ tid: string,
+): Promise<QueueEntry | null> {
+ const startedAt = Date.now();
+ const range = ks.teamShardCountRange(tid);
+ const counts = await tn.snapshot().getRangeAll(range.begin, range.end);
+ const nonEmpty: number[] = [];
+ for (const [key, value] of counts) {
+ if (decodeI64(value as Buffer) > 0)
+ nonEmpty.push(Number(ks.unpackId(key as Buffer)));
+ }
+ // shuffle so concurrent finishers spread across shards
+ for (let i = nonEmpty.length - 1; i > 0; i--) {
+ const j = Math.floor(Math.random() * (i + 1));
+ [nonEmpty[i], nonEmpty[j]] = [nonEmpty[j], nonEmpty[i]];
+ }
+ let probedShards = 0;
+ let staleShardCounters = 0;
+ for (const shard of nonEmpty.slice(0, 3)) {
+ probedShards++;
+ const r = ks.teamPendingShardRange(tid, shard);
+ const head = await tn.getRangeAll(r.begin, r.end, { limit: 1 });
+ if (head.length === 0) {
+ staleShardCounters++;
+ continue; // counter was stale
+ }
+ const [key, value] = head[0];
+ const e = decodeJson<QueueEntry>(value as Buffer)!;
+ tn.clear(key as Buffer);
+ tn.add(ks.teamShardCount(tid, shard), MINUS_ONE);
+ tn.add(ks.teamPendingCount(tid), MINUS_ONE);
+ clearBacklogTimeout(tn, ks, e);
+ _logger.debug("NuQ FDB team pending promotion", {
+ canonicalLog: "nuq-fdb/team_pending_promotion",
+ queueName: ks.queueName,
+ result: "promoted",
+ durationMs: Date.now() - startedAt,
+ nonEmptyShardCount: nonEmpty.length,
+ probedShardCount: probedShards,
+ staleShardCounterCount: staleShardCounters,
+ selectedShard: shard,
+ priority: e.p,
+ queuedAgeMs: Date.now() - e.c,
+ });
+ return e;
+ }
+ _logger.debug("NuQ FDB team pending promotion", {
+ canonicalLog: "nuq-fdb/team_pending_promotion",
+ queueName: ks.queueName,
+ result: "empty",
+ durationMs: Date.now() - startedAt,
+ nonEmptyShardCount: nonEmpty.length,
+ probedShardCount: probedShards,
+ staleShardCounterCount: staleShardCounters,
+ });
+ return null;
+}
+
+export async function popCrawlPending(
+ tn: Transaction,
+ ks: NuqFdbKeyspace,
+ gid: string,
+): Promise<QueueEntry | null> {
+ const r = ks.groupPendingRange(gid);
+ const head = await tn.getRangeAll(r.begin, r.end, { limit: 1 });
+ if (head.length === 0) return null;
+ const [key, value] = head[0];
+ const e = decodeJson<QueueEntry>(value as Buffer)!;
+ tn.clear(key as Buffer);
+ tn.add(ks.groupPendingCount(gid), MINUS_ONE);
+ tn.add(ks.teamPendingCount(e.o), MINUS_ONE);
+ clearBacklogTimeout(tn, ks, e);
+ return e;
+}
+
+// === Slot release + inline promotion chain
+//
+// Slots are handed off one-for-one: a released slot goes to a pending head in
+// the same transaction whenever possible, so this function never does a
+// conflicting read of the team active counter on the handoff path.
+//
+// `held` describes which slots the leaving job actually holds:
+// - active / ready jobs: team + crawl (if crawl-gated)
+// - team-pending / delayed jobs: crawl only (if crawl-gated)
+// - crawl-pending jobs: none
+
+export async function releaseSlotsAndPromote(
+ tn: Transaction,
+ ks: NuqFdbKeyspace,
+ e: QueueEntry,
+ held: { team: boolean; crawl: boolean },
+ now: number,
+ txc: TxContext,
+): Promise<void> {
+ if (!(e.f & F_GATED)) return;
+ const tid = e.o;
+
+ const holdsCrawl = held.crawl && !!e.g && !!(e.f & F_CRAWL_GATED);
+ if (!held.team && !holdsCrawl) return;
+
+ // Limit-lowering convergence: if the team is over its limit, don't hand off.
+ let overLimit = false;
+ let limit = Infinity;
+ if (held.team) {
+ const limitBuf = await tn.get(ks.teamLimit(tid)); // cold key, rarely written
+ if (limitBuf) {
+ limit = decodeI64(limitBuf);
+ const snapActive = decodeI64(await tn.snapshot().get(ks.teamActive(tid)));
+ overLimit = snapActive > limit;
+ }
+ }
+
+ let teamHead: QueueEntry | null | "consumed" = null;
+ if (held.team && !overLimit) {
+ teamHead = await popTeamPending(tn, ks, tid);
+ }
+
+ // Crawl slot handoff: promote the crawl-pending head out of the crawl gate.
+ let crawlPromoted: QueueEntry | null = null;
+ let crawlDelaySeconds = 0;
+ if (holdsCrawl) {
+ const gMeta = decodeJson<GroupMeta>(
+ await tn.snapshot().get(ks.groupMeta(e.g!)),
+ );
+ if (gMeta && gMeta.s === "active") {
+ crawlPromoted = await popCrawlPending(tn, ks, e.g!);
+ if (crawlPromoted) {
+ crawlDelaySeconds = gMeta.d ?? 0;
+ } else {
+ tn.add(ks.groupCrawlActive(e.g!), MINUS_ONE);
+ }
+ } else {
+ tn.add(ks.groupCrawlActive(e.g!), MINUS_ONE);
+ }
+ }
+
+ if (crawlPromoted) {
+ const j2 = crawlPromoted;
+ if (crawlDelaySeconds > 0) {
+ // crawl delay: park with a not-before timestamp; keeps the crawl slot.
+ const notBefore = now + crawlDelaySeconds * 1000;
+ tn.set(ks.delayed(timeBucket(j2.i), notBefore, j2.i), encodeJson(j2));
+ setStatusPending(tn, ks, j2.i, { k: "dl", at: notBefore });
+ } else if (held.team && !overLimit && teamHead === null) {
+ // the freed team slot goes directly to the crawl-promoted job
+ promoteEntryToReady(tn, ks, j2, txc);
+ teamHead = "consumed";
+ } else if (!held.team) {
+ // no team slot was freed here (job removal/cancellation paths), so the
+ // promoted job must be admitted through the gate on its own
+ await admitThroughTeamGate(tn, ks, j2, txc);
+ } else {
+ // waits in the team gate; keeps the crawl slot
+ const loc = appendTeamPending(tn, ks, j2);
+ setStatusPending(tn, ks, j2.i, loc);
+ }
+ }
+
+ if (held.team) {
+ if (teamHead === "consumed") {
+ // slot handed to crawlPromoted above
+ } else if (teamHead !== null) {
+ promoteEntryToReady(tn, ks, teamHead, txc);
+ } else {
+ bumpTeamActive(tn, ks, tid, -1);
+ }
+ }
+}
+
+// Admits a slotless entry through the team gate, used when a job enters the
+// team gate outside of a one-for-one handoff (delayed promotion, limit raise).
+// Does a conflicting read of the team active counter -- callers are rare paths.
+export async function admitThroughTeamGate(
+ tn: Transaction,
+ ks: NuqFdbKeyspace,
+ e: QueueEntry,
+ txc: TxContext,
+): Promise<void> {
+ const limitBuf = await tn.get(ks.teamLimit(e.o));
+ const limit = limitBuf ? decodeI64(limitBuf) : Infinity;
+ const active = decodeI64(await tn.get(ks.teamActive(e.o)));
+ if (active < limit) {
+ bumpTeamActive(tn, ks, e.o, 1);
+ promoteEntryToReady(tn, ks, e, txc);
+ } else {
+ const loc = appendTeamPending(tn, ks, e);
+ setStatusPending(tn, ks, e.i, loc);
+ }
+}
+
+// === Job record cleanup
+
+export function deleteJobRecords(
+ tn: Transaction,
+ ks: NuqFdbKeyspace,
+ id: string,
+): void {
+ const r = ks.jobRange(id);
+ tn.clearRange(r.begin, r.end);
+}
+
+// === Group accounting helpers
+
+export type GroupJobIndexValue = {
+ m: number; // 1 if countable (single_urls)
+ s: string; // job status
+};
+
+export function setGroupJobIndex(
+ tn: Transaction,
+ ks: NuqFdbKeyspace,
+ gid: string,
+ id: string,
+ countable: boolean,
+ status: string,
+): void {
+ tn.set(
+ ks.groupJob(gid, id),
+ encodeJson({
+ m: countable ? 1 : 0,
+ s: status,
+ } satisfies GroupJobIndexValue),
+ );
+}
+
+export function bumpGroupStatusCount(
+ tn: Transaction,
+ ks: NuqFdbKeyspace,
+ gid: string,
+ status: string,
+ delta: 1 | -1,
+): void {
+ tn.add(ks.groupStatusCount(gid, status), delta === 1 ? ONE : MINUS_ONE);
+}
diff --git a/apps/api/src/services/worker/nuq-fdb/queue.ts b/apps/api/src/services/worker/nuq-fdb/queue.ts
new file mode 100644
index 000000000..9d0541fe4
--- /dev/null
+++ b/apps/api/src/services/worker/nuq-fdb/queue.ts
@@ -0,0 +1,1276 @@
+import { randomUUID } from "crypto";
+import type { Transaction } from "foundationdb";
+import { Logger } from "winston";
+import { logger as _logger } from "../../../lib/logger";
+import { config } from "../../../config";
+import { QueueFullError } from "../../../lib/queue-full-error";
+import { getNuqFdbDatabase } from "./client";
+import {
+ NuqFdbKeyspace,
+ NuqFdbJobStatus,
+ JobMeta,
+ JobStatusRecord,
+ QueueEntry,
+ GroupMeta,
+ encodeI64,
+ decodeI64,
+ encodeJson,
+ decodeJson,
+ timeBucket,
+ READY_SHARDS,
+ F_GATED,
+ F_CRAWL_GATED,
+ F_LISTENABLE,
+ F_ZDR,
+ F_COUNTABLE,
+ F_GACC,
+ normalizeOwnerId,
+} from "./keyspace";
+
+export { normalizeOwnerId, F_GACC };
+import {
+ ONE,
+ MINUS_ONE,
+ EMPTY,
+ LEASE_MS,
+ MAX_STALLS,
+ COMPLETED_STANDALONE_RETENTION_MS,
+ FAILED_STANDALONE_RETENTION_MS,
+ TxContext,
+ newTxContext,
+ pushReady,
+ appendTeamPending,
+ appendCrawlPending,
+ popTeamPending,
+ setStatusQueued,
+ setStatusPending,
+ clearPendingPlacement,
+ releaseSlotsAndPromote,
+ deleteJobRecords,
+ setGroupJobIndex,
+ bumpGroupStatusCount,
+ bumpTeamActive,
+ GroupJobIndexValue,
+} from "./ops";
+import { NuqFdbGroupOps } from "./groups";
+
+const DATA_CHUNK_BYTES = 90 * 1024;
+// FoundationDB caps transactions at 10,000,000 bytes of affected data. Keep
+// enqueue batches much smaller so large payloads cannot accidentally combine
+// into a commit-time transaction_too_large failure.
+const FDB_TRANSACTION_BYTE_LIMIT = 10_000_000;
+const ENQUEUE_BATCH_BYTE_BUDGET = 750 * 1024;
+const ENQUEUE_SINGLE_JOB_BYTE_LIMIT = 8 * 1024 * 1024;
+const ENQUEUE_MAX_JOBS_PER_TRANSACTION = 250;
+
+export { QueueFullError };
+
+type FdbKeySelector = {
+ key: Buffer;
+ orEqual: boolean;
+ offset: number;
+ _isKeySelector: true;
+};
+
+export type NuQJobStatusCompat =
+ | "queued"
+ | "active"
+ | "completed"
+ | "failed"
+ | "backlog";
+
+export type NuQFdbJob<Data = any, ReturnValue = any> = {
+ id: string;
+ status: NuQJobStatusCompat;
+ createdAt: Date;
+ priority: number;
+ data: Data;
+ finishedAt?: Date;
+ returnvalue?: ReturnValue;
+ failedReason?: string;
+ lock?: string;
+ leaseExpiresAt?: Date;
+ ownerId?: string;
+ groupId?: string;
+};
+
+export type NuQFdbJobOptions = {
+ priority?: number;
+ listenable?: boolean;
+ ownerId?: string;
+ groupId?: string;
+ // when set, the job bypasses both concurrency gates (kickoff jobs)
+ bypassGate?: boolean;
+ // backlog timeout: if the job is still waiting for a slot at this time, it
+ // is silently dropped (matches the PG backlog reaper)
+ timesOutAt?: Date;
+};
+
+export type NuQFdbGate = {
+ // null = unlimited (self-hosted)
+ teamLimit: number | null;
+ queueCap: number;
+};
+
+type AddJobInput<Data> = {
+ id: string;
+ data: Data;
+ options: NuQFdbJobOptions;
+};
+
+type PreparedAddJob<Data> = AddJobInput<Data> & {
+ dataBuf: Buffer;
+ dataChunks: Buffer[];
+ estimatedAffectedBytes: number;
+};
+
+function chunkBuffer(buf: Buffer, size: number): Buffer[] {
+ const chunks: Buffer[] = [];
+ for (let i = 0; i < buf.length; i += size) {
+ chunks.push(buf.subarray(i, i + size));
+ }
+ return chunks.length > 0 ? chunks : [Buffer.alloc(0)];
+}
+
+function externalStatus(s: NuqFdbJobStatus): NuQJobStatusCompat | null {
+ if (s === "pending") return "backlog";
+ if (s === "cancelled") return null; // cancelled jobs read as gone, like PG row deletes
+ return s;
+}
+
+function encodedJsonBytes(v: any): number {
+ return Buffer.byteLength(JSON.stringify(v), "utf8");
+}
+
+export class NuQFdbQueue<JobData = any, JobReturnValue = any> {
+ public readonly ks: NuqFdbKeyspace;
+ public readonly groupOps: NuqFdbGroupOps | null;
+ // worker-local lease expiry tracking so renewLock can stay a blind write
+ private leaseExps: Map<string, number> = new Map();
+
+ constructor(
+ public readonly queueName: string,
+ public readonly options: {
+ // whether jobs in this queue participate in group (crawl) accounting
+ hasGroups: boolean;
+ // queue that group-finish jobs are emitted into (scrape -> crawl_finished)
+ finishedQueueName?: string;
+ // lease duration override, used by tests
+ leaseMs?: number;
+ },
+ ) {
+ this.ks = new NuqFdbKeyspace(queueName);
+ this.groupOps = options.hasGroups
+ ? new NuqFdbGroupOps(
+ this.ks,
+ options.finishedQueueName
+ ? new NuqFdbKeyspace(options.finishedQueueName)
+ : null,
+ )
+ : null;
+ }
+
+ private get db() {
+ return getNuqFdbDatabase();
+ }
+
+ private storeReturnvalueInline(): boolean {
+ // cloud stores results in GCS; self-host keeps them in the queue
+ return !config.GCS_BUCKET_NAME;
+ }
+
+ // === Enqueue
+
+ public async addJob(
+ id: string,
+ data: JobData,
+ options: NuQFdbJobOptions,
+ gate: NuQFdbGate,
+ ): Promise<NuQFdbJob<JobData, JobReturnValue>> {
+ const [job] = await this.addJobs([{ id, data, options }], gate);
+ return job;
+ }
+
+ public async addJobs(
+ jobs: AddJobInput<JobData>[],
+ gate: NuQFdbGate,
+ logger: Logger = _logger,
+ ): Promise<NuQFdbJob<JobData, JobReturnValue>[]> {
+ if (jobs.length === 0) return [];
+ const ownerId = normalizeOwnerId(jobs[0].options.ownerId);
+ if (jobs.some(j => normalizeOwnerId(j.options.ownerId) !== ownerId)) {
+ throw new Error("addJobs requires all jobs to share an owner");
+ }
+
+ const prepared = jobs.map(j => this.prepareAddJob(j, ownerId));
+ const results: NuQFdbJob<JobData, JobReturnValue>[] = [];
+ let batch: PreparedAddJob<JobData>[] = [];
+ let batchBytes = 0;
+
+ for (const job of prepared) {
+ if (job.estimatedAffectedBytes > ENQUEUE_SINGLE_JOB_BYTE_LIMIT) {
+ throw new Error(
+ `NuQ FDB job ${job.id} is too large to enqueue safely: estimated ${job.estimatedAffectedBytes} bytes of affected data exceeds ${ENQUEUE_SINGLE_JOB_BYTE_LIMIT}`,
+ );
+ }
+
+ if (job.estimatedAffectedBytes > ENQUEUE_BATCH_BYTE_BUDGET) {
+ logger.warn("NuQ FDB enqueue job exceeds batch budget", {
+ canonicalLog: "nuq-fdb/enqueue_batch",
+ queueName: this.queueName,
+ jobId: job.id,
+ estimatedAffectedBytes: job.estimatedAffectedBytes,
+ batchByteBudget: ENQUEUE_BATCH_BYTE_BUDGET,
+ transactionByteLimit: FDB_TRANSACTION_BYTE_LIMIT,
+ });
+ }
+
+ if (
+ batch.length > 0 &&
+ (batchBytes + job.estimatedAffectedBytes > ENQUEUE_BATCH_BYTE_BUDGET ||
+ batch.length >= ENQUEUE_MAX_JOBS_PER_TRANSACTION)
+ ) {
+ results.push(...(await this.addJobsBatch(batch, ownerId, gate)));
+ batch = [];
+ batchBytes = 0;
+ }
+
+ batch.push(job);
+ batchBytes += job.estimatedAffectedBytes;
+ }
+
+ if (batch.length > 0) {
+ results.push(...(await this.addJobsBatch(batch, ownerId, gate)));
+ }
+ return results;
+ }
+
+ private prepareAddJob(
+ job: AddJobInput<JobData>,
+ ownerId: string | null,
+ ): PreparedAddJob<JobData> {
+ const dataBuf = Buffer.from(JSON.stringify(job.data ?? null), "utf8");
+ const dataChunks = chunkBuffer(dataBuf, DATA_CHUNK_BYTES);
+ const estimatedAffectedBytes = this.estimateEnqueueAffectedBytes(
+ job,
+ ownerId,
+ dataChunks,
+ );
+ return {
+ ...job,
+ dataBuf,
+ dataChunks,
+ estimatedAffectedBytes,
+ };
+ }
+
+ private estimateEnqueueAffectedBytes(
+ job: AddJobInput<JobData>,
+ ownerId: string | null,
+ dataChunks: Buffer[],
+ ): number {
+ const ks = this.ks;
+ const now = Date.now();
+ const priority = job.options.priority ?? 0;
+ const gid = job.options.groupId;
+ const owner = ownerId ?? "";
+ const entry: QueueEntry = {
+ i: job.id,
+ o: owner,
+ g: gid,
+ p: priority,
+ f: F_GATED | F_CRAWL_GATED | F_LISTENABLE | F_ZDR | F_COUNTABLE | F_GACC,
+ c: now,
+ to: job.options.timesOutAt?.getTime(),
+ };
+ const meta: JobMeta = {
+ c: now,
+ p: priority,
+ o: owner,
+ g: gid,
+ f: entry.f,
+ to: entry.to,
+ dc: dataChunks.length,
+ };
+
+ let bytes =
+ ks.jobMeta(job.id).length +
+ encodedJsonBytes(meta) +
+ ks.jobStatus(job.id).length +
+ encodedJsonBytes({ s: "pending", st: 0 }) +
+ ks.readyPrefix(READY_SHARDS - 1, priority).length +
+ encodedJsonBytes(entry) +
+ ks.readyShardCount(READY_SHARDS - 1).length +
+ 8;
+
+ for (let i = 0; i < dataChunks.length; i++) {
+ bytes += ks.jobData(job.id, i).length + dataChunks[i].length;
+ }
+
+ if (owner) {
+ bytes +=
+ ks.teamLimit(owner).length +
+ 8 +
+ ks.teamActive(owner).length +
+ 8 +
+ ks.teamActiveIndex(owner).length +
+ 8 +
+ ks.teamPendingCount(owner).length +
+ 8 +
+ ks.teamShardCount(owner, 0).length +
+ 8 +
+ ks.teamPendingKey(owner, 0, priority, now, job.id).length +
+ encodedJsonBytes(entry);
+ }
+
+ if (gid) {
+ bytes +=
+ ks.groupMeta(gid).length +
+ ks.groupRemaining(gid).length +
+ 8 +
+ ks.groupCrawlActive(gid).length +
+ 8 +
+ ks.groupStatusCount(gid, "pending").length +
+ 8 +
+ ks.groupPendingCount(gid).length +
+ 8 +
+ ks.groupPendingKey(gid, priority, now, job.id).length +
+ encodedJsonBytes(entry) +
+ ks.groupJob(gid, job.id).length +
+ encodedJsonBytes({ m: 1, s: "pending" });
+ }
+
+ if (entry.to !== undefined) {
+ bytes += ks.backlogTimeout(timeBucket(job.id), entry.to, job.id).length;
+ }
+
+ // Account for versionstamp suffixes, conflict ranges, tuple overhead, and
+ // status loc variants. This intentionally overestimates so the batching
+ // boundary stays conservative as placement changes inside the transaction.
+ return bytes + 4096;
+ }
+
+ private async addJobsBatch(
+ jobs: PreparedAddJob<JobData>[],
+ ownerId: string | null,
+ gate: NuQFdbGate,
+ ): Promise<NuQFdbJob<JobData, JobReturnValue>[]> {
+ const ks = this.ks;
+ return await this.db.doTn(async tn => {
+ const txc = newTxContext();
+ const now = Date.now();
+ const out: NuQFdbJob<JobData, JobReturnValue>[] = [];
+
+ let free = Infinity;
+ if (gate.teamLimit !== null && ownerId !== null) {
+ const storedBuf = await tn.get(ks.teamLimit(ownerId));
+ const stored = storedBuf ? decodeI64(storedBuf) : null;
+ if (stored !== gate.teamLimit) {
+ tn.set(ks.teamLimit(ownerId), encodeI64(gate.teamLimit));
+ if (stored !== null && gate.teamLimit > stored) {
+ tn.set(ks.taskTeamRaise(ownerId), EMPTY);
+ }
+ }
+
+ const pend = decodeI64(
+ await tn.snapshot().get(ks.teamPendingCount(ownerId)),
+ );
+ if (pend + jobs.length > gate.queueCap) {
+ throw new QueueFullError(pend, gate.queueCap);
+ }
+
+ // big-limit teams tolerate a small admission overshoot in exchange for
+ // a conflict-free read; small-limit teams get the strict read
+ const active =
+ gate.teamLimit >= 256
+ ? decodeI64(await tn.snapshot().get(ks.teamActive(ownerId)))
+ : decodeI64(await tn.get(ks.teamActive(ownerId)));
+ free = Math.max(0, gate.teamLimit - active);
+ }
+
+ // crawl gate state per distinct live group
+ const groupMetas = new Map<string, GroupMeta | null>();
+ const crawlFree = new Map<string, number>();
+ if (this.options.hasGroups) {
+ for (const j of jobs) {
+ const gid = j.options.groupId;
+ if (!gid || groupMetas.has(gid)) continue;
+ const gMeta = decodeJson<GroupMeta>(await tn.get(ks.groupMeta(gid)));
+ groupMetas.set(gid, gMeta);
+ if (gMeta && gMeta.s === "active" && gate.teamLimit !== null) {
+ const effM = (gMeta.d ?? 0) > 0 ? 1 : gMeta.m;
+ if (effM !== undefined) {
+ const cact = decodeI64(await tn.get(ks.groupCrawlActive(gid)));
+ crawlFree.set(gid, effM - cact);
+ }
+ }
+ }
+ }
+
+ let granted = 0;
+ const crawlAcquired = new Map<string, number>();
+
+ for (const j of jobs) {
+ const gid = j.options.groupId;
+ const gMeta = gid ? groupMetas.get(gid) : null;
+ const groupLive = !!gMeta && gMeta.s === "active";
+ const gated = gate.teamLimit !== null && !j.options.bypassGate;
+ const crawlGated = gated && !!gid && groupLive && crawlFree.has(gid!);
+ const countable =
+ this.options.hasGroups &&
+ groupLive &&
+ (j.data as any)?.mode === "single_urls";
+
+ let flags = 0;
+ if (gated) flags |= F_GATED;
+ if (crawlGated) flags |= F_CRAWL_GATED;
+ if (j.options.listenable) flags |= F_LISTENABLE;
+ if ((j.data as any)?.zeroDataRetention) flags |= F_ZDR;
+ if (countable) flags |= F_COUNTABLE;
+ if (groupLive) flags |= F_GACC;
+
+ const timesOutAt = gated ? j.options.timesOutAt?.getTime() : undefined;
+ const entry: QueueEntry = {
+ i: j.id,
+ o: ownerId ?? "",
+ g: gid,
+ p: j.options.priority ?? 0,
+ f: flags,
+ c: now,
+ to: timesOutAt,
+ };
+
+ const meta: JobMeta = {
+ c: now,
+ p: entry.p,
+ o: entry.o,
+ g: gid,
+ f: flags,
+ to: timesOutAt,
+ dc: j.dataChunks.length,
+ };
+ tn.set(ks.jobMeta(j.id), encodeJson(meta));
+ j.dataChunks.forEach((chunk, ci) =>
+ tn.set(ks.jobData(j.id, ci), chunk),
+ );
+
+ let placedStatus: NuqFdbJobStatus;
+ if (!gated) {
+ pushReady(tn, ks, entry, txc);
+ setStatusQueued(tn, ks, j.id);
+ placedStatus = "queued";
+ } else if (crawlGated && crawlFree.get(gid!)! <= 0) {
+ const loc = appendCrawlPending(tn, ks, entry);
+ setStatusPending(tn, ks, j.id, loc);
+ placedStatus = "pending";
+ } else {
+ if (crawlGated) {
+ crawlFree.set(gid!, crawlFree.get(gid!)! - 1);
+ crawlAcquired.set(gid!, (crawlAcquired.get(gid!) ?? 0) + 1);
+ }
+ if (free > 0) {
+ free--;
+ granted++;
+ pushReady(tn, ks, entry, txc);
+ setStatusQueued(tn, ks, j.id);
+ placedStatus = "queued";
+ } else {
+ const loc = appendTeamPending(tn, ks, entry);
+ setStatusPending(tn, ks, j.id, loc);
+ placedStatus = "pending";
+ }
+ }
+
+ if (groupLive && gid) {
+ tn.add(ks.groupRemaining(gid), ONE);
+ setGroupJobIndex(tn, ks, gid, j.id, countable, placedStatus);
+ if (countable) bumpGroupStatusCount(tn, ks, gid, placedStatus, 1);
+ }
+
+ out.push({
+ id: j.id,
+ status: placedStatus === "pending" ? "backlog" : "queued",
+ createdAt: new Date(now),
+ priority: entry.p,
+ data: j.data,
+ ownerId: entry.o || undefined,
+ groupId: gid,
+ });
+ }
+
+ if (granted > 0 && ownerId !== null) {
+ bumpTeamActive(tn, ks, ownerId, granted);
+ }
+ for (const [gid, n] of crawlAcquired) {
+ tn.add(ks.groupCrawlActive(gid), encodeI64(n));
+ }
+
+ return out;
+ });
+ }
+
+ // === Take (worker dequeue)
+
+ public async getJobToProcess(
+ logger: Logger = _logger,
+ ): Promise<NuQFdbJob<JobData, JobReturnValue> | null> {
+ const startedAt = Date.now();
+ // blind random probes win at high occupancy (no coordination, conflicts
+ // spread across shards); the occupancy scan below covers the sparse case
+ const PROBES = 4;
+ const tried = new Set<number>();
+ let randomDropped = 0;
+ while (tried.size < PROBES) {
+ const shard = Math.floor(Math.random() * READY_SHARDS);
+ if (tried.has(shard)) continue;
+ const result = await this.takeFromShard(shard);
+ if (result === "empty") {
+ tried.add(shard);
+ continue;
+ }
+ if (result === "dropped") {
+ // tombstone or cancelled-group divert consumed an entry; same shard
+ // may hold live work, try it again without burning a probe
+ randomDropped++;
+ continue;
+ }
+ this.leaseExps.set(result.id, result.leaseExpiresAt!.getTime());
+ logger.debug("NuQ FDB dequeue attempt", {
+ canonicalLog: "nuq-fdb/dequeue",
+ queueName: this.queueName,
+ result: "job",
+ path: "random_probe",
+ durationMs: Date.now() - startedAt,
+ readyShards: READY_SHARDS,
+ randomProbeCount: tried.size + 1,
+ randomEmptyCount: tried.size,
+ randomDroppedCount: randomDropped,
+ });
+ return result;
+ }
+
+ // sparse queue: find non-empty shards via their occupancy counters
+ const candidates = await this.db.doTn(async tn => {
+ const r = this.ks.readyShardCountRange();
+ const counts = await tn.snapshot().getRangeAll(r.begin, r.end);
+ const nonEmpty: number[] = [];
+ for (const [key, value] of counts) {
+ if (decodeI64(value as Buffer) > 0) {
+ nonEmpty.push(Number(this.ks.unpackId(key as Buffer)));
+ }
+ }
+ return nonEmpty;
+ });
+ for (let i = candidates.length - 1; i > 0; i--) {
+ const j = Math.floor(Math.random() * (i + 1));
+ [candidates[i], candidates[j]] = [candidates[j], candidates[i]];
+ }
+ let fallbackDropped = 0;
+ let fallbackEmpty = 0;
+ let fallbackAttempts = 0;
+ for (const shard of candidates.slice(0, 8)) {
+ if (tried.has(shard)) continue;
+ for (let attempt = 0; attempt < 4; attempt++) {
+ fallbackAttempts++;
+ const result = await this.takeFromShard(shard);
+ if (result === "empty") {
+ fallbackEmpty++;
+ break;
+ }
+ if (result === "dropped") {
+ fallbackDropped++;
+ continue;
+ }
+ this.leaseExps.set(result.id, result.leaseExpiresAt!.getTime());
+ logger.debug("NuQ FDB dequeue attempt", {
+ canonicalLog: "nuq-fdb/dequeue",
+ queueName: this.queueName,
+ result: "job",
+ path: "sparse_scan",
+ durationMs: Date.now() - startedAt,
+ readyShards: READY_SHARDS,
+ randomProbeCount: tried.size,
+ randomEmptyCount: tried.size,
+ randomDroppedCount: randomDropped,
+ nonEmptyCandidateCount: candidates.length,
+ fallbackAttemptCount: fallbackAttempts,
+ fallbackEmptyCount: fallbackEmpty,
+ fallbackDroppedCount: fallbackDropped,
+ });
+ return result;
+ }
+ }
+ logger.debug("NuQ FDB dequeue attempt", {
+ canonicalLog: "nuq-fdb/dequeue",
+ queueName: this.queueName,
+ result: "empty",
+ path: "sparse_scan",
+ durationMs: Date.now() - startedAt,
+ readyShards: READY_SHARDS,
+ randomProbeCount: tried.size,
+ randomEmptyCount: tried.size,
+ randomDroppedCount: randomDropped,
+ nonEmptyCandidateCount: candidates.length,
+ fallbackAttemptCount: fallbackAttempts,
+ fallbackEmptyCount: fallbackEmpty,
+ fallbackDroppedCount: fallbackDropped,
+ });
+ return null;
+ }
+
+ private async takeFromShard(
+ shard: number,
+ ): Promise<NuQFdbJob<JobData, JobReturnValue> | "empty" | "dropped"> {
+ const ks = this.ks;
+ return await this.db.doTn(async tn => {
+ const txc = newTxContext();
+ const now = Date.now();
+ const range = ks.readyShardRange(shard);
+ const head = await tn.getRangeAll(range.begin, range.end, { limit: 1 });
+ if (head.length === 0) return "empty";
+ const [key, value] = head[0];
+ const e = decodeJson<QueueEntry>(value as Buffer)!;
+ tn.clear(key as Buffer);
+ tn.add(ks.readyShardCount(shard), MINUS_ONE);
+
+ const st = decodeJson<JobStatusRecord>(await tn.get(ks.jobStatus(e.i)));
+ if (!st || st.s !== "queued") {
+ // tombstone (removed job) -- slots were released by the remover
+ return "dropped";
+ }
+
+ if (e.g && this.options.hasGroups) {
+ const gMeta = decodeJson<GroupMeta>(
+ await tn.snapshot().get(ks.groupMeta(e.g)),
+ );
+ if (gMeta && gMeta.s === "cancelled") {
+ // lazy cancellation: this job dies at take time
+ if (e.f & F_GACC) {
+ const gj = decodeJson<GroupJobIndexValue>(
+ await tn.get(ks.groupJob(e.g, e.i)),
+ );
+ if (gj) {
+ tn.clear(ks.groupJob(e.g, e.i));
+ tn.add(ks.groupRemaining(e.g), MINUS_ONE);
+ if (e.f & F_COUNTABLE)
+ bumpGroupStatusCount(tn, ks, e.g, "queued", -1);
+ tn.set(ks.taskGroupFinish(e.g), EMPTY);
+ }
+ }
+ deleteJobRecords(tn, ks, e.i);
+ await releaseSlotsAndPromote(
+ tn,
+ ks,
+ e,
+ { team: true, crawl: true },
+ now,
+ txc,
+ );
+ return "dropped";
+ }
+ }
+
+ const meta = decodeJson<JobMeta>(
+ await tn.snapshot().get(ks.jobMeta(e.i)),
+ );
+ if (!meta) return "dropped";
+ const dataRange = ks.jobDataRange(e.i);
+ const dataParts = await tn
+ .snapshot()
+ .getRangeAll(dataRange.begin, dataRange.end);
+ const data = JSON.parse(
+ Buffer.concat(dataParts.map(([, v]) => v as Buffer)).toString("utf8"),
+ );
+
+ const lock = randomUUID();
+ const exp = now + (this.options.leaseMs ?? LEASE_MS);
+ const rec: JobStatusRecord = { s: "active", l: lock, e: exp, st: st.st };
+ tn.set(ks.jobStatus(e.i), encodeJson(rec));
+ tn.set(ks.lease(timeBucket(e.i), exp, e.i), encodeJson({ l: lock }));
+ if (e.g && e.f & F_COUNTABLE) {
+ bumpGroupStatusCount(tn, ks, e.g, "queued", -1);
+ bumpGroupStatusCount(tn, ks, e.g, "active", 1);
+ setGroupJobIndex(tn, ks, e.g, e.i, true, "active");
+ }
+
+ return {
+ id: e.i,
+ status: "active" as const,
+ createdAt: new Date(meta.c),
+ priority: meta.p,
+ data,
+ lock,
+ leaseExpiresAt: new Date(exp),
+ ownerId: meta.o || undefined,
+ groupId: meta.g,
+ };
+ });
+ }
+
+ // === Leases
+
+ public async renewLock(
+ id: string,
+ lock: string,
+ logger: Logger = _logger,
+ ): Promise<boolean> {
+ const oldExp = this.leaseExps.get(id);
+ if (oldExp === undefined) return false;
+ const ks = this.ks;
+ const newExp = Date.now() + (this.options.leaseMs ?? LEASE_MS);
+ try {
+ await this.db.doTn(async tn => {
+ // blind writes only; the sweeper validates the lock before reaping
+ tn.clear(ks.lease(timeBucket(id), oldExp, id));
+ tn.set(ks.lease(timeBucket(id), newExp, id), encodeJson({ l: lock }));
+ // the lease index moved, keep the status record's expiry in sync for
+ // observability; only the worker holding the lock writes this
+ const st = decodeJson<JobStatusRecord>(
+ await tn.snapshot().get(ks.jobStatus(id)),
+ );
+ if (!st || st.s !== "active" || st.l !== lock) {
+ throw new LockLostError();
+ }
+ tn.set(ks.jobStatus(id), encodeJson({ ...st, e: newExp }));
+ });
+ } catch (e) {
+ if (e instanceof LockLostError) {
+ this.leaseExps.delete(id);
+ return false;
+ }
+ throw e;
+ }
+ this.leaseExps.set(id, newExp);
+ return true;
+ }
+
+ // === Finish / fail
+
+ public async jobFinish(
+ id: string,
+ lock: string,
+ returnvalue: JobReturnValue | null,
+ logger: Logger = _logger,
+ ): Promise<boolean> {
+ return this.finishOrFail(id, lock, "completed", returnvalue ?? null, null);
+ }
+
+ public async jobFail(
+ id: string,
+ lock: string,
+ failedReason: string,
+ logger: Logger = _logger,
+ ): Promise<boolean> {
+ return this.finishOrFail(id, lock, "failed", null, failedReason);
+ }
+
+ private async finishOrFail(
+ id: string,
+ lock: string,
+ outcome: "completed" | "failed",
+ returnvalue: any,
+ failedReason: string | null,
+ ): Promise<boolean> {
+ const ks = this.ks;
+ const ok = await this.db.doTn(async tn => {
+ const txc = newTxContext();
+ const now = Date.now();
+ const st = decodeJson<JobStatusRecord>(await tn.get(ks.jobStatus(id)));
+ if (!st) return false;
+ // idempotency: a commit_unknown_result retry lands here
+ if (st.s === outcome && st.l === lock) return true;
+ if (st.s !== "active" || st.l !== lock) return false;
+ const meta = decodeJson<JobMeta>(await tn.get(ks.jobMeta(id)));
+ if (!meta) return false;
+
+ const rec: JobStatusRecord = {
+ s: outcome,
+ l: lock,
+ st: st.st,
+ fa: now,
+ };
+ tn.set(ks.jobStatus(id), encodeJson(rec));
+ if (st.e !== undefined) tn.clear(ks.lease(timeBucket(id), st.e, id));
+
+ if (outcome === "completed") {
+ if (
+ returnvalue !== null &&
+ this.storeReturnvalueInline() &&
+ !(meta.f & F_ZDR)
+ ) {
+ const buf = Buffer.from(JSON.stringify(returnvalue), "utf8");
+ chunkBuffer(buf, DATA_CHUNK_BYTES).forEach((chunk, ci) =>
+ tn.set(ks.jobReturnvalue(id, ci), chunk),
+ );
+ }
+ } else {
+ tn.set(
+ ks.jobFailedReason(id),
+ Buffer.from((failedReason ?? "").slice(0, DATA_CHUNK_BYTES), "utf8"),
+ );
+ }
+
+ // shed job data early on cloud (results live in GCS) and for ZDR
+ if (config.GCS_BUCKET_NAME || meta.f & F_ZDR) {
+ const r = ks.jobDataRange(id);
+ tn.clearRange(r.begin, r.end);
+ }
+
+ if (meta.g && meta.f & F_GACC && this.groupOps) {
+ await this.groupOps.terminalAccounting(
+ tn,
+ meta.g,
+ id,
+ "active",
+ outcome,
+ !!(meta.f & F_COUNTABLE),
+ now,
+ txc,
+ );
+ }
+
+ const entry: QueueEntry = {
+ i: id,
+ o: meta.o,
+ g: meta.g,
+ p: meta.p,
+ f: meta.f,
+ c: meta.c,
+ to: meta.to,
+ };
+ await releaseSlotsAndPromote(
+ tn,
+ ks,
+ entry,
+ { team: true, crawl: true },
+ now,
+ txc,
+ );
+
+ if (!meta.g) {
+ const retention =
+ outcome === "completed"
+ ? COMPLETED_STANDALONE_RETENTION_MS
+ : FAILED_STANDALONE_RETENTION_MS;
+ tn.set(ks.jobExpiry(timeBucket(id), now + retention, id), EMPTY);
+ }
+ return true;
+ });
+ if (ok) this.leaseExps.delete(id);
+ return ok;
+ }
+
+ // === Reads
+
+ private async readJob(
+ tn: Transaction,
+ id: string,
+ ): Promise<NuQFdbJob<JobData, JobReturnValue> | null> {
+ const ks = this.ks;
+ const snap = tn.snapshot();
+ const [metaBuf, stBuf] = await Promise.all([
+ snap.get(ks.jobMeta(id)),
+ snap.get(ks.jobStatus(id)),
+ ]);
+ const meta = decodeJson<JobMeta>(metaBuf);
+ const st = decodeJson<JobStatusRecord>(stBuf);
+ if (!meta || !st) return null;
+ const status = externalStatus(st.s);
+ if (status === null) return null;
+
+ const dataRange = ks.jobDataRange(id);
+ const dataParts = await snap.getRangeAll(dataRange.begin, dataRange.end);
+ const data =
+ dataParts.length > 0
+ ? JSON.parse(
+ Buffer.concat(dataParts.map(([, v]) => v as Buffer)).toString(
+ "utf8",
+ ),
+ )
+ : null;
+
+ let returnvalue: any = undefined;
+ if (st.s === "completed") {
+ const rvRange = ks.jobReturnvalueRange(id);
+ const rvParts = await snap.getRangeAll(rvRange.begin, rvRange.end);
+ if (rvParts.length > 0) {
+ returnvalue = JSON.parse(
+ Buffer.concat(rvParts.map(([, v]) => v as Buffer)).toString("utf8"),
+ );
+ } else {
+ returnvalue = null;
+ }
+ }
+ let failedReason: string | undefined = undefined;
+ if (st.s === "failed") {
+ const frBuf = await snap.get(ks.jobFailedReason(id));
+ failedReason = frBuf ? frBuf.toString("utf8") : undefined;
+ }
+
+ return {
+ id,
+ status,
+ createdAt: new Date(meta.c),
+ priority: meta.p,
+ data,
+ finishedAt: st.fa !== undefined ? new Date(st.fa) : undefined,
+ returnvalue,
+ failedReason,
+ lock: st.s === "active" ? st.l : undefined,
+ ownerId: meta.o || undefined,
+ groupId: meta.g,
+ };
+ }
+
+ public async getJob(
+ id: string,
+ logger: Logger = _logger,
+ ): Promise<NuQFdbJob<JobData, JobReturnValue> | null> {
+ return await this.db.doTn(async tn => this.readJob(tn, id));
+ }
+
+ // cheap existence probe used by the dual-backend router
+ public async hasJob(id: string): Promise<boolean> {
+ return await this.db.doTn(async tn => {
+ const st = await tn.snapshot().get(this.ks.jobStatus(id));
+ return st !== undefined && st !== null;
+ });
+ }
+
+ public async getJobs(
+ ids: string[],
+ logger: Logger = _logger,
+ ): Promise<NuQFdbJob<JobData, JobReturnValue>[]> {
+ if (ids.length === 0) return [];
+ const out: NuQFdbJob<JobData, JobReturnValue>[] = [];
+ const BATCH = 100;
+ for (let i = 0; i < ids.length; i += BATCH) {
+ const batch = ids.slice(i, i + BATCH);
+ const jobs = await this.db.doTn(async tn =>
+ Promise.all(batch.map(id => this.readJob(tn, id))),
+ );
+ out.push(
+ ...jobs.filter(
+ (j): j is NuQFdbJob<JobData, JobReturnValue> => j !== null,
+ ),
+ );
+ }
+ return out;
+ }
+
+ public async getJobsWithStatus(
+ ids: string[],
+ status: NuQJobStatusCompat,
+ logger: Logger = _logger,
+ ): Promise<NuQFdbJob<JobData, JobReturnValue>[]> {
+ return (await this.getJobs(ids, logger)).filter(j => j.status === status);
+ }
+
+ public async getJobsWithStatuses(
+ ids: string[],
+ statuses: NuQJobStatusCompat[],
+ logger: Logger = _logger,
+ ): Promise<NuQFdbJob<JobData, JobReturnValue>[]> {
+ const set = new Set(statuses);
+ return (await this.getJobs(ids, logger)).filter(j => set.has(j.status));
+ }
+
+ // === Remove
+
+ public async removeJob(
+ id: string,
+ logger: Logger = _logger,
+ ): Promise<boolean> {
+ const ks = this.ks;
+ return await this.db.doTn(async tn => {
+ const txc = newTxContext();
+ const now = Date.now();
+ const st = decodeJson<JobStatusRecord>(await tn.get(ks.jobStatus(id)));
+ if (!st || st.s === "cancelled") return false;
+ const meta = decodeJson<JobMeta>(await tn.get(ks.jobMeta(id)));
+ if (!meta) return false;
+ const entry: QueueEntry = {
+ i: id,
+ o: meta.o,
+ g: meta.g,
+ p: meta.p,
+ f: meta.f,
+ c: meta.c,
+ to: meta.to,
+ };
+ const countable = !!(meta.f & F_COUNTABLE);
+ const accounted = !!(meta.f & F_GACC) && !!meta.g && !!this.groupOps;
+
+ if (st.s === "pending") {
+ clearPendingPlacement(tn, ks, id, meta.o, meta.g, st.loc!, meta.to);
+ // team-pending and delayed jobs hold a crawl slot
+ if (st.loc!.k !== "gq") {
+ await releaseSlotsAndPromote(
+ tn,
+ ks,
+ entry,
+ { team: false, crawl: true },
+ now,
+ txc,
+ );
+ }
+ if (accounted) {
+ tn.clear(ks.groupJob(meta.g!, id));
+ tn.add(ks.groupRemaining(meta.g!), MINUS_ONE);
+ if (countable) bumpGroupStatusCount(tn, ks, meta.g!, "pending", -1);
+ tn.set(ks.taskGroupFinish(meta.g!), EMPTY);
+ }
+ deleteJobRecords(tn, ks, id);
+ } else if (st.s === "queued" || st.s === "active") {
+ tn.set(
+ ks.jobStatus(id),
+ encodeJson({ s: "cancelled", st: st.st } satisfies JobStatusRecord),
+ );
+ if (st.s === "active" && st.e !== undefined) {
+ tn.clear(ks.lease(timeBucket(id), st.e, id));
+ }
+ await releaseSlotsAndPromote(
+ tn,
+ ks,
+ entry,
+ { team: true, crawl: true },
+ now,
+ txc,
+ );
+ if (accounted) {
+ tn.clear(ks.groupJob(meta.g!, id));
+ tn.add(ks.groupRemaining(meta.g!), MINUS_ONE);
+ if (countable) bumpGroupStatusCount(tn, ks, meta.g!, st.s, -1);
+ tn.set(ks.taskGroupFinish(meta.g!), EMPTY);
+ }
+ // status tombstone stays for take-side dedupe; sweeper GCs the records
+ tn.set(
+ ks.jobExpiry(
+ timeBucket(id),
+ now + COMPLETED_STANDALONE_RETENTION_MS,
+ id,
+ ),
+ EMPTY,
+ );
+ } else {
+ // terminal: drop the records, like the PG row delete
+ if (accounted) {
+ tn.clear(ks.groupJob(meta.g!, id));
+ if (countable) bumpGroupStatusCount(tn, ks, meta.g!, st.s, -1);
+ }
+ deleteJobRecords(tn, ks, id);
+ }
+ return true;
+ });
+ }
+
+ public async removeJobs(
+ ids: string[],
+ logger: Logger = _logger,
+ ): Promise<void> {
+ for (const id of ids) {
+ await this.removeJob(id, logger);
+ }
+ }
+
+ // === waitForJob
+
+ public async waitForJob(
+ id: string,
+ timeout: number | null,
+ logger: Logger = _logger,
+ ): Promise<JobReturnValue> {
+ const ks = this.ks;
+ const deadline = timeout !== null ? Date.now() + timeout : null;
+ while (true) {
+ const { st, watch } = await this.db.doTn(async tn => {
+ const stBuf = await tn.get(ks.jobStatus(id));
+ const st = decodeJson<JobStatusRecord>(stBuf);
+ if (
+ st &&
+ (st.s === "completed" || st.s === "failed" || st.s === "cancelled")
+ ) {
+ return { st, watch: null };
+ }
+ return { st, watch: tn.watch(ks.jobStatus(id)) };
+ });
+
+ if (!st) {
+ if (watch) watch.cancel();
+ throw new Error("Job raced out while waiting for it");
+ }
+
+ if (st.s === "completed") {
+ if (watch) watch.cancel();
+ const job = await this.getJob(id, logger);
+ if (!job) throw new Error("Job raced out while waiting for it");
+ return job.returnvalue!;
+ }
+ if (st.s === "failed") {
+ if (watch) watch.cancel();
+ const job = await this.getJob(id, logger);
+ throw new Error(job?.failedReason ?? "Job failed");
+ }
+ if (st.s === "cancelled") {
+ if (watch) watch.cancel();
+ throw new Error("Job raced out while waiting for it");
+ }
+
+ const remaining = deadline !== null ? deadline - Date.now() : null;
+ if (remaining !== null && remaining <= 0) {
+ watch!.cancel();
+ throw new Error("Timed out");
+ }
+
+ const fired = await Promise.race([
+ watch!.promise.then(() => true),
+ new Promise<false>(resolve =>
+ setTimeout(
+ () => resolve(false),
+ remaining !== null ? Math.min(remaining, 30_000) : 30_000,
+ ),
+ ),
+ ]);
+ if (!fired) {
+ watch!.cancel();
+ if (deadline !== null && Date.now() >= deadline) {
+ throw new Error("Timed out");
+ }
+ }
+ }
+ }
+
+ // === Group-scoped reads
+
+ public async getGroupNumericStats(
+ groupId: string,
+ logger: Logger = _logger,
+ ): Promise<Record<NuQJobStatusCompat, number>> {
+ const ks = this.ks;
+ return await this.db.doTn(async tn => {
+ const r = ks.groupStatusCountRange(groupId);
+ const counts = await tn.snapshot().getRangeAll(r.begin, r.end);
+ const out: Record<NuQJobStatusCompat, number> = {
+ queued: 0,
+ active: 0,
+ completed: 0,
+ failed: 0,
+ backlog: 0,
+ };
+ for (const [key, value] of counts) {
+ const status = ks.unpackId(key as Buffer);
+ const n = Math.max(0, decodeI64(value as Buffer));
+ if (status === "pending") out.backlog += n;
+ else if (status in out) out[status as NuQJobStatusCompat] += n;
+ }
+ return out;
+ });
+ }
+
+ public async getGroupAnyJob(
+ groupId: string,
+ ownerId: string,
+ logger: Logger = _logger,
+ ): Promise<NuQFdbJob<JobData, JobReturnValue> | null> {
+ const ks = this.ks;
+ const owner = normalizeOwnerId(ownerId);
+ const candidateId = await this.db.doTn(async tn => {
+ const r = ks.groupJobRange(groupId);
+ let begin: Buffer | FdbKeySelector = r.begin;
+ // scan for the first countable (single_urls) member
+ for (let scanned = 0; scanned < 2000; ) {
+ const batch = await tn
+ .snapshot()
+ .getRangeAll(begin as any, r.end, { limit: 200 });
+ if (batch.length === 0) return null;
+ for (const [key, value] of batch) {
+ const gj = decodeJson<GroupJobIndexValue>(value as Buffer);
+ if (gj?.m === 1) return ks.unpackId(key as Buffer);
+ }
+ scanned += batch.length;
+ const lastKey = batch[batch.length - 1][0] as Buffer;
+ begin = {
+ key: lastKey,
+ orEqual: true,
+ offset: 1,
+ _isKeySelector: true,
+ };
+ }
+ return null;
+ });
+ if (!candidateId) return null;
+ const job = await this.getJob(candidateId, logger);
+ if (!job) return null;
+ if (owner !== null && job.ownerId !== owner) return null;
+ return job;
+ }
+
+ public async getCrawlJobsForListing(
+ groupId: string,
+ limit: number,
+ offset: number,
+ logger: Logger = _logger,
+ ): Promise<NuQFdbJob<JobData, JobReturnValue>[]> {
+ const ks = this.ks;
+ const ids = await this.db.doTn(async tn => {
+ const r = ks.groupDoneRange(groupId);
+ const rows = await tn
+ .snapshot()
+ .getRangeAll(r.begin, r.end, { limit: offset + limit });
+ return rows
+ .slice(offset)
+ .map(([, value]) => (value as Buffer).toString("utf8"));
+ });
+ const jobs = await this.getJobs(ids, logger);
+ const byId = new Map(jobs.map(j => [j.id, j]));
+ return ids
+ .map(id => byId.get(id))
+ .filter(
+ (j): j is NuQFdbJob<JobData, JobReturnValue> =>
+ !!j && j.status === "completed",
+ );
+ }
+
+ // === Introspection used by status/admin endpoints
+
+ public async getTeamActiveCount(teamId: string): Promise<number> {
+ const owner = normalizeOwnerId(teamId);
+ if (owner === null) return 0;
+ return await this.db.doTn(async tn =>
+ Math.max(
+ 0,
+ decodeI64(await tn.snapshot().get(this.ks.teamActive(owner))),
+ ),
+ );
+ }
+
+ public async getTeamActiveCounts(): Promise<Map<string, number>> {
+ const ks = this.ks;
+ return await this.db.doTn(async tn => {
+ const r = ks.teamActiveIndexRange();
+ const rows = await tn.snapshot().getRangeAll(r.begin, r.end);
+ const counts = new Map<string, number>();
+ for (const [key, value] of rows) {
+ const parts = ks.unpack(key as Buffer);
+ const teamId = parts[2];
+ if (typeof teamId !== "string") continue;
+ const count = Math.max(0, decodeI64(value as Buffer));
+ if (count > 0) counts.set(teamId, count);
+ }
+ return counts;
+ });
+ }
+
+ public async getTeamPendingCount(teamId: string): Promise<number> {
+ const owner = normalizeOwnerId(teamId);
+ if (owner === null) return 0;
+ return await this.db.doTn(async tn =>
+ Math.max(
+ 0,
+ decodeI64(await tn.snapshot().get(this.ks.teamPendingCount(owner))),
+ ),
+ );
+ }
+}
+
+class LockLostError extends Error {}
diff --git a/apps/api/src/services/worker/nuq-fdb/slots.ts b/apps/api/src/services/worker/nuq-fdb/slots.ts
new file mode 100644
index 000000000..f85d30a5e
--- /dev/null
+++ b/apps/api/src/services/worker/nuq-fdb/slots.ts
@@ -0,0 +1,136 @@
+import type { Transaction } from "foundationdb";
+import { getNuqFdbDatabase } from "./client";
+import {
+ NuqFdbKeyspace,
+ QueueEntry,
+ decodeJson,
+ encodeJson,
+ timeBucket,
+ F_GATED,
+ normalizeOwnerId,
+} from "./keyspace";
+import { bumpTeamActive, newTxContext, releaseSlotsAndPromote } from "./ops";
+
+// External slots: capacity consumed by things that are not queue jobs (sync
+// scrapes via the team semaphore, browser sessions). They unconditionally bump
+// the team active counter -- possibly past the limit, matching the old Redis
+// behavior where sync holders were mirrored into the same ZSET -- and hand
+// their slot through the normal promotion chain on release.
+
+type ExternalSlotRecord = {
+ e: number; // expiry ms
+};
+
+export class NuqFdbExternalSlots {
+ constructor(public readonly ks: NuqFdbKeyspace) {}
+
+ private get db() {
+ return getNuqFdbDatabase();
+ }
+
+ private key(teamId: string, holderId: string): Buffer {
+ return this.ks.pack(["xs", teamId, holderId]);
+ }
+
+ private expiryKey(bucket: number, expMs: number, holderId: string): Buffer {
+ return this.ks.pack(["xsexp", bucket, expMs, holderId]);
+ }
+
+ public expiryScanRange(bucket: number, untilMs: number) {
+ return {
+ begin: this.ks.pack(["xsexp", bucket]),
+ end: this.ks.pack(["xsexp", bucket, untilMs]),
+ };
+ }
+
+ // Acquires (or renews) an external slot. Unconditional: never blocks on the
+ // team limit; the caller's own gate (Lua semaphore, session limits) decides
+ // admission. Re-acquiring an existing holder just extends its expiry.
+ public async acquire(
+ teamId: string,
+ holderId: string,
+ ttlMs: number,
+ ): Promise<void> {
+ const owner = normalizeOwnerId(teamId);
+ if (owner === null) return;
+ const now = Date.now();
+ const exp = now + ttlMs;
+ await this.db.doTn(async tn => {
+ const existing = decodeJson<ExternalSlotRecord>(
+ await tn.get(this.key(owner, holderId)),
+ );
+ if (existing) {
+ tn.clear(this.expiryKey(timeBucket(holderId), existing.e, holderId));
+ } else {
+ bumpTeamActive(tn, this.ks, owner, 1);
+ }
+ tn.set(
+ this.key(owner, holderId),
+ encodeJson({ e: exp } satisfies ExternalSlotRecord),
+ );
+ tn.set(
+ this.expiryKey(timeBucket(holderId), exp, holderId),
+ encodeJson({ t: owner }),
+ );
+ });
+ }
+
+ // Releases the slot, handing it to a pending job when one exists.
+ public async release(teamId: string, holderId: string): Promise<void> {
+ const owner = normalizeOwnerId(teamId);
+ if (owner === null) return;
+ await this.db.doTn(async tn => {
+ await this.releaseInTxn(tn, owner, holderId);
+ });
+ }
+
+ public async releaseInTxn(
+ tn: Transaction,
+ owner: string,
+ holderId: string,
+ ): Promise<boolean> {
+ const existing = decodeJson<ExternalSlotRecord>(
+ await tn.get(this.key(owner, holderId)),
+ );
+ if (!existing) return false;
+ tn.clear(this.key(owner, holderId));
+ tn.clear(this.expiryKey(timeBucket(holderId), existing.e, holderId));
+ const entry: QueueEntry = {
+ i: holderId,
+ o: owner,
+ p: 0,
+ f: F_GATED,
+ c: 0,
+ };
+ await releaseSlotsAndPromote(
+ tn,
+ this.ks,
+ entry,
+ { team: true, crawl: false },
+ Date.now(),
+ newTxContext(),
+ );
+ return true;
+ }
+
+ // Sweeper hook: releases slots whose holders stopped renewing.
+ public async sweepExpired(now: number, buckets: number): Promise<void> {
+ for (let b = 0; b < buckets; b++) {
+ const r = this.expiryScanRange(b, now);
+ const due = await this.db.doTn(async tn =>
+ tn.snapshot().getRangeAll(r.begin, r.end, { limit: 50 }),
+ );
+ for (const [key, value] of due) {
+ const holderId = this.ks.unpackId(key as Buffer);
+ const rec = decodeJson<{ t: string }>(value as Buffer);
+ if (!rec) continue;
+ await this.db.doTn(async tn => {
+ // releaseInTxn validates the record still exists (and clears this
+ // index entry); if the holder renewed, just drop the stale entry
+ const released = await this.releaseInTxn(tn, rec.t, holderId);
+ if (!released) tn.clear(key as Buffer);
+ });
+ }
+ }
+ }
+}
diff --git a/apps/api/src/services/worker/nuq-fdb/sweeper.ts b/apps/api/src/services/worker/nuq-fdb/sweeper.ts
new file mode 100644
index 000000000..803064a9a
--- /dev/null
+++ b/apps/api/src/services/worker/nuq-fdb/sweeper.ts
@@ -0,0 +1,611 @@
+import { randomUUID } from "crypto";
+import { Logger } from "winston";
+import { logger as _logger } from "../../../lib/logger";
+import { getNuqFdbDatabase } from "./client";
+import {
+ NuqFdbKeyspace,
+ JobMeta,
+ JobStatusRecord,
+ GroupMeta,
+ QueueEntry,
+ decodeI64,
+ decodeJson,
+ encodeJson,
+ timeBucket,
+ TIME_BUCKETS,
+ F_GATED,
+ F_CRAWL_GATED,
+ F_COUNTABLE,
+ F_GACC,
+} from "./keyspace";
+import {
+ ONE,
+ MINUS_ONE,
+ EMPTY,
+ MAX_STALLS,
+ FAILED_STANDALONE_RETENTION_MS,
+ newTxContext,
+ pushReady,
+ setStatusQueued,
+ promoteEntryToReady,
+ clearPendingPlacement,
+ releaseSlotsAndPromote,
+ admitThroughTeamGate,
+ deleteJobRecords,
+ popTeamPending,
+ setGroupJobIndex,
+ bumpGroupStatusCount,
+ bumpTeamActive,
+ GroupJobIndexValue,
+} from "./ops";
+import { NuQFdbQueue } from "./queue";
+import { NuqFdbExternalSlots } from "./slots";
+
+const SWEEP_LOCK_TTL_MS = 15_000;
+const SWEEP_BATCH = 50;
+const STALL_FAILED_REASON = "Job stalled too many times";
+
+type SweepLagStats = {
+ dueCount: number;
+ processedCount: number;
+ oldestOverdueAgeMs: number;
+ saturatedBucketCount: number;
+ durationMs: number;
+};
+
+function keyAfter(key: Buffer): Buffer {
+ return Buffer.concat([key, Buffer.from([0])]);
+}
+
+function entryFromMeta(id: string, meta: JobMeta): QueueEntry {
+ return {
+ i: id,
+ o: meta.o,
+ g: meta.g,
+ p: meta.p,
+ f: meta.f,
+ c: meta.c,
+ to: meta.to,
+ };
+}
+
+function emptySweepLagStats(): SweepLagStats {
+ return {
+ dueCount: 0,
+ processedCount: 0,
+ oldestOverdueAgeMs: 0,
+ saturatedBucketCount: 0,
+ durationMs: 0,
+ };
+}
+
+function addDueKeysToLagStats(
+ stats: SweepLagStats,
+ ks: NuqFdbKeyspace,
+ due: [unknown, unknown][],
+ now: number,
+): void {
+ stats.dueCount += due.length;
+ if (due.length >= SWEEP_BATCH) stats.saturatedBucketCount++;
+ for (const [key] of due) {
+ const dueAt = Number(ks.unpackId(key as Buffer, 1));
+ if (Number.isFinite(dueAt)) {
+ stats.oldestOverdueAgeMs = Math.max(
+ stats.oldestOverdueAgeMs,
+ now - dueAt,
+ );
+ }
+ }
+}
+
+function logSweepLag(
+ logger: Logger,
+ queue: NuQFdbQueue,
+ index: "lease" | "backlog_timeout" | "delay",
+ stats: SweepLagStats,
+): void {
+ if (stats.dueCount === 0 && stats.saturatedBucketCount === 0) return;
+ logger[stats.saturatedBucketCount > 0 ? "warn" : "debug"](
+ "NuQ FDB sweeper lag",
+ {
+ canonicalLog: "nuq-fdb/sweeper_lag",
+ queueName: queue.queueName,
+ index,
+ timeBuckets: TIME_BUCKETS,
+ sweepBatch: SWEEP_BATCH,
+ ...stats,
+ },
+ );
+}
+
+// One sweeper services all queues against the same FDB cluster. Each queue
+// gets its own pass; a leased singleton lock (held on the first queue's
+// keyspace) keeps multiple candidate processes from sweeping concurrently.
+export class NuqFdbSweeper {
+ private readonly sweeperId = randomUUID();
+ private loop: NodeJS.Timeout | null = null;
+ private running = false;
+
+ constructor(
+ public readonly queues: NuQFdbQueue[],
+ public readonly externalSlots: NuqFdbExternalSlots[] = [],
+ ) {}
+
+ private get db() {
+ return getNuqFdbDatabase();
+ }
+
+ private get lockKs(): NuqFdbKeyspace {
+ return this.queues[0].ks;
+ }
+
+ public async tryAcquireLock(now: number = Date.now()): Promise<boolean> {
+ return await this.db.doTn(async tn => {
+ const rec = decodeJson<{ w: string; x: number }>(
+ await tn.get(this.lockKs.sweeperLock()),
+ );
+ if (rec && rec.x > now && rec.w !== this.sweeperId) return false;
+ tn.set(
+ this.lockKs.sweeperLock(),
+ encodeJson({ w: this.sweeperId, x: now + SWEEP_LOCK_TTL_MS }),
+ );
+ return true;
+ });
+ }
+
+ // Runs one full sweep over all queues. Exposed for tests; production uses
+ // start(), which wraps this in the singleton lock loop.
+ public async sweepOnce(logger: Logger = _logger): Promise<void> {
+ const now = Date.now();
+ for (const queue of this.queues) {
+ await this.sweepLeases(queue, now, logger);
+ await this.sweepBacklogTimeouts(queue, now, logger);
+ await this.sweepDelayed(queue, now, logger);
+ await this.sweepGroupFinishTasks(queue, now, logger);
+ await this.sweepGroupCancelTasks(queue, now, logger);
+ await this.sweepTeamRaiseTasks(queue, now, logger);
+ await this.sweepJobExpiry(queue, now, logger);
+ await this.sweepGroupExpiry(queue, now, logger);
+ }
+ for (const slots of this.externalSlots) {
+ await slots.sweepExpired(now, TIME_BUCKETS);
+ }
+ }
+
+ public start(intervalMs: number = 1000, logger: Logger = _logger): void {
+ if (this.loop) return;
+ this.loop = setInterval(async () => {
+ if (this.running) return;
+ this.running = true;
+ try {
+ if (await this.tryAcquireLock()) {
+ await this.sweepOnce(logger);
+ }
+ } catch (error) {
+ logger.warn("NuQ FDB sweeper tick failed", {
+ module: "nuq-fdb/sweeper",
+ error,
+ });
+ } finally {
+ this.running = false;
+ }
+ }, intervalMs);
+ }
+
+ public stop(): void {
+ if (this.loop) {
+ clearInterval(this.loop);
+ this.loop = null;
+ }
+ }
+
+ // === Lease expiry: requeue stalled jobs, fail them after MAX_STALLS
+
+ private async sweepLeases(
+ queue: NuQFdbQueue,
+ now: number,
+ logger: Logger,
+ ): Promise<void> {
+ const startedAt = Date.now();
+ const ks = queue.ks;
+ const stats = emptySweepLagStats();
+ for (let b = 0; b < TIME_BUCKETS; b++) {
+ const r = ks.leaseScanRange(b, now);
+ const due = await this.db.doTn(async tn =>
+ tn.snapshot().getRangeAll(r.begin, r.end, { limit: SWEEP_BATCH }),
+ );
+ addDueKeysToLagStats(stats, ks, due, now);
+ for (const [key, value] of due) {
+ const id = ks.unpackId(key as Buffer);
+ const lease = decodeJson<{ l: string }>(value as Buffer);
+ await this.db.doTn(async tn => {
+ const txc = newTxContext();
+ const st = decodeJson<JobStatusRecord>(
+ await tn.get(ks.jobStatus(id)),
+ );
+ // stale entries: job moved on (renewal, finish) or was reaped already
+ if (!st || st.s !== "active" || st.l !== lease?.l) {
+ tn.clear(key as Buffer);
+ return;
+ }
+ if (st.e !== undefined && st.e > now) {
+ // renewed after our snapshot; the old index entry is what expired
+ tn.clear(key as Buffer);
+ return;
+ }
+ const meta = decodeJson<JobMeta>(await tn.get(ks.jobMeta(id)));
+ tn.clear(key as Buffer);
+ if (!meta) return;
+ const entry = entryFromMeta(id, meta);
+
+ if (st.st < MAX_STALLS) {
+ // requeue directly to ready -- the job retains its slots
+ pushReady(tn, ks, entry, txc);
+ setStatusQueued(tn, ks, id, st.st + 1);
+ if (meta.g && meta.f & F_COUNTABLE) {
+ bumpGroupStatusCount(tn, ks, meta.g, "active", -1);
+ bumpGroupStatusCount(tn, ks, meta.g, "queued", 1);
+ }
+ } else {
+ tn.set(
+ ks.jobStatus(id),
+ encodeJson({
+ s: "failed",
+ st: st.st,
+ fa: now,
+ } satisfies JobStatusRecord),
+ );
+ tn.set(
+ ks.jobFailedReason(id),
+ Buffer.from(STALL_FAILED_REASON, "utf8"),
+ );
+ if (meta.g && meta.f & F_GACC && queue.groupOps) {
+ await queue.groupOps.terminalAccounting(
+ tn,
+ meta.g,
+ id,
+ "active",
+ "failed",
+ !!(meta.f & F_COUNTABLE),
+ now,
+ txc,
+ );
+ }
+ await releaseSlotsAndPromote(
+ tn,
+ ks,
+ entry,
+ { team: true, crawl: true },
+ now,
+ txc,
+ );
+ if (!meta.g) {
+ tn.set(
+ ks.jobExpiry(
+ timeBucket(id),
+ now + FAILED_STANDALONE_RETENTION_MS,
+ id,
+ ),
+ EMPTY,
+ );
+ }
+ }
+ });
+ stats.processedCount++;
+ }
+ }
+ stats.durationMs = Date.now() - startedAt;
+ logSweepLag(logger, queue, "lease", stats);
+ }
+
+ // === Backlog timeouts: silently drop pending jobs past their deadline
+
+ private async sweepBacklogTimeouts(
+ queue: NuQFdbQueue,
+ now: number,
+ logger: Logger,
+ ): Promise<void> {
+ const startedAt = Date.now();
+ const ks = queue.ks;
+ const stats = emptySweepLagStats();
+ for (let b = 0; b < TIME_BUCKETS; b++) {
+ const r = ks.backlogTimeoutScanRange(b, now);
+ const due = await this.db.doTn(async tn =>
+ tn.snapshot().getRangeAll(r.begin, r.end, { limit: SWEEP_BATCH }),
+ );
+ addDueKeysToLagStats(stats, ks, due, now);
+ for (const [key] of due) {
+ const id = ks.unpackId(key as Buffer);
+ await this.db.doTn(async tn => {
+ const txc = newTxContext();
+ tn.clear(key as Buffer);
+ const st = decodeJson<JobStatusRecord>(
+ await tn.get(ks.jobStatus(id)),
+ );
+ if (!st || st.s !== "pending" || !st.loc) return;
+ const meta = decodeJson<JobMeta>(await tn.get(ks.jobMeta(id)));
+ if (!meta) return;
+ clearPendingPlacement(tn, ks, id, meta.o, meta.g, st.loc, meta.to);
+ if (st.loc.k !== "gq" && meta.f & F_CRAWL_GATED) {
+ await releaseSlotsAndPromote(
+ tn,
+ ks,
+ entryFromMeta(id, meta),
+ { team: false, crawl: true },
+ now,
+ txc,
+ );
+ }
+ if (meta.g && meta.f & F_GACC && queue.groupOps) {
+ tn.clear(ks.groupJob(meta.g, id));
+ tn.add(ks.groupRemaining(meta.g), MINUS_ONE);
+ if (meta.f & F_COUNTABLE)
+ bumpGroupStatusCount(tn, ks, meta.g, "pending", -1);
+ tn.set(ks.taskGroupFinish(meta.g), EMPTY);
+ }
+ deleteJobRecords(tn, ks, id);
+ });
+ stats.processedCount++;
+ }
+ }
+ stats.durationMs = Date.now() - startedAt;
+ logSweepLag(logger, queue, "backlog_timeout", stats);
+ }
+
+ // === Delayed (crawl delay) promotions
+
+ private async sweepDelayed(
+ queue: NuQFdbQueue,
+ now: number,
+ logger: Logger,
+ ): Promise<void> {
+ const startedAt = Date.now();
+ const ks = queue.ks;
+ const stats = emptySweepLagStats();
+ for (let b = 0; b < TIME_BUCKETS; b++) {
+ const r = ks.delayedScanRange(b, now);
+ const due = await this.db.doTn(async tn =>
+ tn.snapshot().getRangeAll(r.begin, r.end, { limit: SWEEP_BATCH }),
+ );
+ addDueKeysToLagStats(stats, ks, due, now);
+ for (const [key, value] of due) {
+ const e = decodeJson<QueueEntry>(value as Buffer);
+ if (!e) continue;
+ await this.db.doTn(async tn => {
+ const txc = newTxContext();
+ const st = decodeJson<JobStatusRecord>(
+ await tn.get(ks.jobStatus(e.i)),
+ );
+ tn.clear(key as Buffer);
+ if (!st || st.s !== "pending" || st.loc?.k !== "dl") return;
+ // the job already holds its crawl slot; admit through the team gate
+ await admitThroughTeamGate(tn, ks, e, txc);
+ });
+ stats.processedCount++;
+ }
+ }
+ stats.durationMs = Date.now() - startedAt;
+ logSweepLag(logger, queue, "delay", stats);
+ }
+
+ // === Group finish detection (backstop for the inline path)
+
+ private async sweepGroupFinishTasks(
+ queue: NuQFdbQueue,
+ now: number,
+ logger: Logger,
+ ): Promise<void> {
+ if (!queue.groupOps) return;
+ const ks = queue.ks;
+ const r = ks.taskGroupFinishRange();
+ const tasks = await this.db.doTn(async tn =>
+ tn.snapshot().getRangeAll(r.begin, r.end, { limit: 200 }),
+ );
+ for (const [key] of tasks) {
+ const gid = ks.unpackId(key as Buffer);
+ await this.db.doTn(async tn => {
+ const txc = newTxContext();
+ // normal read so a concurrent finisher's decrement forces a retry --
+ // clearing the task may not race with the group draining to zero
+ const rem = decodeI64(await tn.get(ks.groupRemaining(gid)));
+ if (rem > 0) {
+ tn.clear(key as Buffer);
+ return;
+ }
+ await queue.groupOps!.tryCompleteGroup(tn, gid, now, txc);
+ });
+ }
+ }
+
+ // === Lazy group cancellation cleanup
+
+ private async sweepGroupCancelTasks(
+ queue: NuQFdbQueue,
+ now: number,
+ logger: Logger,
+ ): Promise<void> {
+ if (!queue.groupOps) return;
+ const ks = queue.ks;
+ const r = ks.taskGroupCancelRange();
+ const tasks = await this.db.doTn(async tn =>
+ tn.snapshot().getRangeAll(r.begin, r.end, { limit: 20 }),
+ );
+ for (const [key] of tasks) {
+ const gid = ks.unpackId(key as Buffer);
+ let exhausted = false;
+ let begin: Buffer | null = null;
+ // clean pending members in batches until none remain
+ for (let rounds = 0; rounds < 50 && !exhausted; rounds++) {
+ const result = await this.db.doTn(async tn => {
+ const jr = ks.groupJobRange(gid);
+ const rangeBegin = begin ?? jr.begin;
+ const members = await tn
+ .snapshot()
+ .getRangeAll(rangeBegin, jr.end, { limit: 500 });
+ let cleaned = 0;
+ for (const [mKey, mValue] of members) {
+ const gj = decodeJson<GroupJobIndexValue>(mValue as Buffer);
+ if (!gj || gj.s !== "pending") continue;
+ if (cleaned >= SWEEP_BATCH) {
+ return { exhausted: false, nextBegin: rangeBegin };
+ }
+ const id = ks.unpackId(mKey as Buffer);
+ const st = decodeJson<JobStatusRecord>(
+ await tn.get(ks.jobStatus(id)),
+ );
+ if (!st || st.s !== "pending" || !st.loc) {
+ // moved on; fix the index lazily
+ continue;
+ }
+ const meta = decodeJson<JobMeta>(await tn.get(ks.jobMeta(id)));
+ if (!meta) continue;
+ clearPendingPlacement(tn, ks, id, meta.o, meta.g, st.loc, meta.to);
+ // team-pending/delayed members hold a crawl slot; the group is
+ // cancelled so there is nothing to promote -- just release it
+ if (st.loc.k !== "gq" && meta.f & F_CRAWL_GATED) {
+ tn.add(ks.groupCrawlActive(gid), MINUS_ONE);
+ }
+ tn.clear(mKey as Buffer);
+ tn.add(ks.groupRemaining(gid), MINUS_ONE);
+ if (meta.f & F_COUNTABLE)
+ bumpGroupStatusCount(tn, ks, gid, "pending", -1);
+ deleteJobRecords(tn, ks, id);
+ cleaned++;
+ }
+ const lastKey = members[members.length - 1]?.[0] as
+ | Buffer
+ | undefined;
+ return {
+ exhausted: members.length < 500,
+ nextBegin: lastKey ? keyAfter(lastKey) : jr.end,
+ };
+ });
+ exhausted = result.exhausted;
+ begin = result.nextBegin;
+ }
+ if (exhausted) {
+ await this.db.doTn(async tn => {
+ tn.clear(key as Buffer);
+ tn.set(ks.taskGroupFinish(gid), EMPTY);
+ });
+ }
+ }
+ }
+
+ // === Limit raises: drain newly-available slots
+
+ private async sweepTeamRaiseTasks(
+ queue: NuQFdbQueue,
+ now: number,
+ logger: Logger,
+ ): Promise<void> {
+ const ks = queue.ks;
+ const r = ks.taskTeamRaiseRange();
+ const tasks = await this.db.doTn(async tn =>
+ tn.snapshot().getRangeAll(r.begin, r.end, { limit: 50 }),
+ );
+ for (const [key] of tasks) {
+ const tid = ks.unpackId(key as Buffer);
+ const done = await this.db.doTn(async tn => {
+ const txc = newTxContext();
+ const limitBuf = await tn.get(ks.teamLimit(tid));
+ const limit = limitBuf ? decodeI64(limitBuf) : Infinity;
+ const active = decodeI64(await tn.get(ks.teamActive(tid)));
+ let free = Math.min(Math.max(0, limit - active), 32);
+ let promoted = 0;
+ while (free > 0) {
+ const e = await popTeamPending(tn, ks, tid);
+ if (!e) break;
+ promoteEntryToReady(tn, ks, e, txc);
+ promoted++;
+ free--;
+ }
+ if (promoted > 0) bumpTeamActive(tn, ks, tid, promoted);
+ // done when no free slots remain or the pending queue is drained
+ return free > 0 || limit - active <= 0;
+ });
+ if (done) {
+ await this.db.doTn(async tn => tn.clear(key as Buffer));
+ }
+ }
+ }
+
+ // === Record GC
+
+ private async sweepJobExpiry(
+ queue: NuQFdbQueue,
+ now: number,
+ logger: Logger,
+ ): Promise<void> {
+ const ks = queue.ks;
+ for (let b = 0; b < TIME_BUCKETS; b++) {
+ const r = ks.jobExpiryScanRange(b, now);
+ const due = await this.db.doTn(async tn =>
+ tn.snapshot().getRangeAll(r.begin, r.end, { limit: SWEEP_BATCH * 2 }),
+ );
+ if (due.length === 0) continue;
+ await this.db.doTn(async tn => {
+ for (const [key] of due) {
+ const id = ks.unpackId(key as Buffer);
+ tn.clear(key as Buffer);
+ const st = decodeJson<JobStatusRecord>(
+ await tn.get(ks.jobStatus(id)),
+ );
+ if (
+ st &&
+ (st.s === "completed" || st.s === "failed" || st.s === "cancelled")
+ ) {
+ deleteJobRecords(tn, ks, id);
+ }
+ }
+ });
+ }
+ }
+
+ private async sweepGroupExpiry(
+ queue: NuQFdbQueue,
+ now: number,
+ logger: Logger,
+ ): Promise<void> {
+ if (!queue.groupOps) return;
+ const ks = queue.ks;
+ const r = ks.groupExpiryScanRange(now);
+ const due = await this.db.doTn(async tn =>
+ tn.snapshot().getRangeAll(r.begin, r.end, { limit: 20 }),
+ );
+ for (const [key] of due) {
+ const gid = ks.unpackId(key as Buffer);
+ // delete member job records in batches, then the group's own keyspace
+ let drained = false;
+ for (let rounds = 0; rounds < 200 && !drained; rounds++) {
+ drained = await this.db.doTn(async tn => {
+ const jr = ks.groupJobRange(gid);
+ const members = await tn
+ .snapshot()
+ .getRangeAll(jr.begin, jr.end, { limit: 200 });
+ for (const [mKey] of members) {
+ const id = ks.unpackId(mKey as Buffer);
+ deleteJobRecords(tn, ks, id);
+ tn.clear(mKey as Buffer);
+ }
+ return members.length < 200;
+ });
+ }
+ await this.db.doTn(async tn => {
+ const g = decodeJson<GroupMeta>(await tn.get(ks.groupMeta(gid)));
+ // the crawl-finished job for this group lives in the finished queue
+ const fjobBuf = await tn.get(ks.groupFinishedJob(gid));
+ if (fjobBuf && queue.groupOps!.finishedKs) {
+ const fid = fjobBuf.toString("utf8");
+ deleteJobRecords(tn, queue.groupOps!.finishedKs as any, fid);
+ }
+ const gr = ks.groupRange(gid);
+ tn.clearRange(gr.begin, gr.end);
+ if (g) tn.clear(ks.ongoingGroup(g.o, gid));
+ tn.clear(ks.taskGroupFinish(gid));
+ tn.clear(ks.taskGroupCancel(gid));
+ tn.clear(key as Buffer);
+ });
+ }
+ }
+}
diff --git a/apps/api/src/services/worker/nuq-router.ts b/apps/api/src/services/worker/nuq-router.ts
new file mode 100644
index 000000000..92d8716a7
--- /dev/null
+++ b/apps/api/src/services/worker/nuq-router.ts
@@ -0,0 +1,645 @@
+import { Logger } from "winston";
+import { logger as _logger } from "../../lib/logger";
+import { config } from "../../config";
+import { RateLimiterMode, ScrapeJobData } from "../../types";
+import { getACUCTeam } from "../../controllers/auth";
+import { redisEvictConnection } from "../../services/redis";
+import { isSelfHosted } from "../../lib/deployment";
+import {
+ getTeamQueueLimit,
+ getConcurrencyLimitActiveJobsCount,
+ pushConcurrencyLimitActiveJob,
+ removeConcurrencyLimitActiveJob,
+} from "../../lib/concurrency-redis";
+import {
+ NuQJob,
+ NuQJobStatus,
+ NuQGroupStatus,
+ NuQJobGroupInstance,
+ scrapeQueue as scrapeQueuePg,
+ crawlFinishedQueue as crawlFinishedQueuePg,
+ crawlGroup as crawlGroupPg,
+} from "./nuq";
+import {
+ scrapeQueueFdb,
+ crawlFinishedQueueFdb,
+ crawlGroupFdb,
+ externalSlotsFdb,
+ isFdbConfigured,
+ NuQFdbQueue,
+ NuQFdbJob,
+} from "./nuq-fdb";
+
+// Dual-backend router for the NuQ migration to FoundationDB. Exports the same
+// `scrapeQueue` / `crawlFinishedQueue` / `crawlGroup` names as ./nuq so call
+// sites only swap their import path. Routing rules:
+// - new crawls: team flag (TeamFlags.nuqFdb) or NUQ_BACKEND=fdb decides; the
+// choice is pinned in StoredCrawl.queueBackend so a crawl never spans
+// backends
+// - reads: probe FDB first (cheap point reads), fall back to PG
+// - workers: dual-poll FDB first; in-flight jobs are tagged with their
+// backend so renew/finish/fail route correctly
+
+export type QueueBackend = "pg" | "fdb";
+
+export type { NuQJob, NuQJobStatus, NuQGroupStatus, NuQJobGroupInstance };
+
+export function fdbQueueEnabled(): boolean {
+ return isFdbConfigured();
+}
+
+function fdbForced(): boolean {
+ return config.NUQ_BACKEND === "fdb";
+}
+
+const fdbFallbackLastWarn = new Map<string, number>();
+
+function logFdbFallback(
+ logger: Logger,
+ operation: string,
+ error: unknown,
+): void {
+ const now = Date.now();
+ const lastWarn = fdbFallbackLastWarn.get(operation) ?? 0;
+ if (now - lastWarn < 60_000) return;
+ fdbFallbackLastWarn.set(operation, now);
+ logger.warn("FDB queue operation failed, falling back to PG", {
+ module: "nuq-router",
+ operation,
+ error,
+ });
+}
+
+// Whether NEW work for this team should go to FDB. Existing crawls follow
+// their StoredCrawl.queueBackend marker instead.
+export async function isFdbTeam(teamId: string | undefined): Promise<boolean> {
+ if (!fdbQueueEnabled()) return false;
+ if (fdbForced()) return true;
+ if (!teamId) return false;
+ try {
+ const acuc = await getACUCTeam(teamId, false, true, RateLimiterMode.Crawl);
+ return acuc?.flags?.nuqFdb === true;
+ } catch (error) {
+ _logger.warn("Failed to resolve nuqFdb team flag, defaulting to pg", {
+ module: "nuq-router",
+ teamId,
+ error,
+ });
+ return false;
+ }
+}
+
+export async function resolveNewGroupBackend(
+ teamId: string,
+): Promise<QueueBackend> {
+ return (await isFdbTeam(teamId)) ? "fdb" : "pg";
+}
+
+// Reads only the queueBackend marker off the stored crawl. Deliberately not
+// getCrawl() -- importing crawl-redis pulls the whole scraper tree in.
+async function getCrawlQueueBackend(
+ crawlId: string,
+): Promise<QueueBackend | null> {
+ const raw = await redisEvictConnection.get("crawl:" + crawlId);
+ if (!raw) return null;
+ try {
+ const sc = JSON.parse(raw);
+ return sc?.queueBackend === "fdb" ? "fdb" : sc ? "pg" : null;
+ } catch {
+ return null;
+ }
+}
+
+// Which backend a job belongs to at enqueue time. Crawl jobs follow their
+// crawl's pinned backend; standalone jobs follow the team flag.
+export async function resolveJobBackend(
+ data: ScrapeJobData,
+): Promise<QueueBackend> {
+ if (!fdbQueueEnabled()) return "pg";
+ if (fdbForced()) return "fdb";
+ if (data.crawl_id) {
+ return (await getCrawlQueueBackend(data.crawl_id)) ?? "pg";
+ }
+ return (await isFdbTeam(data.team_id)) ? "fdb" : "pg";
+}
+
+function tagFdbJob<T extends object>(job: T): T & { backend: "fdb" } {
+ (job as any).backend = "fdb";
+ return job as T & { backend: "fdb" };
+}
+
+// === External capacity holders (browser sessions, sync scrapes)
+//
+// Non-queue work that occupies team capacity mirrors itself into whichever
+// ledger the team runs on. Mismatched acquire/release pairs (flag flipped
+// mid-hold) self-heal: Redis entries expire by score, FDB external slots are
+// reaped by the sweeper.
+
+export async function mirrorExternalSlotAcquire(
+ teamId: string,
+ holderId: string,
+ ttlMs: number,
+): Promise<void> {
+ if (await isFdbTeam(teamId)) {
+ await externalSlotsFdb.acquire(teamId, holderId, ttlMs);
+ } else {
+ await pushConcurrencyLimitActiveJob(teamId, holderId, ttlMs);
+ }
+}
+
+export async function mirrorExternalSlotRelease(
+ teamId: string,
+ holderId: string,
+): Promise<void> {
+ if (await isFdbTeam(teamId)) {
+ await externalSlotsFdb.release(teamId, holderId);
+ } else {
+ await removeConcurrencyLimitActiveJob(teamId, holderId);
+ }
+}
+
+// Active count across both ledgers; a migrating team has load on both while
+// its old PG crawls drain.
+export async function getCombinedTeamActiveCount(
+ teamId: string,
+): Promise<number> {
+ const redisCount = await getConcurrencyLimitActiveJobsCount(teamId);
+ if (!fdbQueueEnabled()) return redisCount;
+ try {
+ return redisCount + (await scrapeQueueFdb.getTeamActiveCount(teamId));
+ } catch (error) {
+ if (fdbForced()) throw error;
+ logFdbFallback(_logger, "getCombinedTeamActiveCount", error);
+ return redisCount;
+ }
+}
+
+// === FDB enqueue (the whole gating block of queue-jobs collapses into this)
+
+export function backlogTimeoutMsForGate(timeoutMs: number): Date {
+ return new Date(Date.now() + timeoutMs);
+}
+
+export async function fdbEnqueueScrapeJobs(
+ jobs: {
+ jobId: string;
+ data: ScrapeJobData;
+ priority: number;
+ listenable?: boolean;
+ backlogTimeoutMs: number;
+ }[],
+ teamId: string,
+ options?: { bypassGate?: boolean },
+): Promise<{
+ jobs: (NuQJob<ScrapeJobData> & { backend: "fdb" })[];
+ backloggedCount: number;
+ teamLimit: number | null;
+}> {
+ let teamLimit: number | null = null;
+ if (!isSelfHosted() && !fdbForced()) {
+ const acuc = await getACUCTeam(teamId, false, true, RateLimiterMode.Crawl);
+ teamLimit = acuc?.concurrency ?? 2;
+ } else if (!isSelfHosted()) {
+ const acuc = await getACUCTeam(teamId, false, true, RateLimiterMode.Crawl);
+ teamLimit = acuc?.concurrency ?? null;
+ }
+
+ const queueCap =
+ teamLimit === null ? Number.MAX_SAFE_INTEGER : getTeamQueueLimit(teamLimit);
+
+ const results = await scrapeQueueFdb.addJobs(
+ jobs.map(j => ({
+ id: j.jobId,
+ data: j.data,
+ options: {
+ priority: j.priority,
+ listenable: j.listenable ?? false,
+ ownerId: j.data.team_id ?? undefined,
+ groupId: j.data.crawl_id ?? undefined,
+ bypassGate:
+ options?.bypassGate ||
+ j.data.mode === "kickoff" ||
+ j.data.mode === "kickoff_sitemap",
+ timesOutAt: new Date(Date.now() + j.backlogTimeoutMs),
+ },
+ })),
+ { teamLimit, queueCap },
+ );
+
+ const tagged = results.map(r => tagFdbJob(r as NuQJob<ScrapeJobData>));
+ return {
+ jobs: tagged,
+ backloggedCount: tagged.filter(j => j.status === "backlog").length,
+ teamLimit,
+ };
+}
+
+// === Routed scrape queue
+
+class RoutedScrapeQueue {
+ // in-flight jobs taken by THIS process, so renew/finish/fail can route
+ private inflightBackend = new Map<string, QueueBackend>();
+
+ private backendFor(id: string): QueueBackend {
+ return this.inflightBackend.get(id) ?? "pg";
+ }
+
+ public async getJobToProcess(
+ logger: Logger = _logger,
+ ): Promise<NuQJob<ScrapeJobData> | null> {
+ if (fdbQueueEnabled()) {
+ try {
+ const job = await scrapeQueueFdb.getJobToProcess(logger);
+ if (job) {
+ this.inflightBackend.set(job.id, "fdb");
+ return tagFdbJob(job as NuQJob<ScrapeJobData>);
+ }
+ } catch (error) {
+ if (fdbForced()) throw error;
+ logFdbFallback(logger, "scrape.getJobToProcess", error);
+ }
+ }
+ const job = await scrapeQueuePg.getJobToProcess();
+ if (job) this.inflightBackend.set(job.id, "pg");
+ return job;
+ }
+
+ public async renewLock(
+ id: string,
+ lock: string,
+ logger: Logger = _logger,
+ ): Promise<boolean> {
+ if (this.backendFor(id) === "fdb") {
+ return scrapeQueueFdb.renewLock(id, lock, logger);
+ }
+ return scrapeQueuePg.renewLock(id, lock, logger);
+ }
+
+ public async jobFinish(
+ id: string,
+ lock: string,
+ returnvalue: any | null,
+ logger: Logger = _logger,
+ ): Promise<boolean> {
+ const backend = this.backendFor(id);
+ this.inflightBackend.delete(id);
+ if (backend === "fdb") {
+ return scrapeQueueFdb.jobFinish(id, lock, returnvalue, logger);
+ }
+ return scrapeQueuePg.jobFinish(id, lock, returnvalue, logger);
+ }
+
+ public async jobFail(
+ id: string,
+ lock: string,
+ failedReason: string,
+ logger: Logger = _logger,
+ ): Promise<boolean> {
+ const backend = this.backendFor(id);
+ this.inflightBackend.delete(id);
+ if (backend === "fdb") {
+ return scrapeQueueFdb.jobFail(id, lock, failedReason, logger);
+ }
+ return scrapeQueuePg.jobFail(id, lock, failedReason, logger);
+ }
+
+ public async getJob(
+ id: string,
+ logger: Logger = _logger,
+ ): Promise<NuQJob<ScrapeJobData> | null> {
+ if (fdbQueueEnabled()) {
+ try {
+ const job = await scrapeQueueFdb.getJob(id, logger);
+ if (job) return tagFdbJob(job as NuQJob<ScrapeJobData>);
+ if (fdbForced()) return null;
+ } catch (error) {
+ if (fdbForced()) throw error;
+ logFdbFallback(logger, "scrape.getJob", error);
+ }
+ }
+ return scrapeQueuePg.getJob(id, logger);
+ }
+
+ public async getJobs(
+ ids: string[],
+ logger: Logger = _logger,
+ ): Promise<NuQJob<ScrapeJobData>[]> {
+ if (!fdbQueueEnabled()) return scrapeQueuePg.getJobs(ids, logger);
+ let fdbJobs: NuQFdbJob<ScrapeJobData>[] = [];
+ try {
+ fdbJobs = await scrapeQueueFdb.getJobs(ids, logger);
+ } catch (error) {
+ if (fdbForced()) throw error;
+ logFdbFallback(logger, "scrape.getJobs", error);
+ return scrapeQueuePg.getJobs(ids, logger);
+ }
+ if (fdbForced())
+ return fdbJobs.map(j => tagFdbJob(j as NuQJob<ScrapeJobData>));
+ const found = new Set(fdbJobs.map(j => j.id));
+ const missing = ids.filter(id => !found.has(id));
+ const pgJobs =
+ missing.length > 0 ? await scrapeQueuePg.getJobs(missing, logger) : [];
+ const byId = new Map<string, NuQJob<ScrapeJobData>>();
+ for (const j of fdbJobs)
+ byId.set(j.id, tagFdbJob(j as NuQJob<ScrapeJobData>));
+ for (const j of pgJobs) byId.set(j.id, j);
+ return ids
+ .map(id => byId.get(id))
+ .filter((j): j is NuQJob<ScrapeJobData> => j !== undefined);
+ }
+
+ public async getJobsWithStatus(
+ ids: string[],
+ status: NuQJobStatus,
+ logger: Logger = _logger,
+ ): Promise<NuQJob<ScrapeJobData>[]> {
+ return (await this.getJobs(ids, logger)).filter(j => j.status === status);
+ }
+
+ public async getJobsWithStatuses(
+ ids: string[],
+ statuses: NuQJobStatus[],
+ logger: Logger = _logger,
+ ): Promise<NuQJob<ScrapeJobData>[]> {
+ const set = new Set(statuses);
+ return (await this.getJobs(ids, logger)).filter(j => set.has(j.status));
+ }
+
+ private async isFdbGroup(groupId: string): Promise<boolean> {
+ if (!fdbQueueEnabled()) return false;
+ try {
+ return (await crawlGroupFdb.getGroup(groupId)) !== null;
+ } catch (error) {
+ if (fdbForced()) throw error;
+ logFdbFallback(_logger, "scrape.isFdbGroup", error);
+ return false;
+ }
+ }
+
+ public async getGroupAnyJob(
+ groupId: string,
+ ownerId: string,
+ logger: Logger = _logger,
+ ): Promise<NuQJob<ScrapeJobData> | null> {
+ if (await this.isFdbGroup(groupId)) {
+ const job = await scrapeQueueFdb.getGroupAnyJob(groupId, ownerId, logger);
+ return job ? tagFdbJob(job as NuQJob<ScrapeJobData>) : null;
+ }
+ return scrapeQueuePg.getGroupAnyJob(groupId, ownerId);
+ }
+
+ public async getGroupNumericStats(
+ groupId: string,
+ logger: Logger = _logger,
+ ): Promise<Record<NuQJobStatus, number>> {
+ if (await this.isFdbGroup(groupId)) {
+ return scrapeQueueFdb.getGroupNumericStats(groupId, logger) as Promise<
+ Record<NuQJobStatus, number>
+ >;
+ }
+ return scrapeQueuePg.getGroupNumericStats(groupId, logger);
+ }
+
+ public async getCrawlJobsForListing(
+ groupId: string,
+ limit: number,
+ offset: number,
+ logger: Logger = _logger,
+ ): Promise<NuQJob<ScrapeJobData>[]> {
+ if (await this.isFdbGroup(groupId)) {
+ const jobs = await scrapeQueueFdb.getCrawlJobsForListing(
+ groupId,
+ limit,
+ offset,
+ logger,
+ );
+ return jobs.map(j => tagFdbJob(j as NuQJob<ScrapeJobData>));
+ }
+ return scrapeQueuePg.getCrawlJobsForListing(groupId, limit, offset, logger);
+ }
+
+ public async removeJob(id: string, logger: Logger = _logger): Promise<void> {
+ if (fdbQueueEnabled()) {
+ try {
+ if (await scrapeQueueFdb.hasJob(id)) {
+ await scrapeQueueFdb.removeJob(id, logger);
+ return;
+ }
+ } catch (error) {
+ if (fdbForced()) throw error;
+ logFdbFallback(logger, "scrape.removeJob", error);
+ }
+ }
+ if (fdbForced()) return;
+ await scrapeQueuePg.removeJob(id, logger);
+ }
+
+ public async removeJobs(
+ ids: string[],
+ logger: Logger = _logger,
+ ): Promise<void> {
+ for (const id of ids) {
+ await this.removeJob(id, logger);
+ }
+ }
+
+ public async waitForJob<T = any>(
+ id: string,
+ timeout: number | null,
+ logger: Logger = _logger,
+ ): Promise<T> {
+ if (fdbQueueEnabled()) {
+ try {
+ if (await scrapeQueueFdb.hasJob(id)) {
+ return scrapeQueueFdb.waitForJob(id, timeout, logger);
+ }
+ } catch (error) {
+ if (fdbForced()) throw error;
+ logFdbFallback(logger, "scrape.waitForJob", error);
+ }
+ }
+ if (fdbForced()) throw new Error("Job not found");
+ return scrapeQueuePg.waitForJob(id, timeout, logger) as Promise<T>;
+ }
+
+ public async getMetrics(logger: Logger = _logger) {
+ return scrapeQueuePg.getMetrics();
+ }
+}
+
+// === Routed crawl-finished queue (worker consumer + reads)
+
+class RoutedCrawlFinishedQueue {
+ private inflightBackend = new Map<string, QueueBackend>();
+
+ public async getJobToProcess(
+ logger: Logger = _logger,
+ ): Promise<NuQJob<any> | null> {
+ if (fdbQueueEnabled()) {
+ try {
+ const job = await crawlFinishedQueueFdb.getJobToProcess(logger);
+ if (job) {
+ this.inflightBackend.set(job.id, "fdb");
+ return tagFdbJob(job as NuQJob<any>);
+ }
+ } catch (error) {
+ if (fdbForced()) throw error;
+ logFdbFallback(logger, "crawlFinished.getJobToProcess", error);
+ }
+ }
+ const job = await crawlFinishedQueuePg.getJobToProcess();
+ if (job) this.inflightBackend.set(job.id, "pg");
+ return job;
+ }
+
+ public async renewLock(
+ id: string,
+ lock: string,
+ logger: Logger = _logger,
+ ): Promise<boolean> {
+ if (this.inflightBackend.get(id) === "fdb") {
+ return crawlFinishedQueueFdb.renewLock(id, lock, logger);
+ }
+ return crawlFinishedQueuePg.renewLock(id, lock, logger);
+ }
+
+ public async jobFinish(
+ id: string,
+ lock: string,
+ returnvalue: any | null,
+ logger: Logger = _logger,
+ ): Promise<boolean> {
+ const backend = this.inflightBackend.get(id) ?? "pg";
+ this.inflightBackend.delete(id);
+ if (backend === "fdb") {
+ return crawlFinishedQueueFdb.jobFinish(id, lock, returnvalue, logger);
+ }
+ return crawlFinishedQueuePg.jobFinish(id, lock, returnvalue, logger);
+ }
+
+ public async jobFail(
+ id: string,
+ lock: string,
+ failedReason: string,
+ logger: Logger = _logger,
+ ): Promise<boolean> {
+ const backend = this.inflightBackend.get(id) ?? "pg";
+ this.inflightBackend.delete(id);
+ if (backend === "fdb") {
+ return crawlFinishedQueueFdb.jobFail(id, lock, failedReason, logger);
+ }
+ return crawlFinishedQueuePg.jobFail(id, lock, failedReason, logger);
+ }
+
+ public async getJob(
+ id: string,
+ logger: Logger = _logger,
+ ): Promise<NuQJob<any> | null> {
+ if (fdbQueueEnabled()) {
+ try {
+ const job = await crawlFinishedQueueFdb.getJob(id, logger);
+ if (job) return tagFdbJob(job as NuQJob<any>);
+ if (fdbForced()) return null;
+ } catch (error) {
+ if (fdbForced()) throw error;
+ logFdbFallback(logger, "crawlFinished.getJob", error);
+ }
+ }
+ return crawlFinishedQueuePg.getJob(id, logger);
+ }
+}
+
+// === Routed crawl group
+
+class RoutedCrawlGroup {
+ public async addGroup(
+ id: string,
+ ownerId: string,
+ ttl?: number,
+ opts?: {
+ backend?: QueueBackend;
+ maxConcurrency?: number;
+ delaySeconds?: number;
+ },
+ logger: Logger = _logger,
+ ): Promise<NuQJobGroupInstance> {
+ if (opts?.backend === "fdb") {
+ const g = await crawlGroupFdb.addGroup(
+ id,
+ ownerId,
+ ttl,
+ {
+ maxConcurrency: opts.maxConcurrency,
+ delaySeconds: opts.delaySeconds,
+ },
+ logger,
+ );
+ return g as NuQJobGroupInstance;
+ }
+ return crawlGroupPg.addGroup(id, ownerId, ttl, logger);
+ }
+
+ public async getGroup(
+ id: string,
+ logger: Logger = _logger,
+ ): Promise<NuQJobGroupInstance | null> {
+ if (fdbQueueEnabled()) {
+ try {
+ const g = await crawlGroupFdb.getGroup(id, logger);
+ if (g) return g as NuQJobGroupInstance;
+ if (fdbForced()) return null;
+ } catch (error) {
+ if (fdbForced()) throw error;
+ logFdbFallback(logger, "crawlGroup.getGroup", error);
+ }
+ }
+ return crawlGroupPg.getGroup(id, logger);
+ }
+
+ public async getOngoingByOwner(
+ ownerId: string,
+ logger: Logger = _logger,
+ ): Promise<NuQJobGroupInstance[]> {
+ if (!fdbQueueEnabled()) {
+ return crawlGroupPg.getOngoingByOwner(ownerId, logger);
+ }
+ let fdb: NuQJobGroupInstance[] = [];
+ try {
+ fdb = (await crawlGroupFdb.getOngoingByOwner(
+ ownerId,
+ logger,
+ )) as NuQJobGroupInstance[];
+ } catch (error) {
+ if (fdbForced()) throw error;
+ logFdbFallback(logger, "crawlGroup.getOngoingByOwner", error);
+ return crawlGroupPg.getOngoingByOwner(ownerId, logger);
+ }
+ if (fdbForced()) return fdb as NuQJobGroupInstance[];
+ const pg = await crawlGroupPg.getOngoingByOwner(ownerId, logger);
+ const seen = new Set(fdb.map(g => g.id));
+ return [
+ ...(fdb as NuQJobGroupInstance[]),
+ ...pg.filter(g => !seen.has(g.id)),
+ ];
+ }
+
+ // O(1) cancel; only meaningful for FDB groups. PG crawls keep their
+ // existing Redis-based cancellation path.
+ public async cancelGroup(
+ id: string,
+ logger: Logger = _logger,
+ ): Promise<boolean> {
+ if (!fdbQueueEnabled()) return false;
+ try {
+ return crawlGroupFdb.cancelGroup(id, logger);
+ } catch (error) {
+ if (fdbForced()) throw error;
+ logFdbFallback(logger, "crawlGroup.cancelGroup", error);
+ return false;
+ }
+ }
+}
+
+export const scrapeQueue = new RoutedScrapeQueue();
+export const crawlFinishedQueue = new RoutedCrawlFinishedQueue();
+export const crawlGroup = new RoutedCrawlGroup();
diff --git a/apps/api/src/services/worker/nuq-worker.ts b/apps/api/src/services/worker/nuq-worker.ts
index 3f41a0881..aa5982eae 100644
--- a/apps/api/src/services/worker/nuq-worker.ts
+++ b/apps/api/src/services/worker/nuq-worker.ts
@@ -4,7 +4,13 @@ import "../sentry";
import { setSentryServiceTag } from "../sentry";
import { logger as _logger } from "../../lib/logger";
import { processJobInternal } from "./scrape-worker";
-import { scrapeQueue, nuqGetLocalMetrics, nuqHealthCheck } from "./nuq";
+import {
+ nuqGetLocalMetrics,
+ nuqHealthCheck,
+ scrapeQueue as scrapeQueuePg,
+} from "./nuq";
+import { scrapeQueue, fdbQueueEnabled } from "./nuq-router";
+import { getNuqFdbSweeper } from "./nuq-fdb";
import { jobDurationSeconds } from "../../lib/job-metrics";
import { register } from "prom-client";
import Express from "express";
@@ -57,6 +63,20 @@ import { initializeEngineForcing } from "../../scraper/WebScraper/utils/engine-f
let noJobTimeout = 1500;
+ // the FDB backend has no pg_cron: one worker at a time holds the sweeper
+ // lease and runs lease/timeout/group sweeps for everyone
+ if (fdbQueueEnabled()) {
+ try {
+ getNuqFdbSweeper().start();
+ } catch (error) {
+ if (config.NUQ_BACKEND === "fdb") throw error;
+ _logger.warn("Failed to start FDB sweeper, continuing with PG", {
+ module: "nuq-worker",
+ error,
+ });
+ }
+ }
+
while (!isShuttingDown) {
const job = await scrapeQueue.getJobToProcess();
@@ -137,7 +157,17 @@ import { initializeEngineForcing } from "../../scraper/WebScraper/utils/engine-f
_logger.info("NuQ worker shutting down");
server.close(async () => {
- await scrapeQueue.shutdown();
+ if (fdbQueueEnabled()) {
+ try {
+ getNuqFdbSweeper().stop();
+ } catch (error) {
+ _logger.warn("Failed to stop FDB sweeper", {
+ module: "nuq-worker",
+ error,
+ });
+ }
+ }
+ await scrapeQueuePg.shutdown();
_logger.info("NuQ worker shut down");
process.exit(0);
});
diff --git a/apps/api/src/services/worker/nuq.ts b/apps/api/src/services/worker/nuq.ts
index f891103af..627196e20 100644
--- a/apps/api/src/services/worker/nuq.ts
+++ b/apps/api/src/services/worker/nuq.ts
@@ -4,7 +4,7 @@ import { Client, Pool } from "pg";
import { type ScrapeJobData } from "../../types";
import { withSpan, setSpanAttributes } from "../../lib/otel-tracer";
import amqp from "amqplib";
-import { v5 as uuidv5, validate as isUUID } from "uuid";
+import { normalizeOwnerId } from "../../lib/owner-id";
import { config } from "../../config";
import { nuqRedis } from "./redis";
@@ -53,16 +53,6 @@ type NuQOptions = {
backlog?: boolean;
};
-// owner IDs can sometimes be non-UUID, so let's normalize it to avoid query breakage - mogery
-const normalizedUUIDNamespace = "0f38e00e-d7ee-4b77-8a7a-a787a3537ca2";
-export function normalizeOwnerId(
- ownerId: string | undefined | null,
-): string | null {
- if (typeof ownerId !== "string") return null;
- if (isUUID(ownerId)) return ownerId;
- return uuidv5(ownerId, normalizedUUIDNamespace);
-}
-
function isExpectedAmqpCloseError(error: unknown): boolean {
const message =
error instanceof Error
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index fa084dc01..9c1bc0cad 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -520,6 +520,9 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
[doc.metadata.url ?? doc.metadata.sourceURL!],
1,
sc.crawlerOptions?.maxDepth ?? 10,
+ false,
+ false,
+ true,
);
if (filterResult.links.length === 0) {
const url = doc.metadata.url ?? doc.metadata.sourceURL!;
@@ -1307,10 +1310,14 @@ export const processJobInternal = async (job: NuQJob<ScrapeJobData>) => {
};
async function processJobWithTracing(job: NuQJob<ScrapeJobData>, logger: any) {
+ // FDB-backed jobs hold their concurrency slot through the queue lease; the
+ // Redis slot mirror and promotion-on-done below are PG-backend machinery
+ const isFdbJob = (job as any).backend === "fdb";
try {
try {
let extendLockInterval: NodeJS.Timeout | null = null;
if (
+ !isFdbJob &&
job.data?.mode !== "kickoff" &&
job.data?.team_id &&
!job.data.skipNuq
@@ -1380,7 +1387,7 @@ async function processJobWithTracing(job: NuQJob<ScrapeJobData>, logger: any) {
}
}
} finally {
- if (!job.data.skipNuq) {
+ if (!job.data.skipNuq && !isFdbJob) {
await concurrentJobDone(job);
}
}
diff --git a/apps/api/src/services/worker/team-semaphore.ts b/apps/api/src/services/worker/team-semaphore.ts
index 10eaf3b4b..7b3c9954f 100644
--- a/apps/api/src/services/worker/team-semaphore.ts
+++ b/apps/api/src/services/worker/team-semaphore.ts
@@ -2,6 +2,8 @@ import {
pushConcurrencyLimitActiveJob,
removeConcurrencyLimitActiveJob,
} from "../../lib/concurrency-limit";
+import { isFdbTeam } from "./nuq-router";
+import { externalSlotsFdb } from "./nuq-fdb";
import { isSelfHosted } from "../../lib/deployment";
import { ScrapeJobTimeoutError, TransportableError } from "../../lib/error";
import { logger as _logger } from "../../lib/logger";
@@ -30,6 +32,8 @@ const semaphoreHoldDuration = new Histogram({
const { scripts, runScript, ensure } = nuqRedis;
const SEMAPHORE_TTL = 30 * 1000;
+type MirrorBackend = "pg" | "fdb";
+type MirrorState = { backend?: MirrorBackend; touched: Set<MirrorBackend> };
async function acquire(
teamId: string,
@@ -137,29 +141,50 @@ async function count(teamId: string): Promise<number> {
return count;
}
-function startHeartbeat(teamId: string, holderId: string, intervalMs: number) {
+function startHeartbeat(
+ teamId: string,
+ holderId: string,
+ intervalMs: number,
+ mirrorState: MirrorState,
+) {
let stopped = false;
+ let wake: (() => void) | null = null;
+
+ const sleep = (ms: number) =>
+ new Promise<void>(resolve => {
+ const timer = setTimeout(() => {
+ wake = null;
+ resolve();
+ }, ms);
+ wake = () => {
+ clearTimeout(timer);
+ wake = null;
+ resolve();
+ };
+ });
const promise = (async () => {
try {
while (!stopped) {
- await pushConcurrencyLimitActiveJob(teamId, holderId, 60 * 1000).catch(
- () => {
- _logger.warn("Failed to update concurrency limit active job", {
- teamId,
- jobId: holderId,
- });
- },
- );
+ await mirrorSlotAcquire(teamId, holderId, mirrorState).catch(() => {
+ _logger.warn("Failed to update concurrency limit active job", {
+ teamId,
+ jobId: holderId,
+ });
+ });
+ if (stopped) break;
const ok = await heartbeat(teamId, holderId);
if (!ok) {
throw new TransportableError("SCRAPE_TIMEOUT", "heartbeat_failed");
}
- await new Promise(r => setTimeout(r, intervalMs));
+ if (stopped) break;
+ await sleep(intervalMs);
}
} catch (error) {
- _logger.error("Error in semaphore heartbeat loop", { error });
+ if (!stopped) {
+ _logger.error("Error in semaphore heartbeat loop", { error });
+ }
}
return Promise.reject(
@@ -169,12 +194,72 @@ function startHeartbeat(teamId: string, holderId: string, intervalMs: number) {
return {
promise,
- stop() {
+ async stop() {
stopped = true;
+ wake?.();
+ await promise.catch(() => {});
},
};
}
+// Sync scrapes occupy queue capacity so async jobs see the team's real load.
+// PG-backed teams mirror into the Redis ZSET; FDB-backed teams consume an
+// external slot on the FDB ledger.
+async function resolveMirrorBackend(teamId: string): Promise<MirrorBackend> {
+ return (await isFdbTeam(teamId)) ? "fdb" : "pg";
+}
+
+async function releaseMirrorBackend(
+ teamId: string,
+ holderId: string,
+ backend: MirrorBackend,
+): Promise<void> {
+ if (backend === "fdb") {
+ await externalSlotsFdb.release(teamId, holderId);
+ } else {
+ await removeConcurrencyLimitActiveJob(teamId, holderId);
+ }
+}
+
+async function mirrorSlotAcquire(
+ teamId: string,
+ holderId: string,
+ state: MirrorState,
+): Promise<void> {
+ const backend = await resolveMirrorBackend(teamId);
+ if (backend === "fdb") {
+ await externalSlotsFdb.acquire(teamId, holderId, 60 * 1000);
+ } else {
+ await pushConcurrencyLimitActiveJob(teamId, holderId, 60 * 1000);
+ }
+ const previous = state.backend;
+ state.backend = backend;
+ state.touched.add(backend);
+ if (previous && previous !== backend) {
+ await releaseMirrorBackend(teamId, holderId, previous);
+ state.touched.delete(previous);
+ }
+}
+
+async function mirrorSlotRelease(
+ teamId: string,
+ holderId: string,
+ state: MirrorState,
+): Promise<void> {
+ const backends = Array.from(state.touched);
+ const results = await Promise.allSettled(
+ backends.map(backend => releaseMirrorBackend(teamId, holderId, backend)),
+ );
+ const failed = results.find(
+ (result): result is PromiseRejectedResult => result.status === "rejected",
+ );
+ if (failed) {
+ throw failed.reason;
+ }
+ state.backend = undefined;
+ state.touched.clear();
+}
+
async function withSemaphore<T>(
teamId: string,
holderId: string,
@@ -200,16 +285,20 @@ async function withSemaphore<T>(
});
const endTimer = semaphoreHoldDuration.startTimer();
- const hb = startHeartbeat(teamId, holderId, SEMAPHORE_TTL / 2);
+ const mirrorState: MirrorState = { touched: new Set() };
+ let hb: ReturnType<typeof startHeartbeat> | null = null;
activeSemaphores.inc();
try {
- await pushConcurrencyLimitActiveJob(teamId, holderId, 60 * 1000);
+ await mirrorSlotAcquire(teamId, holderId, mirrorState);
+ hb = startHeartbeat(teamId, holderId, SEMAPHORE_TTL / 2, mirrorState);
const result = await Promise.race([func(limited), hb.promise]);
return result;
} finally {
- await removeConcurrencyLimitActiveJob(teamId, holderId).catch(() => {
+ await hb?.stop();
+
+ await mirrorSlotRelease(teamId, holderId, mirrorState).catch(() => {
_logger.warn("Failed to remove concurrency limit active job", {
teamId,
jobId: holderId,
@@ -217,7 +306,6 @@ async function withSemaphore<T>(
});
activeSemaphores.dec();
- hb.stop();
endTimer();
await release(teamId, holderId).catch(() => {});
diff --git a/docker-compose.yaml b/docker-compose.yaml
index b6a1778e3..abf6980e7 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -38,8 +38,8 @@ x-common-env: &common-env
OPENAI_API_KEY: ${OPENAI_API_KEY}
OPENAI_BASE_URL: ${OPENAI_BASE_URL}
MODEL_NAME: ${MODEL_NAME}
- MODEL_EMBEDDING_NAME: ${MODEL_EMBEDDING_NAME}
- OLLAMA_BASE_URL: ${OLLAMA_BASE_URL}
+ MODEL_EMBEDDING_NAME: ${MODEL_EMBEDDING_NAME}
+ OLLAMA_BASE_URL: ${OLLAMA_BASE_URL}
AUTUMN_SECRET_KEY: ${AUTUMN_SECRET_KEY}
SLACK_WEBHOOK_URL: ${SLACK_WEBHOOK_URL}
BULL_AUTH_KEY: ${BULL_AUTH_KEY}
@@ -55,6 +55,9 @@ x-common-env: &common-env
SEARXNG_ENDPOINT: ${SEARXNG_ENDPOINT}
SEARXNG_ENGINES: ${SEARXNG_ENGINES}
SEARXNG_CATEGORIES: ${SEARXNG_CATEGORIES}
+ # Set NUQ_BACKEND=fdb to run the queue on FoundationDB instead of Postgres
+ NUQ_BACKEND: ${NUQ_BACKEND}
+ FDB_CLUSTER_FILE: ${NUQ_BACKEND:+/var/fdb/fdb.cluster}
services:
playwright-service:
@@ -106,6 +109,8 @@ services:
condition: service_healthy
ports:
- "${PORT:-3002}:${INTERNAL_PORT:-3002}"
+ volumes:
+ - fdb-cluster-file:/var/fdb:ro
command: node dist/src/harness.js --start-docker
# Resource limits for Docker Compose (not Swarm)
# Increase if you have more CPU cores/RAM available
@@ -129,7 +134,7 @@ services:
max-size: "5m"
max-file: "2"
compress: "true"
-
+
rabbitmq:
image: rabbitmq:3-management
networks:
@@ -147,7 +152,7 @@ services:
max-size: "5m"
max-file: "2"
compress: "true"
-
+
nuq-postgres:
# NOTE: If you don't want to build the image locally,
# comment out the build: statement and uncomment the image: statement
@@ -166,6 +171,45 @@ services:
max-file: "3"
compress: "true"
+ # FoundationDB queue backend (experimental). Enable by setting
+ # NUQ_BACKEND=fdb; the api container reads the shared cluster file from the
+ # fdb-cluster-file volume. Replaces nuq-postgres once stable.
+ foundationdb:
+ image: foundationdb/foundationdb:7.3.63
+ environment:
+ FDB_NETWORKING_MODE: container
+ FDB_COORDINATOR_PORT: 4500
+ networks:
+ - backend
+ volumes:
+ - fdb-data:/var/fdb/data
+ - fdb-cluster-file:/var/fdb
+ logging:
+ driver: "json-file"
+ options:
+ max-size: "10m"
+ max-file: "3"
+ compress: "true"
+
+ # one-shot: creates the database on first boot (no-op afterwards)
+ foundationdb-init:
+ image: foundationdb/foundationdb:7.3.63
+ depends_on:
+ - foundationdb
+ entrypoint:
+ - /bin/bash
+ - -c
+ - "sleep 5 && out=$(fdbcli -C /var/fdb/fdb.cluster --exec 'configure new single ssd' 2>&1); status=$$?; printf '%s\n' \"$$out\"; if [ \"$$status\" -eq 0 ]; then exit 0; fi; printf '%s\n' \"$$out\" | grep -Eiq 'already.*configured|database.*configured'"
+ volumes:
+ - fdb-cluster-file:/var/fdb
+ networks:
+ - backend
+ restart: "no"
+
networks:
backend:
driver: bridge
+
+volumes:
+ fdb-data:
+ fdb-cluster-file: