Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(nuq): rebuild queue + concurrency limiting on FoundationDB (core engine) (#3758)
Gergő Móriczb9eb2faecee6
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.ymlindex 2002b926b..6fd812c34 100644--- a/.github/workflows/test-server.yml+++ b/.github/workflows/test-server.yml@@ -19,6 +19,15 @@ jobs:proxy: ['proxy', 'no-proxy'] # proxy / no-proxy run different tests, keep bothsearch: ['searxng'] # disabled google for now, should be fine with just Searxngai: ['openai'] # AI only should be fine, as it simply adds tests, if non-AI fails, AI will fail.+ nuq: ['postgres']+ include:+ # Exercise the self-hosted API/harness path through the FDB NuQ router+ # without doubling the expensive Playwright/proxy matrix.+ - engine: fetch+ proxy: no-proxy+ search: searxng+ ai: openai+ nuq: fdb# legacy matrix:# engine: ["playwright", "fetch"]@@ -48,6 +57,9 @@ jobs:PROXY_PASSWORD: ${{ matrix.proxy == 'proxy' && secrets.PROXY_PASSWORD || '' }}NUQ_DATABASE_URL: postgres://postgres:postgres@localhost:5432/postgresNUQ_RABBITMQ_URL: amqp://localhost:5672+ NUQ_BACKEND: ${{ matrix.nuq == 'fdb' && 'fdb' || '' }}+ FDB_CLUSTER_FILE: ${{ matrix.nuq == 'fdb' && '/tmp/fdb.cluster' || '' }}+ FDB_VERSION: 7.3.63USE_GO_MARKDOWN_PARSER: trueALLOW_LOCAL_WEBHOOKS: trueFIRECRAWL_LOG_TO_FILE: true@@ -215,10 +227,29 @@ jobs:PORT: 3003- name: Run Docker Postgres+ if: matrix.nuq == 'postgres'run: |docker build -t firecrawl/nuq-postgres:latest ./apps/nuq-postgresdocker run -d -p 5432:5432 -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=postgres --name postgres firecrawl/nuq-postgres:latest+ - name: Install FoundationDB client library+ if: matrix.nuq == 'fdb'+ run: |+ curl -fsSL -o /tmp/fdb-clients.deb \+ "https://github.com/apple/foundationdb/releases/download/${FDB_VERSION}/foundationdb-clients_${FDB_VERSION}-1_amd64.deb"+ sudo dpkg -i /tmp/fdb-clients.deb++ - name: Start FoundationDB+ if: matrix.nuq == 'fdb'+ run: |+ docker run -d --name fdb -p 4500:4500 -e FDB_NETWORKING_MODE=host foundationdb/foundationdb:${FDB_VERSION}+ for i in $(seq 1 30); do+ if docker exec fdb fdbcli --exec "status minimal" 2>/dev/null; then break; fi+ sleep 1+ done+ docker exec fdb fdbcli --exec "configure new single memory"+ echo "docker:[email protected]:4500" > /tmp/fdb.cluster+- name: Run testsrun: pnpm harness pnpm test:snipsworking-directory: apps/api@@ -230,7 +261,7 @@ jobs:shell: python3 {0}env:JUNIT: apps/api/test-results/junit.xml- MATRIX_LABEL: "${{ matrix.engine }}, ${{ matrix.proxy }}, ${{ matrix.search }}, ${{ matrix.ai }}"+ MATRIX_LABEL: "${{ matrix.engine }}, ${{ matrix.proxy }}, ${{ matrix.search }}, ${{ matrix.ai }}, ${{ matrix.nuq }}"run: |import os, sys, xml.etree.ElementTree as ET@@ -367,7 +398,7 @@ jobs:- uses: actions/upload-artifact@v4if: always()with:- name: Logs (kubernetes, ${{ matrix.ai }}, ${{ matrix.search }}, ${{ matrix.engine }}, ${{ matrix.proxy }})+ name: Logs (kubernetes, ${{ matrix.ai }}, ${{ matrix.search }}, ${{ matrix.engine }}, ${{ matrix.proxy }}, ${{ matrix.nuq }})path: logs/logs.zip# temp disabled@@ -536,3 +567,52 @@ jobs:# name: Encrypted Logs# path: logs/logs.zip.gpg# retention-days: 5++ nuq-fdb-core:+ name: NuQ FoundationDB core tests+ runs-on: blacksmith-4vcpu-ubuntu-2404+ env:+ FDB_VERSION: 7.3.63+ steps:+ - uses: actions/checkout@v5++ - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4+ with:+ version: 10++ - run: pnpm config set store-dir ~/.pnpm-store+ - run: mkdir -p ~/.pnpm-store++ - uses: actions/setup-node@v6+ with:+ node-version: 22+ cache: pnpm+ cache-dependency-path: apps/api/pnpm-lock.yaml++ - name: Install API dependencies+ run: pnpm install --frozen-lockfile --ignore-scripts+ working-directory: apps/api+ env:+ npm_config_ignore_scripts: 'true'++ - name: Install FoundationDB client library+ run: |+ curl -fsSL -o /tmp/fdb-clients.deb \+ "https://github.com/apple/foundationdb/releases/download/${FDB_VERSION}/foundationdb-clients_${FDB_VERSION}-1_amd64.deb"+ sudo dpkg -i /tmp/fdb-clients.deb++ - name: Start FoundationDB+ run: |+ docker run -d --name fdb -p 4500:4500 -e FDB_NETWORKING_MODE=host foundationdb/foundationdb:${FDB_VERSION}+ for i in $(seq 1 30); do+ if docker exec fdb fdbcli --exec "status minimal" 2>/dev/null; then break; fi+ sleep 1+ done+ docker exec fdb fdbcli --exec "configure new single memory"+ echo "docker:[email protected]:4500" > /tmp/fdb.cluster++ - name: Run NuQ FDB core tests+ run: npx jest src/__tests__/nuq-fdb --reporters=default+ working-directory: apps/api+ env:+ FDB_CLUSTER_FILE: /tmp/fdb.clusterdiff --git a/apps/api/Dockerfile b/apps/api/Dockerfileindex ec85c8900..061b1c381 100644--- a/apps/api/Dockerfile+++ b/apps/api/Dockerfile@@ -27,6 +27,15 @@ RUN apt-get update && apt-get install -y \python3 \&& rm -rf /var/lib/apt/lists/*+# FoundationDB client library (libfdb_c + headers), needed by the+# `foundationdb` npm package at install time on arches without prebuilds+ARG FDB_VERSION=7.3.63+RUN ARCH=$(dpkg --print-architecture) && \+ case "$ARCH" in arm64) FDB_ARCH=aarch64 ;; *) FDB_ARCH=$ARCH ;; esac && \+ curl -fsSL -o /tmp/fdb-clients.deb \+ "https://github.com/apple/foundationdb/releases/download/${FDB_VERSION}/foundationdb-clients_${FDB_VERSION}-1_${FDB_ARCH}.deb" && \+ dpkg -i /tmp/fdb-clients.deb && rm /tmp/fdb-clients.deb+# Install RustENV RUSTUP_HOME=/usr/local/rustup \CARGO_HOME=/usr/local/cargo \@@ -58,8 +67,17 @@ FROM base AS runtimeRUN apt-get update && apt-get install -y \git \procps \+ curl \&& rm -rf /var/lib/apt/lists/*+# FoundationDB client library (libfdb_c + fdbcli)+ARG FDB_VERSION=7.3.63+RUN ARCH=$(dpkg --print-architecture) && \+ case "$ARCH" in arm64) FDB_ARCH=aarch64 ;; *) FDB_ARCH=$ARCH ;; esac && \+ curl -fsSL -o /tmp/fdb-clients.deb \+ "https://github.com/apple/foundationdb/releases/download/${FDB_VERSION}/foundationdb-clients_${FDB_VERSION}-1_${FDB_ARCH}.deb" && \+ dpkg -i /tmp/fdb-clients.deb && rm /tmp/fdb-clients.deb+EXPOSE 8080WORKDIR /appdiff --git a/apps/api/package.json b/apps/api/package.jsonindex b8cd62c33..3af53eef9 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -115,6 +115,7 @@"escape-html": "^1.0.3","express": "4.22.0","express-ws": "^5.0.2",+ "foundationdb": "^2.0.1","geoip-country": "^5.0.202510312342","git-diff": "^2.0.6","he": "^1.2.0",diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yamlindex c16588556..c41f6554d 100644--- a/apps/api/pnpm-lock.yaml+++ b/apps/api/pnpm-lock.yaml@@ -160,6 +160,9 @@ importers:express-ws:specifier: ^5.0.2+ foundationdb:+ specifier: ^2.0.1+ version: 2.0.1geoip-country:specifier: ^5.0.202510312342version: 5.0.202510312342@@ -3789,6 +3792,9 @@ packages:resolution: {integrity: sha512-jKmm9YtsNXN789RS/0mSzOC1NUq9mkVd65vbSSVsKdjGvYXBuE4oWe2QOEoFeRmJg+lPuZxpmrfFclNhoRMneQ==}+ resolution: {integrity: sha512-8jSvKPCYCgTpi9Pt87qlfTk6griyMx4Gk3Xv31Dp72Qp8b6XgIyFsMm8KzPmFJ9iJ8K4pGvRxvOS8D0XGnrkjw==}+resolution: {integrity: sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw==}@@ -3861,6 +3867,10 @@ packages:resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==}engines: {node: '>= 0.6'}+ resolution: {integrity: sha512-ZMmpAoEI88H8d4DWk6MzXf4Kw9pGaQrxNygVdz/7g+qwlMrjHvmxUQsQz1zG4176+B0YNY+ufgmcT40kEv/TnQ==}+ engines: {node: '>=14.0.0'}+resolution: {integrity: sha512-hJOBnKk2e89/FCSXU0UN7B2CpMnqtN6RrIKzoG3qWSjLUXGCTEU86aTdNEio/1Eu2NyExIAVJUyHuZvmEA9iuw==}engines: {node: '>= 10.0.0'}@@ -9411,6 +9421,8 @@ snapshots:dependencies:walk-up-path: 4.0.0+ [email protected]: {}+@@ -9495,6 +9507,11 @@ snapshots:+ dependencies:+ fdb-tuple: 1.0.0+ node-gyp-build: 4.8.4+dependencies:bindings: 1.5.0@@ -10623,8 +10640,7 @@ snapshots:detect-libc: 2.1.2optional: true- optional: true+ [email protected]: {}diff --git a/apps/api/src/__tests__/nuq-fdb/core.test.ts b/apps/api/src/__tests__/nuq-fdb/core.test.tsnew file mode 100644index 000000000..b43dd5da0--- /dev/null+++ b/apps/api/src/__tests__/nuq-fdb/core.test.ts@@ -0,0 +1,700 @@+import { randomUUID } from "crypto";+import { config } from "../../config";+import {+ NuQFdbQueue,+ NuQFdbJobGroup,+ NuqFdbSweeper,+} from "../../services/worker/nuq-fdb";+import {+ getNuqFdbDatabase,+ getFdb,+} from "../../services/worker/nuq-fdb/client";+import { encodeJson } from "../../services/worker/nuq-fdb/keyspace";++// These tests exercise the FDB queue core directly against a real FoundationDB+// cluster (no API server needed). They are skipped when FDB is not configured.+const describeIf = config.FDB_CLUSTER_FILE ? describe : describe.skip;++// inline returnvalues only exist on the self-host path; cloud stores to GCS+const expectInlineReturnvalue = !config.GCS_BUCKET_NAME;++const RUN = randomUUID().slice(0, 8);+const TEST_LEASE_MS = 1500;++const createdQueueNames: string[] = [];++type Ctx = {+ queue: NuQFdbQueue;+ finishedQueue: NuQFdbQueue;+ group: NuQFdbJobGroup;+ sweeper: NuqFdbSweeper;+};++// Each test gets its own queue keyspace so leaked jobs (some tests leave them+// behind on purpose) can never bleed into other tests' takes.+async function makeCtx(name: string): Promise<Ctx> {+ const scrapeName = `t-${RUN}-${name}`;+ const finishedName = `t-${RUN}-${name}-fin`;+ createdQueueNames.push(scrapeName, finishedName);+ const queue = new NuQFdbQueue(scrapeName, {+ hasGroups: true,+ finishedQueueName: finishedName,+ leaseMs: TEST_LEASE_MS,+ });+ const finishedQueue = new NuQFdbQueue(finishedName, { hasGroups: false });+ const group = new NuQFdbJobGroup(queue.ks, queue.groupOps!);+ const sweeper = new NuqFdbSweeper([queue, finishedQueue]);+ return { queue, finishedQueue, group, sweeper };+}++async function takeAll(+ queue: NuQFdbQueue,+ maxJobs: number = 50,+): Promise<any[]> {+ const out: any[] = [];+ while (out.length < maxJobs) {+ const job = await queue.getJobToProcess();+ if (job === null) break;+ out.push(job);+ }+ return out;+}++function freshOwner(): string {+ return randomUUID();+}++function scrapeData(extra: Record<string, any> = {}): any {+ return { mode: "single_urls", url: "https://example.com", ...extra };+}++const UNLIMITED = { teamLimit: null, queueCap: 1_000_000 };+const gate = (limit: number, cap: number = 1_000_000) => ({+ teamLimit: limit,+ queueCap: cap,+});++describeIf("NuQ FDB core", () => {+ afterAll(async () => {+ const fdb = getFdb();+ const db = getNuqFdbDatabase();+ for (const name of createdQueueNames) {+ const r = fdb.tuple.range(["nuq", name]);+ await db.doTn(async tn =>+ tn.clearRange(r.begin as Buffer, r.end as Buffer),+ );+ }+ });++ test("enqueue -> take -> finish roundtrip (ungated)", async () => {+ const { queue } = await makeCtx("roundtrip");+ const id = randomUUID();+ await queue.addJob(id, scrapeData(), { ownerId: freshOwner() }, UNLIMITED);++ const taken = await takeAll(queue, 1);+ expect(taken.length).toBe(1);+ expect(taken[0].id).toBe(id);+ expect(taken[0].lock).toBeDefined();+ expect(taken[0].data.url).toBe("https://example.com");++ const ok = await queue.jobFinish(id, taken[0].lock!, { result: "yay" });+ expect(ok).toBe(true);++ const job = await queue.getJob(id);+ expect(job?.status).toBe("completed");+ if (expectInlineReturnvalue) {+ expect(job?.returnvalue).toEqual({ result: "yay" });+ } else {+ expect(job?.returnvalue).toBeNull();+ }+ });++ test("finish with wrong lock is rejected; double finish is idempotent", async () => {+ const { queue } = await makeCtx("locks");+ const id = randomUUID();+ await queue.addJob(id, scrapeData(), { ownerId: freshOwner() }, UNLIMITED);+ const [job] = await takeAll(queue, 1);+ expect(job.id).toBe(id);++ expect(await queue.jobFinish(id, randomUUID(), null)).toBe(false);+ expect(await queue.jobFinish(id, job.lock!, { ok: 1 })).toBe(true);+ expect(await queue.jobFinish(id, job.lock!, { ok: 1 })).toBe(true);+ expect(await queue.jobFail(id, job.lock!, "nope")).toBe(false);+ });++ test("team concurrency gate: limit 2 admits 2, backlogs the rest, promotes on finish", async () => {+ const { queue } = await makeCtx("teamgate");+ const owner = freshOwner();+ const ids = Array.from({ length: 5 }, () => randomUUID());+ const jobs = await queue.addJobs(+ ids.map(id => ({+ id,+ data: scrapeData(),+ options: { ownerId: owner, timesOutAt: new Date(Date.now() + 60_000) },+ })),+ gate(2),+ );++ expect(jobs.filter(j => j.status === "queued").length).toBe(2);+ expect(jobs.filter(j => j.status === "backlog").length).toBe(3);+ expect(await queue.getTeamActiveCount(owner)).toBe(2);+ expect(await queue.getTeamPendingCount(owner)).toBe(3);++ const taken = await takeAll(queue, 5);+ expect(taken.length).toBe(2);++ // finishing one job promotes exactly one backlogged job+ await queue.jobFinish(taken[0].id, taken[0].lock!, null);+ expect(await queue.getTeamActiveCount(owner)).toBe(2);+ expect(await queue.getTeamPendingCount(owner)).toBe(2);++ const next = await takeAll(queue, 5);+ expect(next.length).toBe(1);++ await queue.jobFinish(taken[1].id, taken[1].lock!, null);+ await queue.jobFinish(next[0].id, next[0].lock!, null);+ const rest = await takeAll(queue, 5);+ expect(rest.length).toBe(2);+ for (const j of rest) await queue.jobFinish(j.id, j.lock!, null);+ expect(await queue.getTeamActiveCount(owner)).toBe(0);+ expect(await queue.getTeamPendingCount(owner)).toBe(0);+ });++ test("promotion respects priority order", async () => {+ const { queue } = await makeCtx("priority");+ const owner = freshOwner();+ const blocker = randomUUID();+ await queue.addJob(+ blocker,+ scrapeData(),+ { ownerId: owner, priority: 0 },+ gate(1),+ );+ const lowPrio = randomUUID();+ const highPrio = randomUUID();+ await queue.addJob(+ lowPrio,+ scrapeData(),+ {+ ownerId: owner,+ priority: 50,+ timesOutAt: new Date(Date.now() + 60_000),+ },+ gate(1),+ );+ await queue.addJob(+ highPrio,+ scrapeData(),+ {+ ownerId: owner,+ priority: 1,+ timesOutAt: new Date(Date.now() + 60_000),+ },+ gate(1),+ );++ const [b] = await takeAll(queue, 1);+ expect(b.id).toBe(blocker);+ await queue.jobFinish(b.id, b.lock!, null);++ const [promoted] = await takeAll(queue, 1);+ expect(promoted.id).toBe(highPrio);+ });++ test("kickoff jobs bypass the gate and hold no slot", async () => {+ const { queue } = await makeCtx("kickoff");+ const owner = freshOwner();+ const kickoff = randomUUID();+ await queue.addJob(+ kickoff,+ { mode: "kickoff", crawl_id: randomUUID() },+ { ownerId: owner, bypassGate: true },+ gate(1),+ );+ expect(await queue.getTeamActiveCount(owner)).toBe(0);++ const [k] = await takeAll(queue, 1);+ expect(k.id).toBe(kickoff);+ expect(await queue.getTeamActiveCount(owner)).toBe(0);+ await queue.jobFinish(k.id, k.lock!, null);+ expect(await queue.getTeamActiveCount(owner)).toBe(0);+ });++ test("QueueFullError when the backlog cap is exceeded", async () => {+ const { queue } = await makeCtx("qfull");+ const owner = freshOwner();+ const blocker = randomUUID();+ await queue.addJob(blocker, scrapeData(), { ownerId: owner }, gate(1, 2));+ // 1 active, cap 2: two backlogged jobs fit, the third addJobs blows up+ await queue.addJobs(+ [randomUUID(), randomUUID()].map(id => ({+ id,+ data: scrapeData(),+ options: { ownerId: owner, timesOutAt: new Date(Date.now() + 60_000) },+ })),+ gate(1, 2),+ );+ await expect(+ queue.addJob(+ randomUUID(),+ scrapeData(),+ { ownerId: owner, timesOutAt: new Date(Date.now() + 60_000) },+ gate(1, 2),+ ),+ ).rejects.toThrow(/queue limit reached/i);+ });++ test("group lifecycle: numeric stats, finish detection, crawl_finished emission", async () => {+ const { queue, finishedQueue, group } = await makeCtx("glife");+ const owner = freshOwner();+ const gid = randomUUID();+ await group.addGroup(gid, owner);++ const ids = Array.from({ length: 3 }, () => randomUUID());+ await queue.addJobs(+ ids.map(id => ({+ id,+ data: scrapeData(),+ options: { ownerId: owner, groupId: gid },+ })),+ gate(10),+ );++ let stats = await queue.getGroupNumericStats(gid);+ expect(stats.queued).toBe(3);++ const taken = await takeAll(queue, 3);+ expect(taken.length).toBe(3);+ stats = await queue.getGroupNumericStats(gid);+ expect(stats.active).toBe(3);++ for (const j of taken.slice(0, 2))+ await queue.jobFinish(j.id, j.lock!, null);+ await queue.jobFail(taken[2].id, taken[2].lock!, "boom");++ stats = await queue.getGroupNumericStats(gid);+ expect(stats.completed).toBe(2);+ expect(stats.failed).toBe(1);+ expect(stats.active).toBe(0);++ // the last terminal transition completes the group inline+ const g = await group.getGroup(gid);+ expect(g?.status).toBe("completed");+ expect(g?.expiresAt).toBeDefined();++ // and emits exactly one crawl_finished job+ const finishedJob = await finishedQueue.getJobToProcess();+ expect(finishedJob).not.toBeNull();+ expect(finishedJob!.groupId).toBe(gid);+ expect(await finishedQueue.getJobToProcess()).toBeNull();+ await finishedQueue.jobFinish(finishedJob!.id, finishedJob!.lock!, null);++ const ongoing = await group.getOngoingByOwner(owner);+ expect(ongoing.find(o => o.id === gid)).toBeUndefined();+ });++ test("crawl maxConcurrency gates within the team limit", async () => {+ const { queue, group } = await makeCtx("crawlmax");+ const owner = freshOwner();+ const gid = randomUUID();+ await group.addGroup(gid, owner, undefined, { maxConcurrency: 1 });++ const ids = Array.from({ length: 3 }, () => randomUUID());+ await queue.addJobs(+ ids.map(id => ({+ id,+ data: scrapeData(),+ options: {+ ownerId: owner,+ groupId: gid,+ timesOutAt: new Date(Date.now() + 60_000),+ },+ })),+ gate(10),+ );++ // only 1 crawl slot: one job ready, two crawl-pending+ expect(await queue.getTeamActiveCount(owner)).toBe(1);+ const taken1 = await takeAll(queue, 3);+ expect(taken1.length).toBe(1);++ await queue.jobFinish(taken1[0].id, taken1[0].lock!, null);+ const taken2 = await takeAll(queue, 3);+ expect(taken2.length).toBe(1);++ await queue.jobFinish(taken2[0].id, taken2[0].lock!, null);+ const taken3 = await takeAll(queue, 3);+ expect(taken3.length).toBe(1);+ await queue.jobFinish(taken3[0].id, taken3[0].lock!, null);++ const g = await group.getGroup(gid);+ expect(g?.status).toBe("completed");+ });++ test("crawl delay: next job is parked until the not-before time", async () => {+ const { queue, group, sweeper } = await makeCtx("delay");+ const owner = freshOwner();+ const gid = randomUUID();+ await group.addGroup(gid, owner, undefined, { delaySeconds: 1 });++ const ids = [randomUUID(), randomUUID()];+ await queue.addJobs(+ ids.map(id => ({+ id,+ data: scrapeData(),+ options: {+ ownerId: owner,+ groupId: gid,+ timesOutAt: new Date(Date.now() + 60_000),+ },+ })),+ gate(10),+ );++ const [first] = await takeAll(queue, 2);+ expect(first).toBeDefined();+ await queue.jobFinish(first.id, first.lock!, null);++ // second job is in the delay index, not takeable yet+ await sweeper.sweepOnce();+ expect(await queue.getJobToProcess()).toBeNull();++ await new Promise(resolve => setTimeout(resolve, 1200));+ await sweeper.sweepOnce();+ const [second] = await takeAll(queue, 1);+ expect(second).toBeDefined();+ expect(second.id).toBe(ids.find(i => i !== first.id));+ await queue.jobFinish(second.id, second.lock!, null);+ });++ test("lease expiry: sweeper requeues a stalled job, then fails it after MAX_STALLS", async () => {+ const { queue, sweeper } = await makeCtx("stalls");+ const owner = freshOwner();+ const id = randomUUID();+ await queue.addJob(id, scrapeData(), { ownerId: owner }, gate(5));++ const [job] = await takeAll(queue, 1);+ expect(job.id).toBe(id);++ // let the lease expire without renewal+ await new Promise(resolve => setTimeout(resolve, TEST_LEASE_MS + 200));+ await sweeper.sweepOnce();++ let j = await queue.getJob(id);+ expect(j?.status).toBe("queued");++ // worker that lost its lease can no longer finish+ expect(await queue.jobFinish(id, job.lock!, null)).toBe(false);++ // stall it to death+ for (let i = 0; i < 9; i++) {+ const [again] = await takeAll(queue, 1);+ expect(again.id).toBe(id);+ await new Promise(resolve => setTimeout(resolve, TEST_LEASE_MS + 200));+ await sweeper.sweepOnce();+ }+ j = await queue.getJob(id);+ expect(j?.status).toBe("failed");+ expect(j?.failedReason).toMatch(/stalled/i);+ expect(await queue.getTeamActiveCount(owner)).toBe(0);+ }, 60_000);++ test("renewLock keeps the lease alive", async () => {+ const { queue, sweeper } = await makeCtx("renew");+ const owner = freshOwner();+ const id = randomUUID();+ await queue.addJob(id, scrapeData(), { ownerId: owner }, gate(5));+ const [job] = await takeAll(queue, 1);++ for (let i = 0; i < 3; i++) {+ await new Promise(resolve => setTimeout(resolve, TEST_LEASE_MS / 2));+ expect(await queue.renewLock(id, job.lock!)).toBe(true);+ }+ await sweeper.sweepOnce();+ const j = await queue.getJob(id);+ expect(j?.status).toBe("active");+ expect(await queue.jobFinish(id, job.lock!, null)).toBe(true);+ }, 30_000);++ test("backlog timeout: pending jobs are silently dropped at their deadline", async () => {+ const { queue, sweeper } = await makeCtx("bto");+ const owner = freshOwner();+ const blocker = randomUUID();+ await queue.addJob(blocker, scrapeData(), { ownerId: owner }, gate(1));+ const doomed = randomUUID();+ await queue.addJob(+ doomed,+ scrapeData(),+ { ownerId: owner, timesOutAt: new Date(Date.now() - 1000) },+ gate(1),+ );++ expect(await queue.getTeamPendingCount(owner)).toBe(1);+ await sweeper.sweepOnce();+ expect(await queue.getTeamPendingCount(owner)).toBe(0);+ expect(await queue.getJob(doomed)).toBeNull();++ const [b] = await takeAll(queue, 1);+ await queue.jobFinish(b.id, b.lock!, null);+ });++ test("group cancellation: pending dropped, ready diverted, group still completes", async () => {+ const { queue, finishedQueue, group, sweeper } = await makeCtx("cancel");+ const owner = freshOwner();+ const gid = randomUUID();+ await group.addGroup(gid, owner);++ const ids = Array.from({ length: 4 }, () => randomUUID());+ await queue.addJobs(+ ids.map(id => ({+ id,+ data: scrapeData(),+ options: {+ ownerId: owner,+ groupId: gid,+ timesOutAt: new Date(Date.now() + 60_000),+ },+ })),+ gate(2),+ );+ // 2 ready, 2 team-pending+ const [active] = await takeAll(queue, 1);++ expect(await group.cancelGroup(gid)).toBe(true);+ await sweeper.sweepOnce(); // cleans pending members++ // the remaining ready job is diverted at take time+ const after = await takeAll(queue, 4);+ expect(after.length).toBe(0);++ // active job finishes normally; that drains the group+ await queue.jobFinish(active.id, active.lock!, null);+ await sweeper.sweepOnce();++ const g = await group.getGroup(gid);+ expect(g?.status).toBe("completed");+ expect(await queue.getTeamActiveCount(owner)).toBe(0);++ // cancelled crawls still emit their crawl_finished job+ const fin = await finishedQueue.getJobToProcess();+ expect(fin).not.toBeNull();+ expect(fin!.groupId).toBe(gid);+ await finishedQueue.jobFinish(fin!.id, fin!.lock!, null);+ });++ test("group cancellation scans past stale group-index rows before clearing task", async () => {+ const { queue, group, sweeper } = await makeCtx("cancel-stale-index");+ const db = getNuqFdbDatabase();+ const owner = freshOwner();+ const gid = randomUUID();+ const pendingId = "zzzz-real-pending";+ await group.addGroup(gid, owner);++ await db.doTn(async tn => {+ for (let i = 0; i < 600; i++) {+ tn.set(+ queue.ks.groupJob(gid, `0000-stale-${String(i).padStart(4, "0")}`),+ encodeJson({ m: 1, s: "pending" }),+ );+ }+ });++ await queue.addJob(+ pendingId,+ scrapeData(),+ {+ ownerId: owner,+ groupId: gid,+ timesOutAt: new Date(Date.now() + 60_000),+ },+ gate(0),+ );+ expect(await queue.getTeamPendingCount(owner)).toBe(1);++ expect(await group.cancelGroup(gid)).toBe(true);+ await sweeper.sweepOnce();++ expect(await queue.getJob(pendingId)).toBeNull();+ expect(await queue.getTeamPendingCount(owner)).toBe(0);+ await db.doTn(async tn => {+ expect(await tn.get(queue.ks.taskGroupCancel(gid))).toBeFalsy();+ });+ });++ test("waitForJob resolves on completion and rejects on failure", async () => {+ const { queue } = await makeCtx("wait");+ const owner = freshOwner();+ const id1 = randomUUID();+ await queue.addJob(id1, scrapeData(), { ownerId: owner }, UNLIMITED);++ const wait1 = queue.waitForJob(id1, 15_000);+ const [j1] = await takeAll(queue, 1);+ await queue.jobFinish(j1.id, j1.lock!, { doc: "ok" });+ if (expectInlineReturnvalue) {+ await expect(wait1).resolves.toEqual({ doc: "ok" });+ } else {+ await expect(wait1).resolves.toBeNull();+ }++ const id2 = randomUUID();+ await queue.addJob(id2, scrapeData(), { ownerId: owner }, UNLIMITED);+ const wait2 = queue.waitForJob(id2, 15_000);+ const [j2] = await takeAll(queue, 1);+ await queue.jobFail(j2.id, j2.lock!, "scrape exploded");+ await expect(wait2).rejects.toThrow("scrape exploded");++ const id3 = randomUUID();+ await queue.addJob(id3, scrapeData(), { ownerId: owner }, UNLIMITED);+ await expect(queue.waitForJob(id3, 500)).rejects.toThrow(/timed out/i);+ const [j3] = await takeAll(queue, 1);+ await queue.jobFinish(j3.id, j3.lock!, null);+ }, 30_000);++ test("large returnvalue is chunked and reassembled (self-host path)", async () => {+ if (!expectInlineReturnvalue) return;+ const { queue } = await makeCtx("chunks");+ const owner = freshOwner();+ const id = randomUUID();+ await queue.addJob(id, scrapeData(), { ownerId: owner }, UNLIMITED);+ const [job] = await takeAll(queue, 1);++ const big = { blob: "x".repeat(300 * 1024) };+ await queue.jobFinish(id, job.lock!, big);++ const j = await queue.getJob(id);+ expect(j?.status).toBe("completed");+ expect(j?.returnvalue?.blob?.length).toBe(300 * 1024);+ });++ test("getCrawlJobsForListing paginates completed jobs in finish order", async () => {+ const { queue, group } = await makeCtx("listing");+ const owner = freshOwner();+ const gid = randomUUID();+ await group.addGroup(gid, owner);++ const ids = Array.from({ length: 5 }, () => randomUUID());+ await queue.addJobs(+ ids.map(id => ({+ id,+ data: scrapeData(),+ options: { ownerId: owner, groupId: gid },+ })),+ gate(10),+ );+ const taken = await takeAll(queue, 5);+ expect(taken.length).toBe(5);+ const finishOrder: string[] = [];+ for (const j of taken) {+ await queue.jobFinish(j.id, j.lock!, null);+ finishOrder.push(j.id);+ }++ const page1 = await queue.getCrawlJobsForListing(gid, 3, 0);+ const page2 = await queue.getCrawlJobsForListing(gid, 3, 3);+ expect(page1.map(j => j.id)).toEqual(finishOrder.slice(0, 3));+ expect(page2.map(j => j.id)).toEqual(finishOrder.slice(3));+ });++ test("getGroupAnyJob returns a single_urls member and checks ownership", async () => {+ const { queue, group } = await makeCtx("anyjob");+ const owner = freshOwner();+ const gid = randomUUID();+ await group.addGroup(gid, owner);+ const id = randomUUID();+ await queue.addJob(+ id,+ scrapeData(),+ { ownerId: owner, groupId: gid },+ gate(10),+ );++ const any = await queue.getGroupAnyJob(gid, owner);+ expect(any?.id).toBe(id);+ expect(await queue.getGroupAnyJob(gid, randomUUID())).toBeNull();++ const [j] = await takeAll(queue, 1);+ await queue.jobFinish(j.id, j.lock!, null);+ });++ test("removeJob releases slots and promotes backlog", async () => {+ const { queue } = await makeCtx("remove");+ const owner = freshOwner();+ const a = randomUUID();+ const b = randomUUID();+ await queue.addJob(a, scrapeData(), { ownerId: owner }, gate(1));+ await queue.addJob(+ b,+ scrapeData(),+ { ownerId: owner, timesOutAt: new Date(Date.now() + 60_000) },+ gate(1),+ );+ expect(await queue.getTeamPendingCount(owner)).toBe(1);++ // removing the slot-holding job promotes the backlogged one+ expect(await queue.removeJob(a)).toBe(true);+ expect(await queue.getTeamPendingCount(owner)).toBe(0);+ expect(await queue.getJob(a)).toBeNull();++ const [j] = await takeAll(queue, 1);+ expect(j.id).toBe(b);+ await queue.jobFinish(j.id, j.lock!, null);+ });++ test("limit raise promotes backlogged jobs via the sweeper", async () => {+ const { queue, sweeper } = await makeCtx("raise");+ const owner = freshOwner();+ const ids = Array.from({ length: 4 }, () => randomUUID());+ await queue.addJobs(+ ids.map(id => ({+ id,+ data: scrapeData(),+ options: { ownerId: owner, timesOutAt: new Date(Date.now() + 60_000) },+ })),+ gate(1),+ );+ expect(await queue.getTeamActiveCount(owner)).toBe(1);+ expect(await queue.getTeamPendingCount(owner)).toBe(3);++ // a later enqueue arrives with a raised limit (ACUC change)+ await queue.addJob(+ randomUUID(),+ scrapeData(),+ { ownerId: owner, timesOutAt: new Date(Date.now() + 60_000) },+ gate(4),+ );+ await sweeper.sweepOnce();++ expect(await queue.getTeamActiveCount(owner)).toBe(4);+ expect(await queue.getTeamPendingCount(owner)).toBe(1);++ const taken = await takeAll(queue, 4);+ expect(taken.length).toBe(4);+ for (const j of taken) await queue.jobFinish(j.id, j.lock!, null);+ const [last] = await takeAll(queue, 1);+ await queue.jobFinish(last.id, last.lock!, null);+ });++ test("group TTL cleanup removes job records", async () => {+ const { queue, group, sweeper } = await makeCtx("gttl");+ const owner = freshOwner();+ const gid = randomUUID();+ await group.addGroup(gid, owner, 1000); // 1s TTL++ const id = randomUUID();+ await queue.addJob(+ id,+ scrapeData(),+ { ownerId: owner, groupId: gid },+ gate(10),+ );+ const [j] = await takeAll(queue, 1);+ await queue.jobFinish(j.id, j.lock!, null);++ expect((await group.getGroup(gid))?.status).toBe("completed");+ await new Promise(resolve => setTimeout(resolve, 1100));+ await sweeper.sweepOnce();++ expect(await group.getGroup(gid)).toBeNull();+ expect(await queue.getJob(id)).toBeNull();+ });+});diff --git a/apps/api/src/__tests__/nuq-fdb/router.test.ts b/apps/api/src/__tests__/nuq-fdb/router.test.tsnew file mode 100644index 000000000..5c5a9fdb7--- /dev/null+++ b/apps/api/src/__tests__/nuq-fdb/router.test.ts@@ -0,0 +1,187 @@+import { randomUUID } from "crypto";+import { config } from "../../config";+import {+ fdbQueueEnabled,+ isFdbTeam,+ resolveJobBackend,+ resolveNewGroupBackend,+ fdbEnqueueScrapeJobs,+ scrapeQueue,+ crawlGroup,+ crawlFinishedQueue,+ mirrorExternalSlotAcquire,+ mirrorExternalSlotRelease,+} from "../../services/worker/nuq-router";+import { scrapeQueueFdb } from "../../services/worker/nuq-fdb";+import {+ getNuqFdbDatabase,+ getFdb,+} from "../../services/worker/nuq-fdb/client";++// Exercises the dual-backend router in forced-FDB mode (NUQ_BACKEND=fdb,+// self-hosted), which needs neither ACUC nor a PG nuq database: everything+// must route to FDB and never touch the PG pool. Requires a live FDB cluster.+const describeIf = config.FDB_CLUSTER_FILE ? describe : describe.skip;++const prevNuqBackend = config.NUQ_BACKEND;+const prevDbAuth = config.USE_DB_AUTHENTICATION;++describeIf("NuQ router (forced FDB mode)", () => {+ beforeAll(() => {+ config.NUQ_BACKEND = "fdb";+ config.USE_DB_AUTHENTICATION = false; // self-hosted: unlimited concurrency+ });++ afterAll(async () => {+ config.NUQ_BACKEND = prevNuqBackend;+ config.USE_DB_AUTHENTICATION = prevDbAuth;+ // forced mode writes into the real "scrape"/"crawl_finished" queue+ // namespaces; wipe them so reruns and other suites start clean+ const fdb = getFdb();+ const db = getNuqFdbDatabase();+ for (const name of ["scrape", "crawl_finished"]) {+ const r = fdb.tuple.range(["nuq", name]);+ await db.doTn(async tn =>+ tn.clearRange(r.begin as Buffer, r.end as Buffer),+ );+ }+ });++ test("routing decisions resolve to fdb when forced", async () => {+ expect(fdbQueueEnabled()).toBe(true);+ expect(await isFdbTeam(randomUUID())).toBe(true);+ expect(await resolveNewGroupBackend(randomUUID())).toBe("fdb");+ expect(+ await resolveJobBackend({+ mode: "single_urls",+ url: "https://example.com",+ team_id: randomUUID(),+ } as any),+ ).toBe("fdb");+ });++ test("enqueue -> routed take -> routed finish -> routed waitForJob, all on FDB", async () => {+ const teamId = randomUUID();+ const jobId = randomUUID();+ const { jobs, backloggedCount } = await fdbEnqueueScrapeJobs(+ [+ {+ jobId,+ data: {+ mode: "single_urls",+ url: "https://example.com",+ team_id: teamId,+ } as any,+ priority: 0,+ backlogTimeoutMs: 60_000,+ },+ ],+ teamId,+ );+ expect(jobs[0].backend).toBe("fdb");+ expect(jobs[0].status).toBe("queued"); // self-hosted: no gate+ expect(backloggedCount).toBe(0);++ const wait = scrapeQueue.waitForJob(jobId, 15_000);++ // routed take must find the FDB job without ever polling PG (no PG here)+ let taken: any = null;+ for (let i = 0; i < 10 && !taken; i++) {+ try {+ taken = await scrapeQueue.getJobToProcess();+ } catch {+ // PG fallback poll can throw without a database; FDB must still win+ }+ }+ expect(taken).not.toBeNull();+ expect(taken.id).toBe(jobId);+ expect((taken as any).backend).toBe("fdb");++ expect(await scrapeQueue.renewLock(jobId, taken.lock!)).toBe(true);+ expect(await scrapeQueue.jobFinish(jobId, taken.lock!, { ok: true })).toBe(+ true,+ );++ await expect(wait).resolves.toBeDefined();++ const job = await scrapeQueue.getJob(jobId);+ expect(job?.status).toBe("completed");+ const jobsRead = await scrapeQueue.getJobs([jobId]);+ expect(jobsRead.length).toBe(1);+ });++ test("routed group lifecycle incl. crawl_finished consumption and cancel", async () => {+ const teamId = randomUUID();+ const gid = randomUUID();+ const group = await crawlGroup.addGroup(gid, teamId, 60_000, {+ backend: "fdb",+ });+ expect(group.status).toBe("active");+ expect((await crawlGroup.getGroup(gid))?.id).toBe(gid);++ const jobId = randomUUID();+ await fdbEnqueueScrapeJobs(+ [+ {+ jobId,+ data: {+ mode: "single_urls",+ url: "https://example.com",+ team_id: teamId,+ crawl_id: gid,+ } as any,+ priority: 0,+ backlogTimeoutMs: 60_000,+ },+ ],+ teamId,+ );++ let taken: any = null;+ for (let i = 0; i < 10 && !taken; i++) {+ try {+ taken = await scrapeQueue.getJobToProcess();+ } catch {}+ }+ expect(taken?.id).toBe(jobId);+ await scrapeQueue.jobFinish(jobId, taken.lock!, null);++ expect((await crawlGroup.getGroup(gid))?.status).toBe("completed");+ expect(await scrapeQueue.getGroupNumericStats(gid)).toMatchObject({+ completed: 1,+ });+ const listing = await scrapeQueue.getCrawlJobsForListing(gid, 10, 0);+ expect(listing.map(j => j.id)).toEqual([jobId]);++ // the emitted crawl_finished job is consumable through the router+ let fin: any = null;+ for (let i = 0; i < 10 && !fin; i++) {+ try {+ fin = await crawlFinishedQueue.getJobToProcess();+ } catch {}+ }+ expect(fin).not.toBeNull();+ expect(fin.groupId).toBe(gid);+ expect(await crawlFinishedQueue.jobFinish(fin.id, fin.lock!, null)).toBe(+ true,+ );++ // cancel on an already-completed group is a no-op+ expect(await crawlGroup.cancelGroup(gid)).toBe(false);+ });++ test("external slot mirror consumes and releases FDB capacity", async () => {+ const teamId = randomUUID();+ const holder = randomUUID();+ await mirrorExternalSlotAcquire(teamId, holder, 30_000);+ expect(await scrapeQueueFdb.getTeamActiveCount(teamId)).toBe(1);+ // re-acquire (heartbeat) must not double-count+ await mirrorExternalSlotAcquire(teamId, holder, 30_000);+ expect(await scrapeQueueFdb.getTeamActiveCount(teamId)).toBe(1);+ await mirrorExternalSlotRelease(teamId, holder);+ expect(await scrapeQueueFdb.getTeamActiveCount(teamId)).toBe(0);+ // double release is a no-op+ await mirrorExternalSlotRelease(teamId, holder);+ expect(await scrapeQueueFdb.getTeamActiveCount(teamId)).toBe(0);+ });+});diff --git a/apps/api/src/__tests__/nuq-fdb/stress.test.ts b/apps/api/src/__tests__/nuq-fdb/stress.test.tsnew file mode 100644index 000000000..db6daeb56--- /dev/null+++ b/apps/api/src/__tests__/nuq-fdb/stress.test.ts@@ -0,0 +1,176 @@+import { randomUUID } from "crypto";+import { config } from "../../config";+import { NuQFdbQueue, NuQFdbJobGroup } from "../../services/worker/nuq-fdb";+import {+ getNuqFdbDatabase,+ getFdb,+} from "../../services/worker/nuq-fdb/client";++const describeIf = config.FDB_CLUSTER_FILE ? describe : describe.skip;++const RUN = randomUUID().slice(0, 8);++describeIf("NuQ FDB concurrency stress", () => {+ const queueNames: string[] = [];++ afterAll(async () => {+ const fdb = getFdb();+ const db = getNuqFdbDatabase();+ for (const name of queueNames) {+ const r = fdb.tuple.range(["nuq", name]);+ await db.doTn(async tn =>+ tn.clearRange(r.begin as Buffer, r.end as Buffer),+ );+ }+ });++ test("parallel workers never exceed the team limit and drain everything", async () => {+ const name = `t-${RUN}-stress`;+ queueNames.push(name);+ const queue: NuQFdbQueue = new NuQFdbQueue(name, { hasGroups: true });++ const owner = randomUUID();+ const LIMIT = 5;+ const JOBS = 60;+ const WORKERS = 12;+ const gate = { teamLimit: LIMIT, queueCap: 1_000_000 };++ await queue.addJobs(+ Array.from({ length: JOBS }, () => ({+ id: randomUUID(),+ data: { mode: "single_urls", url: "https://example.com" },+ options: {+ ownerId: owner,+ timesOutAt: new Date(Date.now() + 120_000),+ },+ })),+ gate,+ );++ expect(await queue.getTeamActiveCount(owner)).toBe(LIMIT);+ expect(await queue.getTeamPendingCount(owner)).toBe(JOBS - LIMIT);++ let inFlight = 0;+ let maxInFlight = 0;+ let processed = 0;+ const seen = new Set<string>();++ // simulated workers: take, hold briefly, finish+ const worker = async () => {+ let idleStrikes = 0;+ while (idleStrikes < 8) {+ const job = await queue.getJobToProcess();+ if (!job) {+ idleStrikes++;+ await new Promise(resolve => setTimeout(resolve, 50));+ continue;+ }+ idleStrikes = 0;+ expect(seen.has(job.id)).toBe(false);+ seen.add(job.id);+ inFlight++;+ maxInFlight = Math.max(maxInFlight, inFlight);+ await new Promise(resolve =>+ setTimeout(resolve, 10 + Math.random() * 30),+ );+ inFlight--;+ const ok = await queue.jobFinish(job.id, job.lock!, null);+ expect(ok).toBe(true);+ processed++;+ }+ };++ await Promise.all(Array.from({ length: WORKERS }, () => worker()));++ expect(processed).toBe(JOBS);+ expect(maxInFlight).toBeLessThanOrEqual(LIMIT);+ expect(await queue.getTeamActiveCount(owner)).toBe(0);+ expect(await queue.getTeamPendingCount(owner)).toBe(0);+ }, 120_000);++ test("parallel enqueuers + workers with a crawl gate stay within both limits", async () => {+ const name = `t-${RUN}-stress2`;+ queueNames.push(name);+ const queue: NuQFdbQueue = new NuQFdbQueue(name, {+ hasGroups: true,+ finishedQueueName: `${name}-fin`,+ });+ queueNames.push(`${name}-fin`);+ const finishedQueue: NuQFdbQueue = new NuQFdbQueue(`${name}-fin`, {+ hasGroups: false,+ });+ const group = new NuQFdbJobGroup(queue.ks, queue.groupOps!);++ const owner = randomUUID();+ const gid = randomUUID();+ const CRAWL_LIMIT = 3;+ const TEAM_LIMIT = 10;+ const JOBS = 40;+ const gate = { teamLimit: TEAM_LIMIT, queueCap: 1_000_000 };+ await group.addGroup(gid, owner, undefined, {+ maxConcurrency: CRAWL_LIMIT,+ });++ // enqueue from 4 parallel producers while workers consume+ const producers = Array.from({ length: 4 }, (_, p) =>+ queue.addJobs(+ Array.from({ length: JOBS / 4 }, () => ({+ id: randomUUID(),+ data: { mode: "single_urls", url: "https://example.com" },+ options: {+ ownerId: owner,+ groupId: gid,+ timesOutAt: new Date(Date.now() + 120_000),+ },+ })),+ gate,+ ),+ );++ let inFlight = 0;+ let maxInFlight = 0;+ let processed = 0;+ const worker = async () => {+ let idleStrikes = 0;+ while (idleStrikes < 10) {+ const job = await queue.getJobToProcess();+ if (!job) {+ idleStrikes++;+ await new Promise(resolve => setTimeout(resolve, 50));+ continue;+ }+ idleStrikes = 0;+ inFlight++;+ maxInFlight = Math.max(maxInFlight, inFlight);+ await new Promise(resolve =>+ setTimeout(resolve, 5 + Math.random() * 20),+ );+ inFlight--;+ await queue.jobFinish(job.id, job.lock!, null);+ processed++;+ }+ };++ const [results] = await Promise.all([+ Promise.all(producers),+ ...Array.from({ length: 6 }, () => worker()),+ ]);+ expect(results.flat().length).toBe(JOBS);+ expect(processed).toBe(JOBS);+ expect(maxInFlight).toBeLessThanOrEqual(CRAWL_LIMIT);++ // the group drained, so it must have completed and emitted exactly one+ // crawl_finished job even under concurrent finishers+ const g = await group.getGroup(gid);+ expect(g?.status).toBe("completed");+ const fin = await finishedQueue.getJobToProcess();+ expect(fin).not.toBeNull();+ expect(await finishedQueue.getJobToProcess()).toBeNull();++ const stats = await queue.getGroupNumericStats(gid);+ expect(stats.completed).toBe(JOBS);+ expect(stats.active).toBe(0);+ expect(stats.queued).toBe(0);+ expect(stats.backlog).toBe(0);+ }, 120_000);+});diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex ac6c4bacc..ca83f0f68 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -9,6 +9,12 @@ const delimitedList = (separator = ",") => {});};+const emptyStringAsUndefined = <T extends z.ZodTypeAny>(schema: T) =>+ z.preprocess(value => (value === "" ? undefined : value), schema.optional());++const emptyStringAsDefault = <T extends z.ZodTypeAny>(schema: T) =>+ z.preprocess(value => (value === "" ? undefined : value), schema);+// Ethereum address schema: validates 0x followed by 40 hex charactersconst ethereumAddress = z.string()@@ -96,6 +102,19 @@ const configSchema = z.object({NUQ_DATABASE_URL: z.string().optional(),NUQ_DATABASE_URL_LISTEN: z.string().optional(),NUQ_RABBITMQ_URL: z.string().optional(),+ FDB_CLUSTER_FILE: emptyStringAsUndefined(z.string()),+ NUQ_BACKEND: emptyStringAsUndefined(z.enum(["pg", "fdb"])),+ NUQ_FDB_READY_SHARDS: emptyStringAsDefault(+ z.coerce.number().int().positive().default(2048),+ ),+ // 1 = strict (priority, FIFO) promotion order per team; raise for teams with+ // extreme finish rates at the cost of approximate cross-shard ordering+ NUQ_FDB_TEAM_PENDING_SHARDS: emptyStringAsDefault(+ z.coerce.number().int().positive().default(1),+ ),+ NUQ_FDB_TIME_BUCKETS: emptyStringAsDefault(+ z.coerce.number().int().positive().default(16),+ ),// Google Cloud StorageGCS_BUCKET_NAME: z.string().optional(),diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.tsindex 86d116c29..0e43c324b 100644--- a/apps/api/src/controllers/auth.ts+++ b/apps/api/src/controllers/auth.ts@@ -1,6 +1,6 @@import * as crypto from "crypto";import { RateLimiterRedis } from "rate-limiter-flexible";-import { validate } from "uuid";+import { isValidUuid } from "../lib/owner-id";import { config } from "../config";import { logger } from "../lib/logger";import { parseApi } from "../lib/parseApi";@@ -23,7 +23,7 @@ import { AuthCreditUsageChunk, AuthCreditUsageChunkFromTeam } from "./v1/types";function normalizedApiIsUuid(potentialUuid: string): boolean {// Check if the string is a valid UUID- return validate(potentialUuid);+ return isValidUuid(potentialUuid);}export async function setCachedACUC(diff --git a/apps/api/src/controllers/v0/admin/cclog.ts b/apps/api/src/controllers/v0/admin/cclog.tsindex d77159a34..a2303bb51 100644--- a/apps/api/src/controllers/v0/admin/cclog.ts+++ b/apps/api/src/controllers/v0/admin/cclog.ts@@ -3,15 +3,19 @@ import { db } from "../../../db/connection";import * as schema from "../../../db/schema";import { logger as _logger } from "../../../lib/logger";import { Request, Response } from "express";+import { scrapeQueueFdb } from "../../../services/worker/nuq-fdb";+import { fdbQueueEnabled } from "../../../services/worker/nuq-router";async function cclog() {const logger = _logger.child({module: "cclog",});+ const concurrencyByTeam = new Map<string, number>();+ const redis = getRedisConnection();let cursor = 0;do {- const result = await getRedisConnection().scan(+ const result = await redis.scan(cursor,"MATCH","concurrency-limiter:*",@@ -24,41 +28,47 @@ async function cclog() {logger.info("Stepped", { cursor, usable: usable.length });if (usable.length > 0) {- const entries: {- team_id: string;- concurrency: number;- created_at: Date;- }[] = [];-for (const x of usable) {- const at = new Date();- const concurrency = await getRedisConnection().zrangebyscore(- x,- Date.now(),- Infinity,+ const concurrency = await redis.zrangebyscore(x, Date.now(), Infinity);+ const teamId = x.split(":")[1];+ concurrencyByTeam.set(+ teamId,+ (concurrencyByTeam.get(teamId) ?? 0) + concurrency.length,);- if (concurrency) {- entries.push({- team_id: x.split(":")[1],- concurrency: concurrency.length,- created_at: at,- });- }}+ }+ } while (cursor != 0);- try {- await db.insert(schema.concurrency_log).values(- entries.map(e => ({- team_id: e.team_id,- concurrency: e.concurrency,- created_at: e.created_at.toISOString(),- })),+ if (fdbQueueEnabled()) {+ try {+ const fdbCounts = await scrapeQueueFdb.getTeamActiveCounts();+ for (const [teamId, concurrency] of fdbCounts) {+ concurrencyByTeam.set(+ teamId,+ (concurrencyByTeam.get(teamId) ?? 0) + concurrency,);- } catch (e) {- logger.error("Error inserting", { error: e });}+ } catch (e) {+ logger.warn("Error reading FDB concurrency", { error: e });}- } while (cursor != 0);+ }++ const at = new Date();+ const entries = Array.from(concurrencyByTeam.entries()).map(+ ([team_id, concurrency]) => ({+ team_id,+ concurrency,+ created_at: at.toISOString(),+ }),+ );++ if (entries.length === 0) return;++ try {+ await db.insert(schema.concurrency_log).values(entries);+ } catch (e) {+ logger.error("Error inserting", { error: e });+ }}export async function cclogController(req: Request, res: Response) {diff --git a/apps/api/src/controllers/v0/admin/crawl-monitor.ts b/apps/api/src/controllers/v0/admin/crawl-monitor.tsindex c2a6d6e28..a1a9c4682 100644--- a/apps/api/src/controllers/v0/admin/crawl-monitor.ts+++ b/apps/api/src/controllers/v0/admin/crawl-monitor.ts@@ -15,7 +15,10 @@ import {StoredCrawl,} from "../../../lib/crawl-redis";import { config } from "../../../config";-import { crawlGroup } from "../../../services/worker/nuq";+import {+ crawlGroup,+ resolveNewGroupBackend,+} from "../../../services/worker/nuq-router";import { _addScrapeJobToBullMQ } from "../../../services/queue-jobs";type ResponseType = {@@ -77,10 +80,16 @@ export async function crawlMonitorController(});}+ sc.queueBackend = await resolveNewGroupBackend(sc.team_id);await crawlGroup.addGroup(id,sc.team_id,(req.acuc?.flags?.crawlTtlHours ?? 24) * 60 * 60 * 1000,+ {+ backend: sc.queueBackend,+ maxConcurrency: sc.maxConcurrency,+ delaySeconds: sc.crawlerOptions?.delay,+ },);await saveCrawl(id, sc);diff --git a/apps/api/src/controllers/v0/admin/metrics.ts b/apps/api/src/controllers/v0/admin/metrics.tsindex b64a839f0..3866a6a5c 100644--- a/apps/api/src/controllers/v0/admin/metrics.ts+++ b/apps/api/src/controllers/v0/admin/metrics.ts@@ -1,6 +1,7 @@import type { Request, Response } from "express";import { getRedisConnection } from "../../../services/queue-service";-import { nuqGetLocalMetrics, scrapeQueue } from "../../../services/worker/nuq";+import { nuqGetLocalMetrics } from "../../../services/worker/nuq";+import { scrapeQueue } from "../../../services/worker/nuq-router";import { teamConcurrencySemaphore } from "../../../services/worker/team-semaphore";export async function metricsController(_: Request, res: Response) {diff --git a/apps/api/src/controllers/v0/crawl-cancel.ts b/apps/api/src/controllers/v0/crawl-cancel.tsindex b091009f1..49617bbf6 100644--- a/apps/api/src/controllers/v0/crawl-cancel.ts+++ b/apps/api/src/controllers/v0/crawl-cancel.ts@@ -6,7 +6,7 @@ import { getCrawl, saveCrawl } from "../../../src/lib/crawl-redis";import * as Sentry from "@sentry/node";import { configDotenv } from "dotenv";import { redisEvictConnection } from "../../../src/services/redis";-import { crawlGroup } from "../../services/worker/nuq";+import { crawlGroup } from "../../services/worker/nuq-router";import { getScrapeZDR } from "../../lib/zdr-helpers";import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";configDotenv();@@ -73,6 +73,10 @@ export async function crawlCancelController(req: Request, res: Response) {logger.error(error);}+ if (sc.queueBackend === "fdb") {+ await crawlGroup.cancelGroup(req.params.jobId);+ }+res.json({status: "cancelled",});diff --git a/apps/api/src/controllers/v0/crawl-status.ts b/apps/api/src/controllers/v0/crawl-status.tsindex 0ce22a4af..f63525bf2 100644--- a/apps/api/src/controllers/v0/crawl-status.ts+++ b/apps/api/src/controllers/v0/crawl-status.ts@@ -11,7 +11,7 @@ import { configDotenv } from "dotenv";import { toLegacyDocument } from "../v1/types";import type { DBScrape, PseudoJob } from "../v1/crawl-status";import { getJobFromGCS } from "../../lib/gcs-jobs";-import { scrapeQueue, NuQJob } from "../../services/worker/nuq";+import { scrapeQueue, NuQJob } from "../../services/worker/nuq-router";import { includesFormat } from "../../lib/format-utils";import { getScrapeZDR } from "../../lib/zdr-helpers";import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.tsindex f532c135b..c75a3ac1c 100644--- a/apps/api/src/controllers/v0/crawl.ts+++ b/apps/api/src/controllers/v0/crawl.ts@@ -33,7 +33,10 @@ import { ZodError } from "zod";import { UNSUPPORTED_SITE_MESSAGE } from "../../lib/strings";import { fromV0ScrapeOptions } from "../v2/types";import { isSelfHosted } from "../../lib/deployment";-import { crawlGroup } from "../../services/worker/nuq";+import {+ crawlGroup,+ resolveNewGroupBackend,+} from "../../services/worker/nuq-router";import { logRequest } from "../../services/logging/log_job";import { getScrapeZDR } from "../../lib/zdr-helpers";import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";@@ -225,10 +228,16 @@ export async function crawlController(req: Request, res: Response) {sc.robots = await crawler.getRobotsTxt();} catch (_) {}+ sc.queueBackend = await resolveNewGroupBackend(sc.team_id);await crawlGroup.addGroup(id,sc.team_id,(chunk?.flags?.crawlTtlHours ?? 24) * 60 * 60 * 1000,+ {+ backend: sc.queueBackend,+ maxConcurrency: sc.maxConcurrency,+ delaySeconds: sc.crawlerOptions?.delay,+ },);await saveCrawl(id, sc);diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.tsindex e4fd44656..cb0b88a09 100644--- a/apps/api/src/controllers/v0/scrape.ts+++ b/apps/api/src/controllers/v0/scrape.ts@@ -22,7 +22,7 @@ import { Document as V0Document } from "./../../lib/entities";import { UNSUPPORTED_SITE_MESSAGE } from "../../lib/strings";import { fromV0Combo } from "../v2/types";import { ScrapeJobTimeoutError } from "../../lib/error";-import { scrapeQueue } from "../../services/worker/nuq";+import { scrapeQueue } from "../../services/worker/nuq-router";import { getErrorContactMessage } from "../../lib/deployment";import { logRequest } from "../../services/logging/log_job";import { getScrapeZDR } from "../../lib/zdr-helpers";diff --git a/apps/api/src/controllers/v0/search.ts b/apps/api/src/controllers/v0/search.tsindex f0b995dbc..105dafe77 100644--- a/apps/api/src/controllers/v0/search.ts+++ b/apps/api/src/controllers/v0/search.ts@@ -22,7 +22,7 @@ import {} from "../v1/types";import { fromV0Combo } from "../v2/types";import { ScrapeJobTimeoutError } from "../../lib/error";-import { scrapeQueue } from "../../services/worker/nuq";+import { scrapeQueue } from "../../services/worker/nuq-router";import { defaultOrigin } from "../../lib/default-values";import { getSearchZDR } from "../../lib/zdr-helpers";import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.tsindex d5bf86d37..39b95f717 100644--- a/apps/api/src/controllers/v1/batch-scrape.ts+++ b/apps/api/src/controllers/v1/batch-scrape.ts@@ -26,7 +26,10 @@ import { UNSUPPORTED_SITE_MESSAGE } from "../../lib/strings";import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";import { fromV1ScrapeOptions } from "../v2/types";import { checkPermissions } from "../../lib/permissions";-import { crawlGroup } from "../../services/worker/nuq";+import {+ crawlGroup,+ resolveNewGroupBackend,+} from "../../services/worker/nuq-router";import { logRequest } from "../../services/logging/log_job";import { getScrapeZDR } from "../../lib/zdr-helpers";@@ -163,10 +166,16 @@ export async function batchScrapeController(};if (!req.body.appendToId) {+ sc.queueBackend = await resolveNewGroupBackend(sc.team_id);await crawlGroup.addGroup(id,sc.team_id,(req.acuc?.flags?.crawlTtlHours ?? 24) * 60 * 60 * 1000,+ {+ backend: sc.queueBackend,+ maxConcurrency: sc.maxConcurrency,+ delaySeconds: sc.crawlerOptions?.delay,+ },);await saveCrawl(id, sc);await markCrawlActive(id);diff --git a/apps/api/src/controllers/v1/concurrency-check.ts b/apps/api/src/controllers/v1/concurrency-check.tsindex b25fedbc9..0824f9e8d 100644--- a/apps/api/src/controllers/v1/concurrency-check.ts+++ b/apps/api/src/controllers/v1/concurrency-check.ts@@ -4,24 +4,18 @@ import {RequestWithAuth,} from "./types";import { Response } from "express";-import { getRedisConnection } from "../../../src/services/queue-service";+import { getCombinedTeamActiveCount } from "../../services/worker/nuq-router";// Basically just middleware and error wrappingexport async function concurrencyCheckController(req: RequestWithAuth<ConcurrencyCheckParams, undefined, undefined>,res: Response<ConcurrencyCheckResponse>,) {- const concurrencyLimiterKey = "concurrency-limiter:" + req.auth.team_id;- const now = Date.now();- const activeJobsOfTeam = await getRedisConnection().zrangebyscore(- concurrencyLimiterKey,- now,- Infinity,- );+ const activeJobsOfTeam = await getCombinedTeamActiveCount(req.auth.team_id);return res.status(200).json({success: true,- concurrency: activeJobsOfTeam.length,+ concurrency: activeJobsOfTeam,maxConcurrency: req.acuc?.concurrency ?? 0,});}diff --git a/apps/api/src/controllers/v1/crawl-cancel.ts b/apps/api/src/controllers/v1/crawl-cancel.tsindex cd783f824..2e1d80c82 100644--- a/apps/api/src/controllers/v1/crawl-cancel.ts+++ b/apps/api/src/controllers/v1/crawl-cancel.ts@@ -4,7 +4,7 @@ import { getCrawl, getCrawlJobs, saveCrawl } from "../../lib/crawl-redis";import * as Sentry from "@sentry/node";import { configDotenv } from "dotenv";import { RequestWithAuth } from "./types";-import { crawlGroup } from "../../services/worker/nuq";+import { crawlGroup } from "../../services/worker/nuq-router";import { removeConcurrencyLimitedJobs } from "../../lib/concurrency-limit";configDotenv();@@ -38,8 +38,12 @@ export async function crawlCancelController(logger.error(error);}- const jobIds = await getCrawlJobs(req.params.jobId);- await removeConcurrencyLimitedJobs(sc.team_id, jobIds);+ if (sc.queueBackend === "fdb") {+ await crawlGroup.cancelGroup(req.params.jobId);+ } else {+ const jobIds = await getCrawlJobs(req.params.jobId);+ await removeConcurrencyLimitedJobs(sc.team_id, jobIds);+ }res.json({status: "cancelled",diff --git a/apps/api/src/controllers/v1/crawl-errors.ts b/apps/api/src/controllers/v1/crawl-errors.tsindex d89fe4ce0..b8ad56762 100644--- a/apps/api/src/controllers/v1/crawl-errors.ts+++ b/apps/api/src/controllers/v1/crawl-errors.ts@@ -14,7 +14,7 @@ import * as schema from "../../db/schema";import { logger as _logger } from "../../lib/logger";import { deserializeTransportableError } from "../../lib/error-serde";import { TransportableError } from "../../lib/error";-import { scrapeQueue } from "../../services/worker/nuq";+import { scrapeQueue } from "../../services/worker/nuq-router";configDotenv();export async function crawlErrorsController(diff --git a/apps/api/src/controllers/v1/crawl-ongoing.ts b/apps/api/src/controllers/v1/crawl-ongoing.tsindex 74f7143be..833ac3160 100644--- a/apps/api/src/controllers/v1/crawl-ongoing.ts+++ b/apps/api/src/controllers/v1/crawl-ongoing.ts@@ -6,7 +6,7 @@ import {} from "./types";import { getCrawl } from "../../lib/crawl-redis";import { configDotenv } from "dotenv";-import { crawlGroup } from "../../services/worker/nuq";+import { crawlGroup } from "../../services/worker/nuq-router";configDotenv();export async function ongoingCrawlsController(diff --git a/apps/api/src/controllers/v1/crawl-status-ws.ts b/apps/api/src/controllers/v1/crawl-status-ws.tsindex ffe8439cf..762c25213 100644--- a/apps/api/src/controllers/v1/crawl-status-ws.ts+++ b/apps/api/src/controllers/v1/crawl-status-ws.ts@@ -19,7 +19,7 @@ import {import { getJobs, PseudoJob } from "./crawl-status";import * as Sentry from "@sentry/node";import { getConcurrencyLimitedJobs } from "../../lib/concurrency-limit";-import { scrapeQueue, NuQJobStatus } from "../../services/worker/nuq";+import { scrapeQueue, NuQJobStatus } from "../../services/worker/nuq-router";import { getErrorContactMessage } from "../../lib/deployment";type ErrorMessage = {diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.tsindex 9bb4cbe58..bf4fcd82c 100644--- a/apps/api/src/controllers/v1/crawl-status.ts+++ b/apps/api/src/controllers/v1/crawl-status.ts@@ -28,7 +28,7 @@ import {NuQJob,NuQJobStatus,crawlGroup,-} from "../../services/worker/nuq";+} from "../../services/worker/nuq-router";import { ScrapeJobSingleUrls } from "../../types";configDotenv();diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.tsindex a2aea48b2..92320a234 100644--- a/apps/api/src/controllers/v1/crawl.ts+++ b/apps/api/src/controllers/v1/crawl.ts@@ -18,7 +18,10 @@ import { _addScrapeJobToBullMQ } from "../../services/queue-jobs";import { logger as _logger } from "../../lib/logger";import { fromV1ScrapeOptions } from "../v2/types";import { checkPermissions } from "../../lib/permissions";-import { crawlGroup } from "../../services/worker/nuq";+import {+ crawlGroup,+ resolveNewGroupBackend,+} from "../../services/worker/nuq-router";import { logRequest } from "../../services/logging/log_job";import { getScrapeZDR } from "../../lib/zdr-helpers";@@ -156,10 +159,16 @@ export async function crawlController(});}+ sc.queueBackend = await resolveNewGroupBackend(sc.team_id);await crawlGroup.addGroup(id,sc.team_id,(req.acuc?.flags?.crawlTtlHours ?? 24) * 60 * 60 * 1000,+ {+ backend: sc.queueBackend,+ maxConcurrency: sc.maxConcurrency,+ delaySeconds: sc.crawlerOptions?.delay,+ },);await saveCrawl(id, sc);diff --git a/apps/api/src/controllers/v1/queue-status.ts b/apps/api/src/controllers/v1/queue-status.tsindex dcec60b99..e86f3202c 100644--- a/apps/api/src/controllers/v1/queue-status.ts+++ b/apps/api/src/controllers/v1/queue-status.ts@@ -3,6 +3,9 @@ import { getACUCTeam } from "../auth";import { AuthCreditUsageChunkFromTeam, RequestWithAuth } from "./types";import { Response } from "express";import { getRedisConnection } from "../../services/queue-service";+import { fdbQueueEnabled } from "../../services/worker/nuq-router";+import { scrapeQueueFdb } from "../../services/worker/nuq-fdb";+import { logger } from "../../lib/logger";import {cleanOldConcurrencyLimitedJobs,cleanOldConcurrencyLimitEntries,@@ -41,11 +44,29 @@ export async function queueStatusController(}await cleanOldConcurrencyLimitEntries(req.auth.team_id);- const activeJobsOfTeam = await getConcurrencyLimitActiveJobsCount(+ let activeJobsOfTeam = await getConcurrencyLimitActiveJobsCount(req.auth.team_id,);await cleanOldConcurrencyLimitedJobs(req.auth.team_id);- const queuedJobsOfTeam = await getConcurrencyQueueJobsCount(req.auth.team_id);+ let queuedJobsOfTeam = await getConcurrencyQueueJobsCount(req.auth.team_id);++ // during the FDB migration a team can have load on both ledgers+ if (fdbQueueEnabled()) {+ try {+ const [fdbActive, fdbPending] = await Promise.all([+ scrapeQueueFdb.getTeamActiveCount(req.auth.team_id),+ scrapeQueueFdb.getTeamPendingCount(req.auth.team_id),+ ]);+ activeJobsOfTeam += fdbActive;+ queuedJobsOfTeam += fdbPending;+ } catch (error) {+ logger.warn("Failed to read FDB queue counts, falling back to Redis", {+ module: "queue-status",+ version: "v1",+ error,+ });+ }+ }const mostRecentSuccess = await getRedisConnection().get("most-recent-success:" + req.auth.team_id,diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex ff1de3ac8..4b4bd5c9a 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -1311,6 +1311,8 @@ export type TeamFlags = {searchFeedbackOptOut?: boolean;researchBeta?: boolean;highlightsBeta?: boolean;+ // routes the team's new queue work to the FoundationDB backend+ nuqFdb?: boolean;} | null;export type AuthCreditUsageChunkFromTeam = Omit<diff --git a/apps/api/src/controllers/v1/x402-search.ts b/apps/api/src/controllers/v1/x402-search.tsindex 606f2254f..cd8d6db79 100644--- a/apps/api/src/controllers/v1/x402-search.ts+++ b/apps/api/src/controllers/v1/x402-search.ts@@ -23,7 +23,7 @@ import { db } from "../../db/connection";import * as schema from "../../db/schema";import { fromV1ScrapeOptions } from "../v2/types";import { ScrapeJobTimeoutError } from "../../lib/error";-import { scrapeQueue } from "../../services/worker/nuq";+import { scrapeQueue } from "../../services/worker/nuq-router";import {applyZdrScope,captureExceptionWithZdrCheck,diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.tsindex 79351421f..440054e45 100644--- a/apps/api/src/controllers/v2/batch-scrape.ts+++ b/apps/api/src/controllers/v2/batch-scrape.ts@@ -26,7 +26,10 @@ import { logger as _logger } from "../../lib/logger";import { UNSUPPORTED_SITE_MESSAGE } from "../../lib/strings";import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";import { checkPermissions } from "../../lib/permissions";-import { crawlGroup } from "../../services/worker/nuq";+import {+ crawlGroup,+ resolveNewGroupBackend,+} from "../../services/worker/nuq-router";import { logRequest } from "../../services/logging/log_job";import type { BillingMetadata } from "../../services/billing/types";import { getScrapeZDR } from "../../lib/zdr-helpers";@@ -51,7 +54,8 @@ export async function batchScrapeController(}const zeroDataRetention =- getScrapeZDR(req.acuc?.flags) === "forced" || (req.body.zeroDataRetention ?? false);+ getScrapeZDR(req.acuc?.flags) === "forced" ||+ (req.body.zeroDataRetention ?? false);if (req.body.__agentInterop &&@@ -186,10 +190,16 @@ export async function batchScrapeController(}if (!req.body.appendToId) {+ sc.queueBackend = await resolveNewGroupBackend(sc.team_id);await crawlGroup.addGroup(id,sc.team_id,(req.acuc?.flags?.crawlTtlHours ?? 24) * 60 * 60 * 1000,+ {+ backend: sc.queueBackend,+ maxConcurrency: sc.maxConcurrency,+ delaySeconds: sc.crawlerOptions?.delay,+ },);await saveCrawl(id, sc);await markCrawlActive(id);diff --git a/apps/api/src/controllers/v2/browser.ts b/apps/api/src/controllers/v2/browser.tsindex 256ebf22f..ae15531f5 100644--- a/apps/api/src/controllers/v2/browser.ts+++ b/apps/api/src/controllers/v2/browser.ts@@ -18,10 +18,10 @@ import {clearBrowserSessionPromptFlag,} from "../../lib/browser-sessions";import {- getConcurrencyLimitActiveJobsCount,- pushConcurrencyLimitActiveJob,- removeConcurrencyLimitActiveJob,-} from "../../lib/concurrency-limit";+ getCombinedTeamActiveCount,+ mirrorExternalSlotAcquire,+ mirrorExternalSlotRelease,+} from "../../services/worker/nuq-router";import { RequestWithAuth } from "./types";import { billTeam } from "../../services/billing/credit_billing";import { enqueueBrowserSessionActivity } from "../../lib/browser-session-activity";@@ -245,9 +245,7 @@ export async function browserCreateController(// 0b. Enforce concurrency limit (shared pool with scrape/crawl/interact)const concurrencyLimit = req.acuc?.concurrency ?? 2;- const activeCount = await getConcurrencyLimitActiveJobsCount(- req.auth.team_id,- );+ const activeCount = await getCombinedTeamActiveCount(req.auth.team_id);if (activeCount >= concurrencyLimit) {logger.warn("Concurrency limit reached for browser session", {activeCount,@@ -374,7 +372,7 @@ export async function browserCreateController(// Register in the shared concurrency limiter so this session counts// against the team's concurrent job limit while it's active.- pushConcurrencyLimitActiveJob(req.auth.team_id, sessionId, ttl * 1000).catch(+ mirrorExternalSlotAcquire(req.auth.team_id, sessionId, ttl * 1000).catch(() => {},);@@ -554,7 +552,7 @@ export async function browserDeleteController(// Invalidate cached count so next check reflects the destroyed sessioninvalidateActiveBrowserSessionCount(session.team_id).catch(() => {});- removeConcurrencyLimitActiveJob(session.team_id, session.id).catch(error => {+ mirrorExternalSlotRelease(session.team_id, session.id).catch(error => {logger.error("Failed to remove concurrency limiter entry for browser session",{@@ -701,7 +699,7 @@ export async function browserWebhookDestroyedController(const claimed = await claimBrowserSessionDestroyed(session.id);invalidateActiveBrowserSessionCount(session.team_id).catch(() => {});- removeConcurrencyLimitActiveJob(session.team_id, session.id).catch(error => {+ mirrorExternalSlotRelease(session.team_id, session.id).catch(error => {logger.error("Failed to remove concurrency limiter entry for browser session via webhook",{diff --git a/apps/api/src/controllers/v2/concurrency-check.ts b/apps/api/src/controllers/v2/concurrency-check.tsindex 4431ac41f..36017e8e9 100644--- a/apps/api/src/controllers/v2/concurrency-check.ts+++ b/apps/api/src/controllers/v2/concurrency-check.ts@@ -5,9 +5,9 @@ import {} from "./types";import { AuthCreditUsageChunkFromTeam } from "../v1/types";import { Response } from "express";-import { getRedisConnection } from "../../../src/services/queue-service";import { getACUCTeam } from "../auth";import { RateLimiterMode } from "../../types";+import { getCombinedTeamActiveCount } from "../../services/worker/nuq-router";// Basically just middleware and error wrappingexport async function concurrencyCheckController(@@ -38,17 +38,11 @@ export async function concurrencyCheckController();}- const concurrencyLimiterKey = "concurrency-limiter:" + req.auth.team_id;- const now = Date.now();- const activeJobsOfTeam = await getRedisConnection().zrangebyscore(- concurrencyLimiterKey,- now,- Infinity,- );+ const activeJobsOfTeam = await getCombinedTeamActiveCount(req.auth.team_id);return res.status(200).json({success: true,- concurrency: activeJobsOfTeam.length,+ concurrency: activeJobsOfTeam,maxConcurrency: Math.max(req.acuc.concurrency, otherACUC?.concurrency ?? 0),});}diff --git a/apps/api/src/controllers/v2/crawl-cancel.ts b/apps/api/src/controllers/v2/crawl-cancel.tsindex 86dfc5123..f3cfe561a 100644--- a/apps/api/src/controllers/v2/crawl-cancel.ts+++ b/apps/api/src/controllers/v2/crawl-cancel.ts@@ -9,7 +9,8 @@ import {import * as Sentry from "@sentry/node";import { configDotenv } from "dotenv";import { RequestWithAuth, scrapeOptions } from "./types";-import { crawlGroup, normalizeOwnerId } from "../../services/worker/nuq";+import { crawlGroup } from "../../services/worker/nuq-router";+import { normalizeOwnerId } from "../../lib/owner-id";import { removeConcurrencyLimitedJobs } from "../../lib/concurrency-limit";configDotenv();@@ -50,8 +51,12 @@ export async function crawlCancelController(logger.error(error);}- const jobIds = await getCrawlJobs(req.params.jobId);- await removeConcurrencyLimitedJobs(sc.team_id, jobIds);+ if (sc.queueBackend === "fdb") {+ await crawlGroup.cancelGroup(req.params.jobId);+ } else {+ const jobIds = await getCrawlJobs(req.params.jobId);+ await removeConcurrencyLimitedJobs(sc.team_id, jobIds);+ }res.json({status: "cancelled",diff --git a/apps/api/src/controllers/v2/crawl-errors.ts b/apps/api/src/controllers/v2/crawl-errors.tsindex d89fe4ce0..b8ad56762 100644--- a/apps/api/src/controllers/v2/crawl-errors.ts+++ b/apps/api/src/controllers/v2/crawl-errors.ts@@ -14,7 +14,7 @@ import * as schema from "../../db/schema";import { logger as _logger } from "../../lib/logger";import { deserializeTransportableError } from "../../lib/error-serde";import { TransportableError } from "../../lib/error";-import { scrapeQueue } from "../../services/worker/nuq";+import { scrapeQueue } from "../../services/worker/nuq-router";configDotenv();export async function crawlErrorsController(diff --git a/apps/api/src/controllers/v2/crawl-ongoing.ts b/apps/api/src/controllers/v2/crawl-ongoing.tsindex a27771da3..d22e45ecd 100644--- a/apps/api/src/controllers/v2/crawl-ongoing.ts+++ b/apps/api/src/controllers/v2/crawl-ongoing.ts@@ -6,7 +6,7 @@ import {} from "./types";import { getCrawl } from "../../lib/crawl-redis";import { configDotenv } from "dotenv";-import { crawlGroup } from "../../services/worker/nuq";+import { crawlGroup } from "../../services/worker/nuq-router";configDotenv();export async function ongoingCrawlsController(diff --git a/apps/api/src/controllers/v2/crawl-status-ws.ts b/apps/api/src/controllers/v2/crawl-status-ws.tsindex f5c4113d6..b44137223 100644--- a/apps/api/src/controllers/v2/crawl-status-ws.ts+++ b/apps/api/src/controllers/v2/crawl-status-ws.ts@@ -20,7 +20,7 @@ import {import { getJobs, PseudoJob } from "./crawl-status";import * as Sentry from "@sentry/node";import { getConcurrencyLimitedJobs } from "../../lib/concurrency-limit";-import { scrapeQueue, NuQJobStatus } from "../../services/worker/nuq";+import { scrapeQueue, NuQJobStatus } from "../../services/worker/nuq-router";import { getErrorContactMessage } from "../../lib/deployment";type ErrorMessage = {diff --git a/apps/api/src/controllers/v2/crawl-status.ts b/apps/api/src/controllers/v2/crawl-status.tsindex ba8b6f677..cae21d208 100644--- a/apps/api/src/controllers/v2/crawl-status.ts+++ b/apps/api/src/controllers/v2/crawl-status.ts@@ -29,7 +29,7 @@ import {NuQJob,NuQJobStatus,crawlGroup,-} from "../../services/worker/nuq";+} from "../../services/worker/nuq-router";import { ScrapeJobSingleUrls } from "../../types";import { redisEvictConnection } from "../../../src/services/redis";import { isBaseDomain, extractBaseDomain } from "../../lib/url-utils";diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.tsindex 91de703ed..79d5c1497 100644--- a/apps/api/src/controllers/v2/crawl.ts+++ b/apps/api/src/controllers/v2/crawl.ts@@ -20,7 +20,10 @@ import { generateCrawlerOptionsFromPrompt } from "../../scraper/scrapeURL/transfimport { CostTracking } from "../../lib/cost-tracking";import { checkPermissions } from "../../lib/permissions";import { buildPromptWithWebsiteStructure } from "../../lib/map-utils";-import { crawlGroup } from "../../services/worker/nuq";+import {+ crawlGroup,+ resolveNewGroupBackend,+} from "../../services/worker/nuq-router";import { logRequest } from "../../services/logging/log_job";import { getScrapeZDR } from "../../lib/zdr-helpers";@@ -215,10 +218,16 @@ export async function crawlController(});}+ sc.queueBackend = await resolveNewGroupBackend(sc.team_id);await crawlGroup.addGroup(id,sc.team_id,(req.acuc?.flags?.crawlTtlHours ?? 24) * 60 * 60 * 1000,+ {+ backend: sc.queueBackend,+ maxConcurrency: sc.maxConcurrency,+ delaySeconds: sc.crawlerOptions?.delay,+ },);await saveCrawl(id, sc);diff --git a/apps/api/src/controllers/v2/queue-status.ts b/apps/api/src/controllers/v2/queue-status.tsindex 1322e544d..ef9d383ec 100644--- a/apps/api/src/controllers/v2/queue-status.ts+++ b/apps/api/src/controllers/v2/queue-status.ts@@ -4,6 +4,9 @@ import { RequestWithAuth } from "./types";import { AuthCreditUsageChunkFromTeam } from "../v1/types";import { Response } from "express";import { getRedisConnection } from "../../services/queue-service";+import { fdbQueueEnabled } from "../../services/worker/nuq-router";+import { scrapeQueueFdb } from "../../services/worker/nuq-fdb";+import { logger } from "../../lib/logger";import {cleanOldConcurrencyLimitedJobs,cleanOldConcurrencyLimitEntries,@@ -42,11 +45,29 @@ export async function queueStatusController(}await cleanOldConcurrencyLimitEntries(req.auth.team_id);- const activeJobsOfTeam = await getConcurrencyLimitActiveJobsCount(+ let activeJobsOfTeam = await getConcurrencyLimitActiveJobsCount(req.auth.team_id,);await cleanOldConcurrencyLimitedJobs(req.auth.team_id);- const queuedJobsOfTeam = await getConcurrencyQueueJobsCount(req.auth.team_id);+ let queuedJobsOfTeam = await getConcurrencyQueueJobsCount(req.auth.team_id);++ // during the FDB migration a team can have load on both ledgers+ if (fdbQueueEnabled()) {+ try {+ const [fdbActive, fdbPending] = await Promise.all([+ scrapeQueueFdb.getTeamActiveCount(req.auth.team_id),+ scrapeQueueFdb.getTeamPendingCount(req.auth.team_id),+ ]);+ activeJobsOfTeam += fdbActive;+ queuedJobsOfTeam += fdbPending;+ } catch (error) {+ logger.warn("Failed to read FDB queue counts, falling back to Redis", {+ module: "queue-status",+ version: "v2",+ error,+ });+ }+ }const mostRecentSuccess = await getRedisConnection().get("most-recent-success:" + req.auth.team_id,diff --git a/apps/api/src/controllers/v2/scrape-browser.ts b/apps/api/src/controllers/v2/scrape-browser.tsindex fc8e11cfc..08f46feb4 100644--- a/apps/api/src/controllers/v2/scrape-browser.ts+++ b/apps/api/src/controllers/v2/scrape-browser.ts@@ -17,11 +17,12 @@ import {didBrowserSessionUsePrompt,clearBrowserSessionPromptFlag,} from "../../lib/browser-sessions";+import {} from "../../lib/concurrency-limit";import {- getConcurrencyLimitActiveJobsCount,- pushConcurrencyLimitActiveJob,- removeConcurrencyLimitActiveJob,-} from "../../lib/concurrency-limit";+ getCombinedTeamActiveCount,+ mirrorExternalSlotAcquire,+ mirrorExternalSlotRelease,+} from "../../services/worker/nuq-router";import {browserServiceRequest,BrowserServiceError,@@ -402,7 +403,7 @@ export async function scrapeStopInteractiveBrowserController(const claimed = await claimBrowserSessionDestroyed(session.id);invalidateActiveBrowserSessionCount(session.team_id).catch(() => {});- removeConcurrencyLimitActiveJob(session.team_id, session.id).catch(error => {+ mirrorExternalSlotRelease(session.team_id, session.id).catch(error => {logger.error("Failed to remove concurrency limiter entry for browser session",{@@ -533,9 +534,7 @@ async function createSessionForScrape(// Active session limit — uses the same concurrency pool as scrape/crawlconst concurrencyLimit = req.acuc?.concurrency ?? 2;- const activeCount = await getConcurrencyLimitActiveJobsCount(- req.auth.team_id,- );+ const activeCount = await getCombinedTeamActiveCount(req.auth.team_id);if (activeCount >= concurrencyLimit) {return {status: 429,@@ -738,11 +737,9 @@ async function createSessionForScrape(// Register in the shared concurrency limiter so this session counts// against the team's concurrent job limit while it's active.- pushConcurrencyLimitActiveJob(- req.auth.team_id,- sessionId,- ttl * 1000,- ).catch(() => {});+ mirrorExternalSlotAcquire(req.auth.team_id, sessionId, ttl * 1000).catch(+ () => {},+ );return { session };} catch (err) {diff --git a/apps/api/src/controllers/v2/x402-search.ts b/apps/api/src/controllers/v2/x402-search.tsindex 1f6114373..493ee9736 100644--- a/apps/api/src/controllers/v2/x402-search.ts+++ b/apps/api/src/controllers/v2/x402-search.ts@@ -23,7 +23,7 @@ import { db } from "../../db/connection";import * as schema from "../../db/schema";import { SearchV2Response } from "../../lib/entities";import { ScrapeJobTimeoutError } from "../../lib/error";-import { scrapeQueue } from "../../services/worker/nuq";+import { scrapeQueue } from "../../services/worker/nuq-router";import { z } from "zod";import {buildSearchQuery,diff --git a/apps/api/src/harness.ts b/apps/api/src/harness.tsindex 51e5c1822..f5a778f29 100644--- a/apps/api/src/harness.ts+++ b/apps/api/src/harness.ts@@ -19,6 +19,10 @@ let nuqRabbitMQContainer: {containerName: string;containerRuntime: string;} | null = null;+let fdbContainer: {+ containerName: string;+ containerRuntime: string;+} | null = null;// Get the monorepo root for both tsx source execution and compiled dist execution.// __dirname is available in CommonJS (which this compiles to)@@ -53,6 +57,10 @@ interface Services {containerName: string;containerRuntime: string;};+ fdb?: {+ containerName: string;+ containerRuntime: string;+ };}const colors = {@@ -675,6 +683,96 @@ async function setupNuqRabbitMQ(): Promise<Services["nuqRabbitMQ"]> {return containerInfo;}+const FDB_IMAGE = "foundationdb/foundationdb:7.3.63";++async function waitForFdb(+ runtime: string,+ containerName: string,+ timeoutMs: number = config.HARNESS_STARTUP_TIMEOUT_MS,+): Promise<void> {+ logger.info("Waiting for FoundationDB to be ready...");+ const start = Date.now();++ while (Date.now() - start < timeoutMs) {+ try {+ const check = execForward(+ `${runtime}@fdb-status`,+ `${runtime} exec ${containerName} fdbcli --exec "status minimal"`,+ );+ await check.promise;+ logger.success("FoundationDB is ready");+ return;+ } catch (e) {+ await new Promise(resolve => setTimeout(resolve, 500));+ }+ }++ throw new Error(`FoundationDB did not become ready within ${timeoutMs}ms`);+}++async function setupFdb(): Promise<Services["fdb"]> {+ // FDB is only needed when the FDB queue backend is in play+ if (config.NUQ_BACKEND !== "fdb" && !process.env.NUQ_FDB_TESTS) {+ return undefined;+ }++ // If FDB_CLUSTER_FILE is already set, respect it (user's explicit choice)+ if (config.FDB_CLUSTER_FILE) {+ logger.info("FDB_CLUSTER_FILE is set, skipping container management");+ return undefined;+ }++ logger.section("Setting up FoundationDB container");++ const runtime = await detectContainerRuntime();+ if (!runtime) {+ throw new Error(+ "Neither Docker nor Podman found. Please install Docker/Podman or set FDB_CLUSTER_FILE manually.",+ );+ }++ logger.success(`Using container runtime: ${runtime}`);++ const containerName = "firecrawl-fdb";+ await stopAndRemoveContainer(runtime, containerName);++ const start = execForward(+ `${runtime}@start`,+ `${runtime} run -d --name ${containerName} -p 4500:4500 -e FDB_NETWORKING_MODE=host ${FDB_IMAGE}`,+ );+ await start.promise;++ await waitForFdb(runtime, containerName);++ // first boot: create the database; only suppress the known "already+ // configured" failure and surface any real configure error.+ const configure = execForward(+ `${runtime}@fdb-configure`,+ `${runtime} exec ${containerName} sh -c 'out=$(fdbcli --exec "configure new single memory" 2>&1); status=$?; printf "%s\\n" "$out"; if [ "$status" -eq 0 ]; then exit 0; fi; printf "%s\\n" "$out" | grep -Eiq "already.*configured|database.*configured"'`,+ );+ await configure.promise;++ // the host-side client needs a cluster file pointing at the published port+ const { writeFileSync, mkdirSync } = await import("fs");+ const { tmpdir } = await import("os");+ const clusterDir = join(tmpdir(), "firecrawl-fdb");+ mkdirSync(clusterDir, { recursive: true });+ const clusterFile = join(clusterDir, "fdb.cluster");+ writeFileSync(clusterFile, "docker:[email protected]:4500");++ config.FDB_CLUSTER_FILE = clusterFile;+ process.env.FDB_CLUSTER_FILE = clusterFile;++ logger.success("FoundationDB container is ready");++ const containerInfo = {+ containerName,+ containerRuntime: runtime,+ };+ fdbContainer = containerInfo;+ return containerInfo;+}+async function installDependencies() {logger.section("Installing dependencies");@@ -724,6 +822,9 @@ async function startServices(command?: string[]): Promise<Services> {// Setup NUQ RabbitMQ container if neededconst nuqRabbitMQ = await setupNuqRabbitMQ();+ // Setup FoundationDB container if the FDB queue backend is enabled+ const fdb = await setupFdb();+logger.section("Starting services");const api = execForward(@@ -775,30 +876,36 @@ async function startServices(command?: string[]): Promise<Services> {),);- const nuqPrefetchWorker = execForward(- "nuq-prefetch-worker",- process.argv[2] === "--start-docker"- ? "node dist/src/services/worker/nuq-prefetch-worker.js"- : "pnpm nuq-prefetch-worker:production",- {- NUQ_PREFETCH_WORKER_PORT: String(NUQ_PREFETCH_WORKER_PORT),- NUQ_REDUCE_NOISE: "true",- NUQ_POD_NAME: "nuq-prefetch-worker-0",- NUQ_PREFETCH_REPLICAS: String(1),- },- );-- const nuqReconcilerWorker = execForward(- "nuq-reconciler",- process.argv[2] === "--start-docker"- ? "node dist/src/services/worker/nuq-reconciler-worker.js"- : "pnpm nuq-reconciler-worker:production",- {- NUQ_RECONCILER_WORKER_PORT: String(NUQ_RECONCILER_WORKER_PORT),- NUQ_REDUCE_NOISE: "true",- NUQ_POD_NAME: "nuq-reconciler-worker-0",- },- );+ const nuqPrefetchWorker =+ config.NUQ_BACKEND === "fdb"+ ? undefined+ : execForward(+ "nuq-prefetch-worker",+ process.argv[2] === "--start-docker"+ ? "node dist/src/services/worker/nuq-prefetch-worker.js"+ : "pnpm nuq-prefetch-worker:production",+ {+ NUQ_PREFETCH_WORKER_PORT: String(NUQ_PREFETCH_WORKER_PORT),+ NUQ_REDUCE_NOISE: "true",+ NUQ_POD_NAME: "nuq-prefetch-worker-0",+ NUQ_PREFETCH_REPLICAS: String(1),+ },+ );++ const nuqReconcilerWorker =+ config.NUQ_BACKEND === "fdb"+ ? undefined+ : execForward(+ "nuq-reconciler",+ process.argv[2] === "--start-docker"+ ? "node dist/src/services/worker/nuq-reconciler-worker.js"+ : "pnpm nuq-reconciler-worker:production",+ {+ NUQ_RECONCILER_WORKER_PORT: String(NUQ_RECONCILER_WORKER_PORT),+ NUQ_REDUCE_NOISE: "true",+ NUQ_POD_NAME: "nuq-reconciler-worker-0",+ },+ );const indexWorker = config.USE_DB_AUTHENTICATION? execForward(@@ -840,6 +947,7 @@ async function startServices(command?: string[]): Promise<Services> {command: commandProcess,nuqPostgres,nuqRabbitMQ,+ fdb,};}@@ -1060,6 +1168,17 @@ async function gracefulShutdown() {nuqRabbitMQContainer = null;}+ // Stop and remove FoundationDB container if it was started by harness+ if (fdbContainer) {+ logger.info("Stopping FoundationDB container");+ await stopAndRemoveContainer(+ fdbContainer.containerRuntime,+ fdbContainer.containerName,+ );+ logger.success("FoundationDB container stopped");+ fdbContainer = null;+ }+logger.success("All processes terminated");}diff --git a/apps/api/src/lib/concurrency-limit.ts b/apps/api/src/lib/concurrency-limit.tsindex 34922fadf..dd332b478 100644--- a/apps/api/src/lib/concurrency-limit.ts+++ b/apps/api/src/lib/concurrency-limit.ts@@ -6,20 +6,22 @@ import { logger } from "./logger";import { abTestJob } from "../services/ab-test";import { scrapeQueue, type NuQJob } from "../services/worker/nuq";export { QueueFullError } from "./queue-full-error";--// min 50k, max 2M, 2000 per concurrent browser-export function getTeamQueueLimit(concurrencyLimit: number): number {- return Math.min(Math.max(concurrencyLimit * 2000, 50_000), 2_000_000);-}--// Upper bound for how long a job may sit in the concurrency-limit backlog.-// This bounds both the Redis ZSET score and the Postgres `times_out_at`-// column on `nuq.queue_scrape_backlog`, so the reaper can always evict-// stale rows. A backlogged crawl job that outlives this window is-// unrecoverable anyway — its StoredCrawl in Redis (24h TTL) is gone.-export const MAX_BACKLOG_TIMEOUT_MS = 172800000; // 48h--const constructKey = (team_id: string) => "concurrency-limiter:" + team_id;+export {+ getTeamQueueLimit,+ MAX_BACKLOG_TIMEOUT_MS,+ getConcurrencyLimitActiveJobsCount,+ pushConcurrencyLimitActiveJob,+ removeConcurrencyLimitActiveJob,+} from "./concurrency-redis";+import {+ getTeamQueueLimit,+ MAX_BACKLOG_TIMEOUT_MS,+ constructConcurrencyLimitKey,+ pushConcurrencyLimitActiveJob,+ removeConcurrencyLimitActiveJob,+} from "./concurrency-redis";++const constructKey = constructConcurrencyLimitKey;const constructQueueKey = (team_id: string) =>"concurrency-limit-queue:" + team_id;@@ -39,16 +41,6 @@ export async function cleanOldConcurrencyLimitEntries();}-export async function getConcurrencyLimitActiveJobsCount(- team_id: string,-): Promise<number> {- return await getRedisConnection().zcount(- constructKey(team_id),- Date.now(),- Infinity,- );-}-export async function getConcurrencyLimitActiveJobs(team_id: string,now: number = Date.now(),@@ -60,22 +52,6 @@ export async function getConcurrencyLimitActiveJobs();}-export async function pushConcurrencyLimitActiveJob(- team_id: string,- id: string,- timeout: number,- now: number = Date.now(),-) {- await getRedisConnection().zadd(constructKey(team_id), now + timeout, id);-}--export async function removeConcurrencyLimitActiveJob(- team_id: string,- id: string,-) {- await getRedisConnection().zrem(constructKey(team_id), id);-}-export async function removeConcurrencyLimitedJobs(team_id: string,job_ids: string[],diff --git a/apps/api/src/lib/concurrency-redis.ts b/apps/api/src/lib/concurrency-redis.tsnew file mode 100644index 000000000..8b9a419eb--- /dev/null+++ b/apps/api/src/lib/concurrency-redis.ts@@ -0,0 +1,50 @@+import { getRedisConnection } from "../services/queue-service";++// Redis primitives of the concurrency limiter, split out of+// concurrency-limit.ts so light consumers (the NuQ dual-backend router) can+// use them without dragging in the scraper tree via crawl-redis.++// min 50k, max 2M, 2000 per concurrent browser+export function getTeamQueueLimit(concurrencyLimit: number): number {+ return Math.min(Math.max(concurrencyLimit * 2000, 50_000), 2_000_000);+}++// Upper bound for how long a job may sit in the concurrency-limit backlog.+// This bounds both the Redis ZSET score and the Postgres `times_out_at`+// column on `nuq.queue_scrape_backlog`, so the reaper can always evict+// stale rows. A backlogged crawl job that outlives this window is+// unrecoverable anyway — its StoredCrawl in Redis (24h TTL) is gone.+export const MAX_BACKLOG_TIMEOUT_MS = 172800000; // 48h++export const constructConcurrencyLimitKey = (team_id: string) =>+ "concurrency-limiter:" + team_id;++export async function getConcurrencyLimitActiveJobsCount(+ team_id: string,+): Promise<number> {+ return await getRedisConnection().zcount(+ constructConcurrencyLimitKey(team_id),+ Date.now(),+ Infinity,+ );+}++export async function pushConcurrencyLimitActiveJob(+ team_id: string,+ id: string,+ timeout: number,+ now: number = Date.now(),+) {+ await getRedisConnection().zadd(+ constructConcurrencyLimitKey(team_id),+ now + timeout,+ id,+ );+}++export async function removeConcurrencyLimitActiveJob(+ team_id: string,+ id: string,+) {+ await getRedisConnection().zrem(constructConcurrencyLimitKey(team_id), id);+}diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.tsindex eddbb4e59..0780a7034 100644--- a/apps/api/src/lib/crawl-redis.ts+++ b/apps/api/src/lib/crawl-redis.ts@@ -19,6 +19,9 @@ export type StoredCrawl = {createdAt: number;maxConcurrency?: number;zeroDataRetention?: boolean;+ // which queue backend this crawl's group + jobs live on; a crawl never+ // spans backends. Absent on crawls created before the FDB rollout (= pg).+ queueBackend?: "pg" | "fdb";};export async function saveCrawl(id: string, crawl: StoredCrawl) {diff --git a/apps/api/src/lib/extract/document-scraper.ts b/apps/api/src/lib/extract/document-scraper.tsindex dcd053ea8..1da8e2740 100644--- a/apps/api/src/lib/extract/document-scraper.ts+++ b/apps/api/src/lib/extract/document-scraper.ts@@ -12,7 +12,7 @@ import { addScrapeJob } from "../../services/queue-jobs";import { getJobPriority } from "../job-priority";import type { Logger } from "winston";import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";-import { scrapeQueue } from "../../services/worker/nuq";+import { scrapeQueue } from "../../services/worker/nuq-router";interface ScrapeDocumentOptions {url: string;diff --git a/apps/api/src/lib/extract/extraction-service.ts b/apps/api/src/lib/extract/extraction-service.tsindex fab347bba..79261ede3 100644--- a/apps/api/src/lib/extract/extraction-service.ts+++ b/apps/api/src/lib/extract/extraction-service.ts@@ -155,13 +155,11 @@ async function performExtraction(apiKeyId,{ endpoint: "extract", jobId: extractId },logger,- ).catch(- error => {- logger.error(- `Failed to bill team ${teamId} for ${creditsToBill} credits: ${error}`,- );- },- );+ ).catch(error => {+ logger.error(+ `Failed to bill team ${teamId} for ${creditsToBill} credits: ${error}`,+ );+ });return {success: false,@@ -673,13 +671,11 @@ async function performExtraction(apiKeyId,{ endpoint: "extract", jobId: extractId },logger,- ).catch(- error => {- logger.error(- `Failed to bill team ${teamId} for ${creditsToBill} credits: ${error}`,- );- },- );+ ).catch(error => {+ logger.error(+ `Failed to bill team ${teamId} for ${creditsToBill} credits: ${error}`,+ );+ });return {success: false,error: getErrorContactMessage(),@@ -787,13 +783,11 @@ async function performExtraction(apiKeyId,{ endpoint: "extract", jobId: extractId },logger,- ).catch(- error => {- logger.error(- `Failed to bill team ${teamId} for thinking tokens: ${error}`,- );- },- );+ ).catch(error => {+ logger.error(+ `Failed to bill team ${teamId} for thinking tokens: ${error}`,+ );+ });return {success: false,error: error.message,@@ -817,13 +811,11 @@ async function performExtraction(apiKeyId,{ endpoint: "extract", jobId: extractId },logger,- ).catch(- error => {- logger.error(- `Failed to bill team ${teamId} for thinking tokens: ${error}`,- );- },- );+ ).catch(error => {+ logger.error(+ `Failed to bill team ${teamId} for thinking tokens: ${error}`,+ );+ });await logExtract({id: extractId,request_id: extractId,@@ -1011,13 +1003,11 @@ async function performExtraction(apiKeyId,{ endpoint: "extract" },logger,- ).catch(- error => {- logger.error(- `Failed to bill team ${teamId} for ${creditsToBill} credits: ${error}`,- );- },- );+ ).catch(error => {+ logger.error(+ `Failed to bill team ${teamId} for ${creditsToBill} credits: ${error}`,+ );+ });// Log job with token usage and sourceslogger.debug("Logging extract to database", {@@ -1081,13 +1071,11 @@ async function performExtraction(apiKeyId,{ endpoint: "extract" },logger,- ).catch(- error => {- logger.error(- `Failed to bill team ${teamId} for ${creditsToBill} credits: ${error}`,- );- },- );+ ).catch(error => {+ logger.error(+ `Failed to bill team ${teamId} for ${creditsToBill} credits: ${error}`,+ );+ });await logExtract({id: extractId,request_id: extractId,diff --git a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts b/apps/api/src/lib/extract/fire-0/document-scraper-f0.tsindex a888aecf1..0b884f62a 100644--- a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts+++ b/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts@@ -11,7 +11,7 @@ import { addScrapeJob } from "../../../services/queue-jobs";import { getJobPriority } from "../../job-priority";import type { Logger } from "winston";import { isUrlBlocked } from "../../../scraper/WebScraper/utils/blocklist";-import { scrapeQueue } from "../../../services/worker/nuq";+import { scrapeQueue } from "../../../services/worker/nuq-router";interface ScrapeDocumentOptions {url: string;diff --git a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts b/apps/api/src/lib/extract/fire-0/extraction-service-f0.tsindex 64dc894eb..1ac0ddefe 100644--- a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts+++ b/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts@@ -852,7 +852,14 @@ export async function performExtraction_F0(const creditsToBill = Math.ceil(tokensToBill / 15);// Bill team for usage- billTeam(teamId, subId, creditsToBill, apiKeyId, { endpoint: "extract", jobId: extractId }, logger).catch(error => {+ billTeam(+ teamId,+ subId,+ creditsToBill,+ apiKeyId,+ { endpoint: "extract", jobId: extractId },+ logger,+ ).catch(error => {logger.error(`Failed to bill team ${teamId} for ${creditsToBill} credits: ${error}`,);diff --git a/apps/api/src/lib/fire-privacy-chunker.ts b/apps/api/src/lib/fire-privacy-chunker.tsindex 2dbdf8524..a1ae85768 100644--- a/apps/api/src/lib/fire-privacy-chunker.ts+++ b/apps/api/src/lib/fire-privacy-chunker.ts@@ -75,10 +75,14 @@ export function chunkMarkdown(text: string, opts?: ChunkOptions): Chunk[] {// Guard against caller bugs that would otherwise leave the cursor parked// at 0 (maxChars ≤ 0 → tentativeEnd ≤ cursor) and spin forever.if (maxChars <= 0) {- throw new RangeError(`chunkMarkdown: maxChars must be > 0 (got ${maxChars})`);+ throw new RangeError(+ `chunkMarkdown: maxChars must be > 0 (got ${maxChars})`,+ );}if (maxBytes <= 0) {- throw new RangeError(`chunkMarkdown: maxBytes must be > 0 (got ${maxBytes})`);+ throw new RangeError(+ `chunkMarkdown: maxBytes must be > 0 (got ${maxBytes})`,+ );}const chunks: Chunk[] = [];diff --git a/apps/api/src/lib/fire-privacy-client.test.ts b/apps/api/src/lib/fire-privacy-client.test.tsindex 951a19753..44227a577 100644--- a/apps/api/src/lib/fire-privacy-client.test.ts+++ b/apps/api/src/lib/fire-privacy-client.test.ts@@ -272,7 +272,12 @@ describe("redactText", () => {handler = withBody({redacted_text: "x",spans: [- { start: 0, end: 5, kind: "PRIVATE_PERSON", source: "openai-privacy-filter" },+ {+ start: 0,+ end: 5,+ kind: "PRIVATE_PERSON",+ source: "openai-privacy-filter",+ },{ start: 6, end: 10, kind: "EMAIL_ADDRESS", source: "EmailRecognizer" },{ start: 11, end: 15, kind: "ORGANIZATION", source: "SpacyRecognizer" },],diff --git a/apps/api/src/lib/owner-id.ts b/apps/api/src/lib/owner-id.tsnew file mode 100644index 000000000..769a60cfc--- /dev/null+++ b/apps/api/src/lib/owner-id.ts@@ -0,0 +1,33 @@+import { createHash } from "crypto";++// mirrors the uuid package's validate(): RFC 9562 plus the nil/max specials+const UUID_RE =+ /^(?:[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$/i;++// RFC 4122 v5 (namespaced SHA-1), matching the uuid package's v5()+function uuidv5(name: string, namespace: string): string {+ const ns = Buffer.from(namespace.replace(/-/g, ""), "hex");+ const hash = createHash("sha1")+ .update(ns)+ .update(Buffer.from(name, "utf8"))+ .digest();+ const bytes = hash.subarray(0, 16);+ bytes[6] = (bytes[6] & 0x0f) | 0x50;+ bytes[8] = (bytes[8] & 0x3f) | 0x80;+ const hex = bytes.toString("hex");+ return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;+}++export function isValidUuid(value: string): boolean {+ return UUID_RE.test(value);+}++// owner IDs can sometimes be non-UUID, so normalize them to avoid breakage+const normalizedUUIDNamespace = "0f38e00e-d7ee-4b77-8a7a-a787a3537ca2";+export function normalizeOwnerId(+ ownerId: string | undefined | null,+): string | null {+ if (typeof ownerId !== "string") return null;+ if (UUID_RE.test(ownerId)) return ownerId;+ return uuidv5(ownerId, normalizedUUIDNamespace);+}diff --git a/apps/api/src/scraper/WebScraper/__tests__/crawler.test.ts b/apps/api/src/scraper/WebScraper/__tests__/crawler.test.tsindex 0f8a110c2..302396a7e 100644--- a/apps/api/src/scraper/WebScraper/__tests__/crawler.test.ts+++ b/apps/api/src/scraper/WebScraper/__tests__/crawler.test.ts@@ -152,4 +152,32 @@ describe("WebCrawler", () => {filteredLinks.denialReasons.has("https://sub.example.com/blog"),).toBe(true);});++ it("can validate the source URL without applying maxDiscoveryDepth", async () => {+ const initialUrl = "https://example.com/blog";++ crawler = new WebCrawler({+ jobId: "TEST",+ initialUrl,+ includes: ["^/blog"],+ excludes: [],+ limit: 10,+ maxCrawledDepth: 10,+ maxDiscoveryDepth: 0,+ currentDiscoveryDepth: 0,+ });++ const discoveryResult = await crawler["filterLinks"]([initialUrl], 1, 10);+ expect(discoveryResult.links).toEqual([]);++ const sourceValidationResult = await crawler["filterLinks"](+ [initialUrl],+ 1,+ 10,+ false,+ false,+ true,+ );+ expect(sourceValidationResult.links).toEqual([initialUrl]);+ });});diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.tsindex 28105c72e..3eb1b7b89 100644--- a/apps/api/src/scraper/WebScraper/crawler.ts+++ b/apps/api/src/scraper/WebScraper/crawler.ts@@ -160,10 +160,14 @@ export class WebCrawler {maxDepth: number,fromMap: boolean = false,skipRobots: boolean = false,+ ignoreDiscoveryDepth: boolean = false,): Promise<FilterLinksResult> {const denialReasons = new Map<string, string>();- if (this.currentDiscoveryDepth === this.maxDiscoveryDepth) {+ if (+ !ignoreDiscoveryDepth &&+ this.currentDiscoveryDepth === this.maxDiscoveryDepth+ ) {this.logger.debug("Max discovery depth hit, filtering off all links", {currentDiscoveryDepth: this.currentDiscoveryDepth,maxDiscoveryDepth: this.maxDiscoveryDepth,diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 6589f6737..796fe4c87 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -416,19 +416,21 @@ async function buildMetaObject(}}+ const normalizedOptions = {+ ...options,+ skipTlsVerification:+ options.skipTlsVerification ??+ ((options.headers && Object.keys(options.headers).length > 0) ||+ (options.actions && options.actions.length > 0)+ ? false+ : true),+ };+return {id,url,rewrittenUrl: rewriteUrl(url),- options: {- ...options,- skipTlsVerification:- options.skipTlsVerification ??- ((options.headers && Object.keys(options.headers).length > 0) ||- (options.actions && options.actions.length > 0)- ? false- : true),- },+ options: normalizedOptions,internalOptions,logger,abortHandle,@@ -445,7 +447,7 @@ async function buildMetaObject(}: undefined,),- featureFlags: buildFeatureFlags(url, options, internalOptions),+ featureFlags: buildFeatureFlags(url, normalizedOptions, internalOptions),mock:options.useMock !== undefined? await loadMock(options.useMock, _logger)diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.tsindex 3072069d7..19c321d4c 100644--- a/apps/api/src/services/indexing/index-worker.ts+++ b/apps/api/src/services/indexing/index-worker.ts@@ -38,7 +38,7 @@ import {import { StoredCrawl, crawlToCrawler, saveCrawl } from "../../lib/crawl-redis";import { _addScrapeJobToBullMQ } from "../queue-jobs";import { withSpan, setSpanAttributes } from "../../lib/otel-tracer";-import { crawlGroup } from "../worker/nuq";+import { crawlGroup, resolveNewGroupBackend } from "../worker/nuq-router";import { getACUCTeam } from "../../controllers/auth";import { processEngpickerJob } from "../../lib/engpicker";import { logRequest } from "../logging/log_job";@@ -507,6 +507,7 @@ const processPrecrawlJob = async (token: string, job: Job) => {zeroDataRetention: false,};+ sc.queueBackend = await resolveNewGroupBackend(sc.team_id);await crawlGroup.addGroup(crawlId,sc.team_id,@@ -514,6 +515,11 @@ const processPrecrawlJob = async (token: string, job: Job) => {60 *60 *1000,+ {+ backend: sc.queueBackend,+ maxConcurrency: sc.maxConcurrency,+ delaySeconds: sc.crawlerOptions?.delay,+ },);await saveCrawl(crawlId, sc);diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.tsindex d4ba63523..a7e0b2fe6 100644--- a/apps/api/src/services/monitoring/runner.ts+++ b/apps/api/src/services/monitoring/runner.ts@@ -4,7 +4,12 @@ import { logger as _logger } from "../../lib/logger";import { logRequest } from "../logging/log_job";import { getMonitorDiffArtifact } from "../../lib/gcs-monitoring";import { processJobInternal } from "../worker/scrape-worker";-import { NuQJob, crawlGroup, scrapeQueue } from "../worker/nuq";+import {+ NuQJob,+ crawlGroup,+ scrapeQueue,+ resolveNewGroupBackend,+} from "../worker/nuq-router";import { ScrapeJobData } from "../../types";import { getJobFromGCS } from "../../lib/gcs-jobs";import { includesFormat } from "../../lib/format-utils";@@ -505,7 +510,12 @@ async function runCrawlTarget(params: {// Crawls tolerate robots fetch failures in the public controller too.}- await crawlGroup.addGroup(crawlId, sc.team_id, 24 * 60 * 60 * 1000);+ sc.queueBackend = await resolveNewGroupBackend(sc.team_id);+ await crawlGroup.addGroup(crawlId, sc.team_id, 24 * 60 * 60 * 1000, {+ backend: sc.queueBackend,+ maxConcurrency: sc.maxConcurrency,+ delaySeconds: sc.crawlerOptions?.delay,+ });await saveCrawl(crawlId, sc);await markCrawlActive(crawlId);@@ -891,7 +901,12 @@ async function enqueueMonitorCrawlTarget(params: {// Non-fatal, same as the public crawl controller.}- await crawlGroup.addGroup(crawlId, sc.team_id, 24 * 60 * 60 * 1000);+ sc.queueBackend = await resolveNewGroupBackend(sc.team_id);+ await crawlGroup.addGroup(crawlId, sc.team_id, 24 * 60 * 60 * 1000, {+ backend: sc.queueBackend,+ maxConcurrency: sc.maxConcurrency,+ delaySeconds: sc.crawlerOptions?.delay,+ });await saveCrawl(crawlId, sc);await markCrawlActive(crawlId);diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.tsindex d933a021d..0e56ae531 100644--- a/apps/api/src/services/queue-jobs.ts+++ b/apps/api/src/services/queue-jobs.ts@@ -25,6 +25,12 @@ import { ScrapeJobTimeoutError, TransportableError } from "../lib/error";import { deserializeTransportableError } from "../lib/error-serde";import { abTestJob } from "./ab-test";import { NuQJob, scrapeQueue } from "./worker/nuq";+import {+ fdbEnqueueScrapeJobs,+ resolveJobBackend,+ scrapeQueue as routedScrapeQueue,+} from "./worker/nuq-router";+import { scrapeQueueFdb } from "./worker/nuq-fdb";import { serializeTraceContext } from "../lib/otel-tracer";import { isSelfHosted } from "../lib/deployment";import { MONITOR_CHECK_STALE_TIMEOUT_MS } from "./monitoring/stale";@@ -145,6 +151,41 @@ export async function _addScrapeJobToBullMQ(jobId: string,priority: number = 0,listenable: boolean = false,+): Promise<NuQJob<ScrapeJobData>> {+ // direct adds bypass the gates; on the FDB backend that's a slotless enqueue+ if ((await resolveJobBackend(webScraperOptions)) === "fdb") {+ if (webScraperOptions.mode === "single_urls") {+ abTestJob(webScraperOptions);+ }+ const { jobs } = await fdbEnqueueScrapeJobs(+ [+ {+ jobId,+ data: webScraperOptions,+ priority,+ listenable,+ backlogTimeoutMs: backlogTimeoutMs(webScraperOptions),+ },+ ],+ webScraperOptions.team_id,+ { bypassGate: true },+ );+ return jobs[0];+ }++ return _addScrapeJobToBullMQPg(+ webScraperOptions,+ jobId,+ priority,+ listenable,+ );+}++async function _addScrapeJobToBullMQPg(+ webScraperOptions: ScrapeJobData,+ jobId: string,+ priority: number = 0,+ listenable: boolean = false,): Promise<NuQJob<ScrapeJobData>> {if (webScraperOptions.mode === "single_urls") {abTestJob(webScraperOptions);@@ -224,6 +265,76 @@ async function _addScrapeJobsToBullMQ();}+async function addScrapeJobFdb(+ webScraperOptions: ScrapeJobData,+ jobId: string,+ priority: number,+ directToBullMQ: boolean,+ listenable: boolean,+): Promise<NuQJob<ScrapeJobData> | null> {+ if (webScraperOptions.mode === "single_urls") {+ abTestJob(webScraperOptions);+ }++ const { jobs, backloggedCount, teamLimit } = await fdbEnqueueScrapeJobs(+ [+ {+ jobId,+ data: webScraperOptions,+ priority,+ listenable,+ backlogTimeoutMs: backlogTimeoutMs(webScraperOptions),+ },+ ],+ webScraperOptions.team_id,+ { bypassGate: directToBullMQ },+ );++ if (backloggedCount > 0) {+ await maybeSendConcurrencyNotificationFdb(+ webScraperOptions.team_id,+ teamLimit,+ isCrawlOrBatchScrape(webScraperOptions),+ );+ // matches the PG contract: null = job waiting in the concurrency queue+ return null;+ }+ return jobs[0];+}++// parity with the PG path: notify when the backlog exceeds the team limit+async function maybeSendConcurrencyNotificationFdb(+ teamId: string,+ teamLimit: number | null,+ crawlOrBatch: boolean,+) {+ if (teamLimit === null || crawlOrBatch) return;+ try {+ const pending = await scrapeQueueFdb.getTeamPendingCount(teamId);+ if (pending <= teamLimit) return;+ const shouldSendNotification =+ await shouldSendConcurrencyLimitNotification(teamId);+ if (shouldSendNotification) {+ sendNotificationWithCustomDays(+ teamId,+ NotificationType.CONCURRENCY_LIMIT_REACHED,+ 15,+ false,+ true,+ ).catch(error => {+ _logger.error(+ "Error sending notification (concurrency limit reached)",+ {+ error,+ },+ );+ });+ }+ } catch (error) {+ _logger.warn("Failed to check FDB concurrency notification", { error });+ }+}+async function addScrapeJobRaw(webScraperOptions: ScrapeJobData,jobId: string,@@ -231,6 +342,16 @@ async function addScrapeJobRaw(directToBullMQ: boolean = false,listenable: boolean = false,): Promise<NuQJob<ScrapeJobData> | null> {+ if ((await resolveJobBackend(webScraperOptions)) === "fdb") {+ return addScrapeJobFdb(+ webScraperOptions,+ jobId,+ priority,+ directToBullMQ,+ listenable,+ );+ }+let concurrencyLimited: "yes" | "yes-crawl" | "no" | null = null;let currentActiveConcurrency: number | null = null;let maxConcurrency = 0;@@ -357,7 +478,7 @@ async function addScrapeJobRaw();return null;} else {- return await _addScrapeJobToBullMQ(+ return await _addScrapeJobToBullMQPg(webScraperOptions,jobId,priority,@@ -419,7 +540,49 @@ export async function addScrapeJobs(jobsByTeam.get(job.data.team_id)!.push(job);}- for (const [teamId, teamJobs] of jobsByTeam) {+ for (const [teamId, allTeamJobs] of jobsByTeam) {+ // jobs can split across backends mid-migration (old crawls drain on PG+ // while the team's new crawls run on FDB); partition by job backend+ const backendByJob = new Map<string, "pg" | "fdb">();+ const backendByCrawl = new Map<string, "pg" | "fdb">();+ for (const job of allTeamJobs) {+ const crawlId = job.data.crawl_id;+ if (crawlId && backendByCrawl.has(crawlId)) {+ backendByJob.set(job.jobId, backendByCrawl.get(crawlId)!);+ continue;+ }+ const backend = await resolveJobBackend(job.data);+ backendByJob.set(job.jobId, backend);+ if (crawlId) backendByCrawl.set(crawlId, backend);+ }++ const fdbJobs = allTeamJobs.filter(+ j => backendByJob.get(j.jobId) === "fdb",+ );+ if (fdbJobs.length > 0) {+ const { backloggedCount, teamLimit } = await fdbEnqueueScrapeJobs(+ fdbJobs.map(job => ({+ jobId: job.jobId,+ data: { ...job.data, traceContext },+ priority: job.priority,+ listenable: job.listenable,+ backlogTimeoutMs: backlogTimeoutMs(job.data),+ })),+ teamId,+ );+ if (backloggedCount > 0) {+ await maybeSendConcurrencyNotificationFdb(+ teamId,+ teamLimit,+ isCrawlOrBatchScrape(fdbJobs[0].data),+ );+ }+ }++ const teamJobs = allTeamJobs.filter(+ j => backendByJob.get(j.jobId) === "pg",+ );+ if (teamJobs.length === 0) continue;// == Buckets for jobs ==let jobsForcedToCQ: {data: ScrapeJobData;@@ -650,7 +813,7 @@ export async function waitForJob(try {doc = await Promise.race([- scrapeQueue.waitForJob(+ routedScrapeQueue.waitForJob<Document>(jobId,timeout !== null ? timeout + 100 : null,logger,diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex a96a70961..cdce794cc 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -21,7 +21,7 @@ import Express from "express";import { robustFetch } from "../scraper/scrapeURL/lib/fetch";import { initializeBlocklist } from "../scraper/WebScraper/utils/blocklist";import { initializeEngineForcing } from "../scraper/WebScraper/utils/engine-forcing";-import { crawlFinishedQueue, NuQJob, scrapeQueue } from "./worker/nuq";+import { crawlFinishedQueue, NuQJob, scrapeQueue } from "./worker/nuq-router";import { finishCrawlSuper } from "./worker/crawl-logic";import { getCrawl } from "../lib/crawl-redis";import { TransportableError } from "../lib/error";diff --git a/apps/api/src/services/worker/nuq-fdb/client.ts b/apps/api/src/services/worker/nuq-fdb/client.tsnew file mode 100644index 000000000..0335147eb--- /dev/null+++ b/apps/api/src/services/worker/nuq-fdb/client.ts@@ -0,0 +1,41 @@+import type * as FDBModule from "foundationdb";+import type { Database } from "foundationdb";+import { config } from "../../../config";++// The foundationdb package loads libfdb_c at require-time. Keep the require+// lazy so processes that never touch the FDB backend (PG-only deploys, unit+// tests, environments without the client library) can still boot.+let fdbModule: typeof FDBModule | null = null;++export function getFdb(): typeof FDBModule {+ if (fdbModule === null) {+ // eslint-disable-next-line @typescript-eslint/no-var-requires+ fdbModule = require("foundationdb") as typeof FDBModule;+ fdbModule.setAPIVersion(720);+ }+ return fdbModule;+}++let db: Database | null = null;++export function getNuqFdbDatabase(): Database {+ if (db === null) {+ db = getFdb().open(config.FDB_CLUSTER_FILE);+ }+ return db;+}++export function isFdbConfigured(): boolean {+ return !!config.FDB_CLUSTER_FILE || config.NUQ_BACKEND === "fdb";+}++export async function nuqFdbHealthCheck(): Promise<boolean> {+ try {+ await getNuqFdbDatabase().doTn(async tn => {+ await tn.getReadVersion();+ });+ return true;+ } catch {+ return false;+ }+}diff --git a/apps/api/src/services/worker/nuq-fdb/groups.ts b/apps/api/src/services/worker/nuq-fdb/groups.tsnew file mode 100644index 000000000..2b4cb4ca0--- /dev/null+++ b/apps/api/src/services/worker/nuq-fdb/groups.ts@@ -0,0 +1,237 @@+import { randomUUID } from "crypto";+import type { Transaction } from "foundationdb";+import { Logger } from "winston";+import { logger as _logger } from "../../../lib/logger";+import { getNuqFdbDatabase } from "./client";+import {+ NuqFdbKeyspace,+ GroupMeta,+ JobMeta,+ QueueEntry,+ encodeJson,+ decodeJson,+ decodeI64,+ normalizeOwnerId,+} from "./keyspace";+import {+ ONE,+ MINUS_ONE,+ EMPTY,+ TxContext,+ newTxContext,+ uvSuffix,+ pushReady,+ setStatusQueued,+ setGroupJobIndex,+ bumpGroupStatusCount,+} from "./ops";++export type NuQFdbGroupStatus = "active" | "completed" | "cancelled";++export type NuQFdbJobGroupInstance = {+ id: string;+ status: NuQFdbGroupStatus;+ createdAt: Date;+ ownerId: string;+ ttl: number;+ expiresAt?: Date;+ maxConcurrency?: number;+ delaySeconds?: number;+};++const DEFAULT_GROUP_TTL_MS = 86400000;++export class NuqFdbGroupOps {+ constructor(+ public readonly ks: NuqFdbKeyspace,+ public readonly finishedKs: NuqFdbKeyspace | null,+ ) {}++ // Group accounting for a job reaching a terminal state. Must run in the same+ // transaction as the status transition. The blind task-key set is the+ // race-free backstop for finish detection; the inline completion attempt+ // covers the common small-crawl case instantly.+ public async terminalAccounting(+ tn: Transaction,+ gid: string,+ id: string,+ prevStatus: string,+ outcome: "completed" | "failed",+ countable: boolean,+ now: number,+ txc: TxContext,+ ): Promise<void> {+ setGroupJobIndex(tn, this.ks, gid, id, countable, outcome);+ if (countable) {+ bumpGroupStatusCount(tn, this.ks, gid, prevStatus, -1);+ bumpGroupStatusCount(tn, this.ks, gid, outcome, 1);+ if (outcome === "completed") {+ tn.setVersionstampSuffixedKey(+ this.ks.groupDonePrefix(gid),+ Buffer.from(id, "utf8"),+ uvSuffix(txc),+ );+ }+ }++ const remSnap = decodeI64(+ await tn.snapshot().get(this.ks.groupRemaining(gid)),+ );+ tn.set(this.ks.taskGroupFinish(gid), EMPTY);+ if (remSnap <= 5) {+ // near the end: read for real (conflicts with concurrent finishers, but+ // contention is bounded to the last few jobs of the group)+ const remReal = decodeI64(await tn.get(this.ks.groupRemaining(gid)));+ tn.add(this.ks.groupRemaining(gid), MINUS_ONE);+ if (remReal - 1 <= 0) {+ await this.tryCompleteGroup(tn, gid, now, txc);+ }+ } else {+ tn.add(this.ks.groupRemaining(gid), MINUS_ONE);+ }+ }++ // Completes a drained group: flips status, schedules TTL cleanup, and emits+ // the crawl-finished job. The normal read of group meta serializes+ // concurrent completers; exactly one transaction performs the emit.+ public async tryCompleteGroup(+ tn: Transaction,+ gid: string,+ now: number,+ txc: TxContext,+ ): Promise<boolean> {+ const gMeta = decodeJson<GroupMeta>(await tn.get(this.ks.groupMeta(gid)));+ if (!gMeta || gMeta.s === "completed") {+ tn.clear(this.ks.taskGroupFinish(gid));+ return false;+ }+ const expiresAt = now + gMeta.t;+ const updated: GroupMeta = { ...gMeta, s: "completed", x: expiresAt };+ tn.set(this.ks.groupMeta(gid), encodeJson(updated));+ tn.set(this.ks.groupExpiry(expiresAt, gid), EMPTY);+ tn.clear(this.ks.ongoingGroup(gMeta.o, gid));+ tn.clear(this.ks.taskGroupFinish(gid));++ if (this.finishedKs) {+ const fid = randomUUID();+ const meta: JobMeta = { c: now, p: 0, o: gMeta.o, g: gid, f: 0, dc: 1 };+ tn.set(this.finishedKs.jobMeta(fid), encodeJson(meta));+ tn.set(this.finishedKs.jobData(fid, 0), encodeJson({}));+ const entry: QueueEntry = {+ i: fid,+ o: gMeta.o,+ g: gid,+ p: 0,+ f: 0,+ c: now,+ };+ pushReady(tn, this.finishedKs, entry, txc);+ setStatusQueued(tn, this.finishedKs, fid);+ // pointer for group TTL cleanup to find the finished job's records+ tn.set(this.ks.groupFinishedJob(gid), Buffer.from(fid, "utf8"));+ }+ return true;+ }+}++export class NuQFdbJobGroup {+ constructor(+ public readonly ks: NuqFdbKeyspace,+ public readonly groupOps: NuqFdbGroupOps,+ ) {}++ private get db() {+ return getNuqFdbDatabase();+ }++ private toInstance(id: string, g: GroupMeta): NuQFdbJobGroupInstance {+ return {+ id,+ status: g.s,+ createdAt: new Date(g.c),+ ownerId: g.o,+ ttl: g.t,+ expiresAt: g.x !== undefined ? new Date(g.x) : undefined,+ maxConcurrency: g.m,+ delaySeconds: g.d,+ };+ }++ public async addGroup(+ id: string,+ ownerId: string,+ ttl?: number,+ opts?: { maxConcurrency?: number; delaySeconds?: number },+ logger: Logger = _logger,+ ): Promise<NuQFdbJobGroupInstance> {+ const owner = normalizeOwnerId(ownerId);+ if (owner === null) throw new Error("Group owner is required");+ return await this.db.doTn(async tn => {+ const existingBuf = await tn.get(this.ks.groupMeta(id));+ const existing = decodeJson<GroupMeta>(existingBuf);+ if (existing) return this.toInstance(id, existing);+ const now = Date.now();+ const g: GroupMeta = {+ o: owner,+ c: now,+ t: ttl ?? DEFAULT_GROUP_TTL_MS,+ s: "active",+ m: opts?.maxConcurrency,+ d: opts?.delaySeconds,+ };+ tn.set(this.ks.groupMeta(id), encodeJson(g));+ tn.set(this.ks.ongoingGroup(owner, id), encodeJson({ c: now }));+ return this.toInstance(id, g);+ });+ }++ public async getGroup(+ id: string,+ logger: Logger = _logger,+ ): Promise<NuQFdbJobGroupInstance | null> {+ return await this.db.doTn(async tn => {+ const g = decodeJson<GroupMeta>(await tn.get(this.ks.groupMeta(id)));+ return g ? this.toInstance(id, g) : null;+ });+ }++ public async getOngoingByOwner(+ ownerId: string,+ logger: Logger = _logger,+ ): Promise<NuQFdbJobGroupInstance[]> {+ const owner = normalizeOwnerId(ownerId);+ if (owner === null) return [];+ return await this.db.doTn(async tn => {+ const r = this.ks.ongoingGroupRange(owner);+ const rows = await tn.snapshot().getRangeAll(r.begin, r.end);+ const out: NuQFdbJobGroupInstance[] = [];+ for (const [key] of rows) {+ const gid = this.ks.unpackId(key as Buffer);+ const g = decodeJson<GroupMeta>(+ await tn.snapshot().get(this.ks.groupMeta(gid)),+ );+ if (g && g.s === "active") out.push(this.toInstance(gid, g));+ }+ return out;+ });+ }++ // O(1) cancellation: flips the group status and leaves the heavy lifting to+ // the sweeper (pending entries) and take-side diversion (ready entries).+ public async cancelGroup(+ id: string,+ logger: Logger = _logger,+ ): Promise<boolean> {+ return await this.db.doTn(async tn => {+ const g = decodeJson<GroupMeta>(await tn.get(this.ks.groupMeta(id)));+ if (!g || g.s !== "active") return false;+ tn.set(+ this.ks.groupMeta(id),+ encodeJson({ ...g, s: "cancelled" } satisfies GroupMeta),+ );+ tn.set(this.ks.taskGroupCancel(id), EMPTY);+ tn.clear(this.ks.ongoingGroup(g.o, id));+ return true;+ });+ }+}diff --git a/apps/api/src/services/worker/nuq-fdb/index.ts b/apps/api/src/services/worker/nuq-fdb/index.tsnew file mode 100644index 000000000..c0ef3d84b--- /dev/null+++ b/apps/api/src/services/worker/nuq-fdb/index.ts@@ -0,0 +1,50 @@+import type { ScrapeJobData } from "../../../types";+import { NuQFdbQueue, QueueFullError, normalizeOwnerId } from "./queue";+import { NuQFdbJobGroup } from "./groups";+import { NuqFdbSweeper } from "./sweeper";+import { NuqFdbExternalSlots } from "./slots";+import { isFdbConfigured, nuqFdbHealthCheck } from "./client";++export { NuQFdbQueue, QueueFullError, normalizeOwnerId } from "./queue";+export type {+ NuQFdbJob,+ NuQFdbJobOptions,+ NuQFdbGate,+ NuQJobStatusCompat,+} from "./queue";+export { NuQFdbJobGroup } from "./groups";+export type { NuQFdbJobGroupInstance, NuQFdbGroupStatus } from "./groups";+export { NuqFdbSweeper } from "./sweeper";+export { NuqFdbExternalSlots } from "./slots";+export { isFdbConfigured, nuqFdbHealthCheck } from "./client";++export const scrapeQueueFdb = new NuQFdbQueue<ScrapeJobData, any>("scrape", {+ hasGroups: true,+ finishedQueueName: "crawl_finished",+});++export const crawlFinishedQueueFdb = new NuQFdbQueue<any, any>(+ "crawl_finished",+ {+ hasGroups: false,+ },+);++export const crawlGroupFdb = new NuQFdbJobGroup(+ scrapeQueueFdb.ks,+ scrapeQueueFdb.groupOps!,+);++export const externalSlotsFdb = new NuqFdbExternalSlots(scrapeQueueFdb.ks);++let sweeper: NuqFdbSweeper | null = null;++export function getNuqFdbSweeper(): NuqFdbSweeper {+ if (!sweeper) {+ sweeper = new NuqFdbSweeper(+ [scrapeQueueFdb, crawlFinishedQueueFdb],+ [externalSlotsFdb],+ );+ }+ return sweeper;+}diff --git a/apps/api/src/services/worker/nuq-fdb/keyspace.ts b/apps/api/src/services/worker/nuq-fdb/keyspace.tsnew file mode 100644index 000000000..65e61287a--- /dev/null+++ b/apps/api/src/services/worker/nuq-fdb/keyspace.ts@@ -0,0 +1,345 @@+import { config } from "../../../config";+import { getFdb } from "./client";++export { normalizeOwnerId } from "../../../lib/owner-id";++// All keys for a queue live under the tuple prefix ("nuq", queueName).+// Counters are 8-byte little-endian signed ints mutated with atomic ADD.++export const READY_SHARDS = config.NUQ_FDB_READY_SHARDS;+export const TEAM_PENDING_SHARDS = config.NUQ_FDB_TEAM_PENDING_SHARDS;+export const TIME_BUCKETS = config.NUQ_FDB_TIME_BUCKETS;++export type NuqFdbJobStatus =+ | "pending" // waiting for a concurrency slot (externally: "backlog")+ | "queued" // holds its slots, in a ready shard+ | "active"+ | "completed"+ | "failed"+ | "cancelled";++// Job flags (bitmask on job meta and queue entries)+export const F_GATED = 1; // holds a team slot while queued/active+export const F_CRAWL_GATED = 2; // holds a crawl slot while team-pending/queued/active+export const F_LISTENABLE = 4;+export const F_ZDR = 8;+export const F_COUNTABLE = 16; // mode === "single_urls": counted in group numeric stats+export const F_GACC = 32; // participates in group remaining-count accounting++// Where a pending job's queue entry lives, stored on its status record so+// sweepers and removers can clear the exact key without scanning.+export type PendingLoc =+ | { k: "tq"; s: number; p: number; c: number } // team-pending: shard, priority, createdAtMs+ | { k: "gq"; p: number; c: number } // crawl-pending: priority, createdAtMs+ | { k: "dl"; at: number }; // delay index: notBeforeMs++export type JobMeta = {+ c: number; // createdAt ms+ p: number; // priority+ o: string; // ownerId (normalized uuid)+ g?: string; // groupId+ f: number; // flags+ to?: number; // backlog timesOutAt ms+ dc: number; // data chunk count+};++export type JobStatusRecord = {+ s: NuqFdbJobStatus;+ l?: string; // lock uuid while active+ e?: number; // lease expiry ms while active+ st: number; // stall count+ fa?: number; // finishedAt ms+ loc?: PendingLoc;+};++// Entry stored in ready shards, pending queues and the delay index.+export type QueueEntry = {+ i: string; // jobId+ o: string; // ownerId+ g?: string; // groupId+ p: number; // priority+ f: number; // flags+ c: number; // createdAt ms+ to?: number; // backlog timesOutAt ms+};++export type GroupMeta = {+ o: string; // ownerId+ c: number; // createdAt ms+ t: number; // ttl ms+ s: "active" | "completed" | "cancelled";+ m?: number; // maxConcurrency+ d?: number; // delay seconds between job starts+ x?: number; // expiresAt ms+};++export function encodeI64(n: number): Buffer {+ const buf = Buffer.alloc(8);+ buf.writeBigInt64LE(BigInt(n));+ return buf;+}++export function decodeI64(buf: Buffer | undefined | null): number {+ if (!buf || buf.length < 8) return 0;+ return Number(buf.readBigInt64LE());+}++export function encodeJson(v: any): Buffer {+ return Buffer.from(JSON.stringify(v), "utf8");+}++export function decodeJson<T = any>(buf: Buffer | undefined | null): T | null {+ if (!buf) return null;+ return JSON.parse(buf.toString("utf8")) as T;+}++export function fnv1a(str: string): number {+ let hash = 0x811c9dc5;+ for (let i = 0; i < str.length; i++) {+ hash ^= str.charCodeAt(i);+ hash = Math.imul(hash, 0x01000193);+ }+ return hash >>> 0;+}++function prefixEnd(prefix: Buffer): Buffer {+ const end = Buffer.from(prefix);+ for (let i = end.length - 1; i >= 0; i--) {+ if (end[i] !== 0xff) {+ end[i]++;+ return end.subarray(0, i + 1);+ }+ }+ throw new Error("Unable to construct FDB prefix range end");+}++export function timeBucket(jobId: string): number {+ return fnv1a(jobId) % TIME_BUCKETS;+}++export function teamPendingShard(jobId: string): number {+ // different rotation than timeBucket so the two don't correlate+ return fnv1a(jobId + "/tq") % TEAM_PENDING_SHARDS;+}++export class NuqFdbKeyspace {+ constructor(public readonly queueName: string) {}++ pack(parts: any[]): Buffer {+ return getFdb().tuple.pack(["nuq", this.queueName, ...parts]) as Buffer;+ }++ private packRange(parts: any[]): { begin: Buffer; end: Buffer } {+ const begin = this.pack(parts);+ return { begin, end: prefixEnd(begin) };+ }++ unpack(key: Buffer): unknown[] {+ return getFdb().tuple.unpack(key);+ }++ // === Job records+ jobMeta(id: string): Buffer {+ return this.pack(["j", id, "m"]);+ }+ jobStatus(id: string): Buffer {+ return this.pack(["j", id, "s"]);+ }+ jobData(id: string, chunk: number): Buffer {+ return this.pack(["j", id, "d", chunk]);+ }+ jobDataRange(id: string) {+ return this.packRange(["j", id, "d"]);+ }+ jobReturnvalue(id: string, chunk: number): Buffer {+ return this.pack(["j", id, "r", chunk]);+ }+ jobReturnvalueRange(id: string) {+ return this.packRange(["j", id, "r"]);+ }+ jobFailedReason(id: string): Buffer {+ return this.pack(["j", id, "f"]);+ }+ jobRange(id: string) {+ return this.packRange(["j", id]);+ }++ // === Team gate+ teamLimit(tid: string): Buffer {+ return this.pack(["t", tid, "limit"]);+ }+ teamActive(tid: string): Buffer {+ return this.pack(["t", tid, "active"]);+ }+ teamActiveIndex(tid: string): Buffer {+ return this.pack(["ta", tid]);+ }+ teamActiveIndexRange() {+ return this.packRange(["ta"]);+ }+ teamPendingCount(tid: string): Buffer {+ return this.pack(["t", tid, "pend"]);+ }+ teamShardCount(tid: string, shard: number): Buffer {+ return this.pack(["t", tid, "qn", shard]);+ }+ teamShardCountRange(tid: string) {+ return this.packRange(["t", tid, "qn"]);+ }+ teamPendingKey(+ tid: string,+ shard: number,+ priority: number,+ createdAtMs: number,+ id: string,+ ): Buffer {+ return this.pack(["t", tid, "q", shard, priority, createdAtMs, id]);+ }+ teamPendingShardRange(tid: string, shard: number) {+ return this.packRange(["t", tid, "q", shard]);+ }++ // === Group (crawl) records+ groupMeta(gid: string): Buffer {+ return this.pack(["g", gid, "meta"]);+ }+ groupRemaining(gid: string): Buffer {+ return this.pack(["g", gid, "rem"]);+ }+ groupCrawlActive(gid: string): Buffer {+ return this.pack(["g", gid, "cact"]);+ }+ groupStatusCount(gid: string, status: string): Buffer {+ return this.pack(["g", gid, "n", status]);+ }+ groupStatusCountRange(gid: string) {+ return this.packRange(["g", gid, "n"]);+ }+ groupPendingCount(gid: string): Buffer {+ return this.pack(["g", gid, "qn"]);+ }+ groupPendingKey(+ gid: string,+ priority: number,+ createdAtMs: number,+ id: string,+ ): Buffer {+ return this.pack(["g", gid, "q", priority, createdAtMs, id]);+ }+ groupPendingRange(gid: string) {+ return this.packRange(["g", gid, "q"]);+ }+ groupJob(gid: string, id: string): Buffer {+ return this.pack(["g", gid, "jobs", id]);+ }+ groupJobRange(gid: string) {+ return this.packRange(["g", gid, "jobs"]);+ }+ groupDonePrefix(gid: string): Buffer {+ return this.pack(["g", gid, "done"]);+ }+ groupDoneRange(gid: string) {+ return this.packRange(["g", gid, "done"]);+ }+ groupFinishedJob(gid: string): Buffer {+ return this.pack(["g", gid, "fjob"]);+ }+ groupRange(gid: string) {+ return this.packRange(["g", gid]);+ }+ ongoingGroup(ownerId: string, gid: string): Buffer {+ return this.pack(["go", ownerId, gid]);+ }+ ongoingGroupRange(ownerId: string) {+ return this.packRange(["go", ownerId]);+ }++ // === Ready shards+ readyPrefix(shard: number, priority: number): Buffer {+ return this.pack(["r", shard, "q", priority]);+ }+ readyShardRange(shard: number) {+ return this.packRange(["r", shard, "q"]);+ }+ readyShardCount(shard: number): Buffer {+ return this.pack(["rn", shard]);+ }+ readyShardCountRange() {+ return this.packRange(["rn"]);+ }++ // === Time-ordered indexes (sweeper-owned)+ lease(bucket: number, expMs: number, id: string): Buffer {+ return this.pack(["lease", bucket, expMs, id]);+ }+ leaseScanRange(bucket: number, untilMs: number) {+ return {+ begin: this.pack(["lease", bucket]),+ end: this.pack(["lease", bucket, untilMs]),+ };+ }+ backlogTimeout(bucket: number, atMs: number, id: string): Buffer {+ return this.pack(["bto", bucket, atMs, id]);+ }+ backlogTimeoutScanRange(bucket: number, untilMs: number) {+ return {+ begin: this.pack(["bto", bucket]),+ end: this.pack(["bto", bucket, untilMs]),+ };+ }+ delayed(bucket: number, notBeforeMs: number, id: string): Buffer {+ return this.pack(["delay", bucket, notBeforeMs, id]);+ }+ delayedScanRange(bucket: number, untilMs: number) {+ return {+ begin: this.pack(["delay", bucket]),+ end: this.pack(["delay", bucket, untilMs]),+ };+ }+ jobExpiry(bucket: number, atMs: number, id: string): Buffer {+ return this.pack(["jexp", bucket, atMs, id]);+ }+ jobExpiryScanRange(bucket: number, untilMs: number) {+ return {+ begin: this.pack(["jexp", bucket]),+ end: this.pack(["jexp", bucket, untilMs]),+ };+ }+ groupExpiry(atMs: number, gid: string): Buffer {+ return this.pack(["gexp", atMs, gid]);+ }+ groupExpiryScanRange(untilMs: number) {+ return {+ begin: this.pack(["gexp"]),+ end: this.pack(["gexp", untilMs]),+ };+ }++ // === Task keys (blind-set, sweeper-drained)+ taskGroupFinish(gid: string): Buffer {+ return this.pack(["task", "gfin", gid]);+ }+ taskGroupFinishRange() {+ return this.packRange(["task", "gfin"]);+ }+ taskGroupCancel(gid: string): Buffer {+ return this.pack(["task", "gcancel", gid]);+ }+ taskGroupCancelRange() {+ return this.packRange(["task", "gcancel"]);+ }+ taskTeamRaise(tid: string): Buffer {+ return this.pack(["task", "traise", tid]);+ }+ taskTeamRaiseRange() {+ return this.packRange(["task", "traise"]);+ }+ sweeperLock(): Buffer {+ return this.pack(["sweep", "lock"]);+ }++ unpackId(key: Buffer, indexFromEnd: number = 0): string {+ const parts = getFdb().tuple.unpack(key);+ return String(parts[parts.length - 1 - indexFromEnd]);+ }+}diff --git a/apps/api/src/services/worker/nuq-fdb/ops.ts b/apps/api/src/services/worker/nuq-fdb/ops.tsnew file mode 100644index 000000000..24526174d--- /dev/null+++ b/apps/api/src/services/worker/nuq-fdb/ops.ts@@ -0,0 +1,433 @@+import type { Transaction } from "foundationdb";+import { logger as _logger } from "../../../lib/logger";+import {+ NuqFdbKeyspace,+ QueueEntry,+ JobStatusRecord,+ GroupMeta,+ PendingLoc,+ encodeI64,+ decodeI64,+ encodeJson,+ decodeJson,+ timeBucket,+ teamPendingShard,+ READY_SHARDS,+ TEAM_PENDING_SHARDS,+ F_GATED,+ F_CRAWL_GATED,+ F_COUNTABLE,+} from "./keyspace";++export const ONE = encodeI64(1);+export const MINUS_ONE = encodeI64(-1);+export const EMPTY = Buffer.alloc(0);++export const LEASE_MS = 90_000;+export const MAX_STALLS = 9;+export const COMPLETED_STANDALONE_RETENTION_MS = 60 * 60 * 1000;+export const FAILED_STANDALONE_RETENTION_MS = 6 * 60 * 60 * 1000;++export function bumpTeamActive(+ tn: Transaction,+ ks: NuqFdbKeyspace,+ teamId: string,+ delta: number,+): void {+ if (delta === 0) return;+ const encoded = encodeI64(delta);+ tn.add(ks.teamActive(teamId), encoded);+ tn.add(ks.teamActiveIndex(teamId), encoded);+}++// Per-transaction-attempt context. uv makes versionstamp-suffixed keys unique+// within one transaction; it resets naturally when doTn retries the closure.+export type TxContext = { uv: number };++export function newTxContext(): TxContext {+ return { uv: 0 };+}++export function uvSuffix(txc: TxContext): Buffer {+ const buf = Buffer.alloc(2);+ buf.writeUInt16BE(txc.uv++);+ return buf;+}++export function randomReadyShard(): number {+ return Math.floor(Math.random() * READY_SHARDS);+}++// === Status record writers++export function setStatusQueued(+ tn: Transaction,+ ks: NuqFdbKeyspace,+ id: string,+ stalls: number = 0,+): void {+ const rec: JobStatusRecord = { s: "queued", st: stalls };+ tn.set(ks.jobStatus(id), encodeJson(rec));+}++export function setStatusPending(+ tn: Transaction,+ ks: NuqFdbKeyspace,+ id: string,+ loc: PendingLoc,+): void {+ const rec: JobStatusRecord = { s: "pending", st: 0, loc };+ tn.set(ks.jobStatus(id), encodeJson(rec));+}++// === Queue placement++export function pushReady(+ tn: Transaction,+ ks: NuqFdbKeyspace,+ e: QueueEntry,+ txc: TxContext,+): void {+ const shard = randomReadyShard();+ tn.setVersionstampSuffixedKey(+ ks.readyPrefix(shard, e.p),+ encodeJson(e),+ uvSuffix(txc),+ );+ tn.add(ks.readyShardCount(shard), ONE);+}++export function appendTeamPending(+ tn: Transaction,+ ks: NuqFdbKeyspace,+ e: QueueEntry,+): PendingLoc {+ const shard = teamPendingShard(e.i);+ tn.set(ks.teamPendingKey(e.o, shard, e.p, e.c, e.i), encodeJson(e));+ tn.add(ks.teamShardCount(e.o, shard), ONE);+ tn.add(ks.teamPendingCount(e.o), ONE);+ if (e.to !== undefined) {+ tn.set(ks.backlogTimeout(timeBucket(e.i), e.to, e.i), EMPTY);+ }+ return { k: "tq", s: shard, p: e.p, c: e.c };+}++export function appendCrawlPending(+ tn: Transaction,+ ks: NuqFdbKeyspace,+ e: QueueEntry,+): PendingLoc {+ tn.set(ks.groupPendingKey(e.g!, e.p, e.c, e.i), encodeJson(e));+ tn.add(ks.groupPendingCount(e.g!), ONE);+ tn.add(ks.teamPendingCount(e.o), ONE);+ if (e.to !== undefined) {+ tn.set(ks.backlogTimeout(timeBucket(e.i), e.to, e.i), EMPTY);+ }+ return { k: "gq", p: e.p, c: e.c };+}++export function clearBacklogTimeout(+ tn: Transaction,+ ks: NuqFdbKeyspace,+ e: { i: string; to?: number },+): void {+ if (e.to !== undefined) {+ tn.clear(ks.backlogTimeout(timeBucket(e.i), e.to, e.i));+ }+}++// Moves a pending entry into a ready shard, with group counter upkeep.+// Callers are responsible for slot accounting.+export function promoteEntryToReady(+ tn: Transaction,+ ks: NuqFdbKeyspace,+ e: QueueEntry,+ txc: TxContext,+): void {+ pushReady(tn, ks, e, txc);+ setStatusQueued(tn, ks, e.i);+ if (e.g && e.f & F_COUNTABLE) {+ tn.add(ks.groupStatusCount(e.g, "pending"), MINUS_ONE);+ tn.add(ks.groupStatusCount(e.g, "queued"), ONE);+ }+}++// Reconstructs the exact pending key for a job from its status record.+export function pendingKeyFromLoc(+ ks: NuqFdbKeyspace,+ id: string,+ ownerId: string,+ groupId: string | undefined,+ loc: PendingLoc,+): Buffer {+ if (loc.k === "tq")+ return ks.teamPendingKey(ownerId, loc.s, loc.p, loc.c, id);+ if (loc.k === "gq") return ks.groupPendingKey(groupId!, loc.p, loc.c, id);+ return ks.delayed(timeBucket(id), loc.at, id);+}++// Clears a pending job's queue entry + counters + backlog timeout. The caller+// is responsible for slot accounting and the job's status/records.+export function clearPendingPlacement(+ tn: Transaction,+ ks: NuqFdbKeyspace,+ id: string,+ ownerId: string,+ groupId: string | undefined,+ loc: PendingLoc,+ timesOutAt: number | undefined,+): void {+ tn.clear(pendingKeyFromLoc(ks, id, ownerId, groupId, loc));+ if (loc.k === "tq") {+ tn.add(ks.teamShardCount(ownerId, loc.s), MINUS_ONE);+ tn.add(ks.teamPendingCount(ownerId), MINUS_ONE);+ } else if (loc.k === "gq") {+ tn.add(ks.groupPendingCount(groupId!), MINUS_ONE);+ tn.add(ks.teamPendingCount(ownerId), MINUS_ONE);+ }+ if (loc.k !== "dl") {+ clearBacklogTimeout(tn, ks, { i: id, to: timesOutAt });+ }+}++// === Pending queue pops++// Pops the best entry from a team's pending shards. Snapshot-reads the shard+// occupancy counters and probes up to 3 non-empty shards. Returns null if all+// probed shards are empty.+export async function popTeamPending(+ tn: Transaction,+ ks: NuqFdbKeyspace,+ tid: string,+): Promise<QueueEntry | null> {+ const startedAt = Date.now();+ const range = ks.teamShardCountRange(tid);+ const counts = await tn.snapshot().getRangeAll(range.begin, range.end);+ const nonEmpty: number[] = [];+ for (const [key, value] of counts) {+ if (decodeI64(value as Buffer) > 0)+ nonEmpty.push(Number(ks.unpackId(key as Buffer)));+ }+ // shuffle so concurrent finishers spread across shards+ for (let i = nonEmpty.length - 1; i > 0; i--) {+ const j = Math.floor(Math.random() * (i + 1));+ [nonEmpty[i], nonEmpty[j]] = [nonEmpty[j], nonEmpty[i]];+ }+ let probedShards = 0;+ let staleShardCounters = 0;+ for (const shard of nonEmpty.slice(0, 3)) {+ probedShards++;+ const r = ks.teamPendingShardRange(tid, shard);+ const head = await tn.getRangeAll(r.begin, r.end, { limit: 1 });+ if (head.length === 0) {+ staleShardCounters++;+ continue; // counter was stale+ }+ const [key, value] = head[0];+ const e = decodeJson<QueueEntry>(value as Buffer)!;+ tn.clear(key as Buffer);+ tn.add(ks.teamShardCount(tid, shard), MINUS_ONE);+ tn.add(ks.teamPendingCount(tid), MINUS_ONE);+ clearBacklogTimeout(tn, ks, e);+ _logger.debug("NuQ FDB team pending promotion", {+ canonicalLog: "nuq-fdb/team_pending_promotion",+ queueName: ks.queueName,+ result: "promoted",+ durationMs: Date.now() - startedAt,+ nonEmptyShardCount: nonEmpty.length,+ probedShardCount: probedShards,+ staleShardCounterCount: staleShardCounters,+ selectedShard: shard,+ priority: e.p,+ queuedAgeMs: Date.now() - e.c,+ });+ return e;+ }+ _logger.debug("NuQ FDB team pending promotion", {+ canonicalLog: "nuq-fdb/team_pending_promotion",+ queueName: ks.queueName,+ result: "empty",+ durationMs: Date.now() - startedAt,+ nonEmptyShardCount: nonEmpty.length,+ probedShardCount: probedShards,+ staleShardCounterCount: staleShardCounters,+ });+ return null;+}++export async function popCrawlPending(+ tn: Transaction,+ ks: NuqFdbKeyspace,+ gid: string,+): Promise<QueueEntry | null> {+ const r = ks.groupPendingRange(gid);+ const head = await tn.getRangeAll(r.begin, r.end, { limit: 1 });+ if (head.length === 0) return null;+ const [key, value] = head[0];+ const e = decodeJson<QueueEntry>(value as Buffer)!;+ tn.clear(key as Buffer);+ tn.add(ks.groupPendingCount(gid), MINUS_ONE);+ tn.add(ks.teamPendingCount(e.o), MINUS_ONE);+ clearBacklogTimeout(tn, ks, e);+ return e;+}++// === Slot release + inline promotion chain+//+// Slots are handed off one-for-one: a released slot goes to a pending head in+// the same transaction whenever possible, so this function never does a+// conflicting read of the team active counter on the handoff path.+//+// `held` describes which slots the leaving job actually holds:+// - active / ready jobs: team + crawl (if crawl-gated)+// - team-pending / delayed jobs: crawl only (if crawl-gated)+// - crawl-pending jobs: none++export async function releaseSlotsAndPromote(+ tn: Transaction,+ ks: NuqFdbKeyspace,+ e: QueueEntry,+ held: { team: boolean; crawl: boolean },+ now: number,+ txc: TxContext,+): Promise<void> {+ if (!(e.f & F_GATED)) return;+ const tid = e.o;++ const holdsCrawl = held.crawl && !!e.g && !!(e.f & F_CRAWL_GATED);+ if (!held.team && !holdsCrawl) return;++ // Limit-lowering convergence: if the team is over its limit, don't hand off.+ let overLimit = false;+ let limit = Infinity;+ if (held.team) {+ const limitBuf = await tn.get(ks.teamLimit(tid)); // cold key, rarely written+ if (limitBuf) {+ limit = decodeI64(limitBuf);+ const snapActive = decodeI64(await tn.snapshot().get(ks.teamActive(tid)));+ overLimit = snapActive > limit;+ }+ }++ let teamHead: QueueEntry | null | "consumed" = null;+ if (held.team && !overLimit) {+ teamHead = await popTeamPending(tn, ks, tid);+ }++ // Crawl slot handoff: promote the crawl-pending head out of the crawl gate.+ let crawlPromoted: QueueEntry | null = null;+ let crawlDelaySeconds = 0;+ if (holdsCrawl) {+ const gMeta = decodeJson<GroupMeta>(+ await tn.snapshot().get(ks.groupMeta(e.g!)),+ );+ if (gMeta && gMeta.s === "active") {+ crawlPromoted = await popCrawlPending(tn, ks, e.g!);+ if (crawlPromoted) {+ crawlDelaySeconds = gMeta.d ?? 0;+ } else {+ tn.add(ks.groupCrawlActive(e.g!), MINUS_ONE);+ }+ } else {+ tn.add(ks.groupCrawlActive(e.g!), MINUS_ONE);+ }+ }++ if (crawlPromoted) {+ const j2 = crawlPromoted;+ if (crawlDelaySeconds > 0) {+ // crawl delay: park with a not-before timestamp; keeps the crawl slot.+ const notBefore = now + crawlDelaySeconds * 1000;+ tn.set(ks.delayed(timeBucket(j2.i), notBefore, j2.i), encodeJson(j2));+ setStatusPending(tn, ks, j2.i, { k: "dl", at: notBefore });+ } else if (held.team && !overLimit && teamHead === null) {+ // the freed team slot goes directly to the crawl-promoted job+ promoteEntryToReady(tn, ks, j2, txc);+ teamHead = "consumed";+ } else if (!held.team) {+ // no team slot was freed here (job removal/cancellation paths), so the+ // promoted job must be admitted through the gate on its own+ await admitThroughTeamGate(tn, ks, j2, txc);+ } else {+ // waits in the team gate; keeps the crawl slot+ const loc = appendTeamPending(tn, ks, j2);+ setStatusPending(tn, ks, j2.i, loc);+ }+ }++ if (held.team) {+ if (teamHead === "consumed") {+ // slot handed to crawlPromoted above+ } else if (teamHead !== null) {+ promoteEntryToReady(tn, ks, teamHead, txc);+ } else {+ bumpTeamActive(tn, ks, tid, -1);+ }+ }+}++// Admits a slotless entry through the team gate, used when a job enters the+// team gate outside of a one-for-one handoff (delayed promotion, limit raise).+// Does a conflicting read of the team active counter -- callers are rare paths.+export async function admitThroughTeamGate(+ tn: Transaction,+ ks: NuqFdbKeyspace,+ e: QueueEntry,+ txc: TxContext,+): Promise<void> {+ const limitBuf = await tn.get(ks.teamLimit(e.o));+ const limit = limitBuf ? decodeI64(limitBuf) : Infinity;+ const active = decodeI64(await tn.get(ks.teamActive(e.o)));+ if (active < limit) {+ bumpTeamActive(tn, ks, e.o, 1);+ promoteEntryToReady(tn, ks, e, txc);+ } else {+ const loc = appendTeamPending(tn, ks, e);+ setStatusPending(tn, ks, e.i, loc);+ }+}++// === Job record cleanup++export function deleteJobRecords(+ tn: Transaction,+ ks: NuqFdbKeyspace,+ id: string,+): void {+ const r = ks.jobRange(id);+ tn.clearRange(r.begin, r.end);+}++// === Group accounting helpers++export type GroupJobIndexValue = {+ m: number; // 1 if countable (single_urls)+ s: string; // job status+};++export function setGroupJobIndex(+ tn: Transaction,+ ks: NuqFdbKeyspace,+ gid: string,+ id: string,+ countable: boolean,+ status: string,+): void {+ tn.set(+ ks.groupJob(gid, id),+ encodeJson({+ m: countable ? 1 : 0,+ s: status,+ } satisfies GroupJobIndexValue),+ );+}++export function bumpGroupStatusCount(+ tn: Transaction,+ ks: NuqFdbKeyspace,+ gid: string,+ status: string,+ delta: 1 | -1,+): void {+ tn.add(ks.groupStatusCount(gid, status), delta === 1 ? ONE : MINUS_ONE);+}diff --git a/apps/api/src/services/worker/nuq-fdb/queue.ts b/apps/api/src/services/worker/nuq-fdb/queue.tsnew file mode 100644index 000000000..9d0541fe4--- /dev/null+++ b/apps/api/src/services/worker/nuq-fdb/queue.ts@@ -0,0 +1,1276 @@+import { randomUUID } from "crypto";+import type { Transaction } from "foundationdb";+import { Logger } from "winston";+import { logger as _logger } from "../../../lib/logger";+import { config } from "../../../config";+import { QueueFullError } from "../../../lib/queue-full-error";+import { getNuqFdbDatabase } from "./client";+import {+ NuqFdbKeyspace,+ NuqFdbJobStatus,+ JobMeta,+ JobStatusRecord,+ QueueEntry,+ GroupMeta,+ encodeI64,+ decodeI64,+ encodeJson,+ decodeJson,+ timeBucket,+ READY_SHARDS,+ F_GATED,+ F_CRAWL_GATED,+ F_LISTENABLE,+ F_ZDR,+ F_COUNTABLE,+ F_GACC,+ normalizeOwnerId,+} from "./keyspace";++export { normalizeOwnerId, F_GACC };+import {+ ONE,+ MINUS_ONE,+ EMPTY,+ LEASE_MS,+ MAX_STALLS,+ COMPLETED_STANDALONE_RETENTION_MS,+ FAILED_STANDALONE_RETENTION_MS,+ TxContext,+ newTxContext,+ pushReady,+ appendTeamPending,+ appendCrawlPending,+ popTeamPending,+ setStatusQueued,+ setStatusPending,+ clearPendingPlacement,+ releaseSlotsAndPromote,+ deleteJobRecords,+ setGroupJobIndex,+ bumpGroupStatusCount,+ bumpTeamActive,+ GroupJobIndexValue,+} from "./ops";+import { NuqFdbGroupOps } from "./groups";++const DATA_CHUNK_BYTES = 90 * 1024;+// FoundationDB caps transactions at 10,000,000 bytes of affected data. Keep+// enqueue batches much smaller so large payloads cannot accidentally combine+// into a commit-time transaction_too_large failure.+const FDB_TRANSACTION_BYTE_LIMIT = 10_000_000;+const ENQUEUE_BATCH_BYTE_BUDGET = 750 * 1024;+const ENQUEUE_SINGLE_JOB_BYTE_LIMIT = 8 * 1024 * 1024;+const ENQUEUE_MAX_JOBS_PER_TRANSACTION = 250;++export { QueueFullError };++type FdbKeySelector = {+ key: Buffer;+ orEqual: boolean;+ offset: number;+ _isKeySelector: true;+};++export type NuQJobStatusCompat =+ | "queued"+ | "active"+ | "completed"+ | "failed"+ | "backlog";++export type NuQFdbJob<Data = any, ReturnValue = any> = {+ id: string;+ status: NuQJobStatusCompat;+ createdAt: Date;+ priority: number;+ data: Data;+ finishedAt?: Date;+ returnvalue?: ReturnValue;+ failedReason?: string;+ lock?: string;+ leaseExpiresAt?: Date;+ ownerId?: string;+ groupId?: string;+};++export type NuQFdbJobOptions = {+ priority?: number;+ listenable?: boolean;+ ownerId?: string;+ groupId?: string;+ // when set, the job bypasses both concurrency gates (kickoff jobs)+ bypassGate?: boolean;+ // backlog timeout: if the job is still waiting for a slot at this time, it+ // is silently dropped (matches the PG backlog reaper)+ timesOutAt?: Date;+};++export type NuQFdbGate = {+ // null = unlimited (self-hosted)+ teamLimit: number | null;+ queueCap: number;+};++type AddJobInput<Data> = {+ id: string;+ data: Data;+ options: NuQFdbJobOptions;+};++type PreparedAddJob<Data> = AddJobInput<Data> & {+ dataBuf: Buffer;+ dataChunks: Buffer[];+ estimatedAffectedBytes: number;+};++function chunkBuffer(buf: Buffer, size: number): Buffer[] {+ const chunks: Buffer[] = [];+ for (let i = 0; i < buf.length; i += size) {+ chunks.push(buf.subarray(i, i + size));+ }+ return chunks.length > 0 ? chunks : [Buffer.alloc(0)];+}++function externalStatus(s: NuqFdbJobStatus): NuQJobStatusCompat | null {+ if (s === "pending") return "backlog";+ if (s === "cancelled") return null; // cancelled jobs read as gone, like PG row deletes+ return s;+}++function encodedJsonBytes(v: any): number {+ return Buffer.byteLength(JSON.stringify(v), "utf8");+}++export class NuQFdbQueue<JobData = any, JobReturnValue = any> {+ public readonly ks: NuqFdbKeyspace;+ public readonly groupOps: NuqFdbGroupOps | null;+ // worker-local lease expiry tracking so renewLock can stay a blind write+ private leaseExps: Map<string, number> = new Map();++ constructor(+ public readonly queueName: string,+ public readonly options: {+ // whether jobs in this queue participate in group (crawl) accounting+ hasGroups: boolean;+ // queue that group-finish jobs are emitted into (scrape -> crawl_finished)+ finishedQueueName?: string;+ // lease duration override, used by tests+ leaseMs?: number;+ },+ ) {+ this.ks = new NuqFdbKeyspace(queueName);+ this.groupOps = options.hasGroups+ ? new NuqFdbGroupOps(+ this.ks,+ options.finishedQueueName+ ? new NuqFdbKeyspace(options.finishedQueueName)+ : null,+ )+ : null;+ }++ private get db() {+ return getNuqFdbDatabase();+ }++ private storeReturnvalueInline(): boolean {+ // cloud stores results in GCS; self-host keeps them in the queue+ return !config.GCS_BUCKET_NAME;+ }++ // === Enqueue++ public async addJob(+ id: string,+ data: JobData,+ options: NuQFdbJobOptions,+ gate: NuQFdbGate,+ ): Promise<NuQFdbJob<JobData, JobReturnValue>> {+ const [job] = await this.addJobs([{ id, data, options }], gate);+ return job;+ }++ public async addJobs(+ jobs: AddJobInput<JobData>[],+ gate: NuQFdbGate,+ logger: Logger = _logger,+ ): Promise<NuQFdbJob<JobData, JobReturnValue>[]> {+ if (jobs.length === 0) return [];+ const ownerId = normalizeOwnerId(jobs[0].options.ownerId);+ if (jobs.some(j => normalizeOwnerId(j.options.ownerId) !== ownerId)) {+ throw new Error("addJobs requires all jobs to share an owner");+ }++ const prepared = jobs.map(j => this.prepareAddJob(j, ownerId));+ const results: NuQFdbJob<JobData, JobReturnValue>[] = [];+ let batch: PreparedAddJob<JobData>[] = [];+ let batchBytes = 0;++ for (const job of prepared) {+ if (job.estimatedAffectedBytes > ENQUEUE_SINGLE_JOB_BYTE_LIMIT) {+ throw new Error(+ `NuQ FDB job ${job.id} is too large to enqueue safely: estimated ${job.estimatedAffectedBytes} bytes of affected data exceeds ${ENQUEUE_SINGLE_JOB_BYTE_LIMIT}`,+ );+ }++ if (job.estimatedAffectedBytes > ENQUEUE_BATCH_BYTE_BUDGET) {+ logger.warn("NuQ FDB enqueue job exceeds batch budget", {+ canonicalLog: "nuq-fdb/enqueue_batch",+ queueName: this.queueName,+ jobId: job.id,+ estimatedAffectedBytes: job.estimatedAffectedBytes,+ batchByteBudget: ENQUEUE_BATCH_BYTE_BUDGET,+ transactionByteLimit: FDB_TRANSACTION_BYTE_LIMIT,+ });+ }++ if (+ batch.length > 0 &&+ (batchBytes + job.estimatedAffectedBytes > ENQUEUE_BATCH_BYTE_BUDGET ||+ batch.length >= ENQUEUE_MAX_JOBS_PER_TRANSACTION)+ ) {+ results.push(...(await this.addJobsBatch(batch, ownerId, gate)));+ batch = [];+ batchBytes = 0;+ }++ batch.push(job);+ batchBytes += job.estimatedAffectedBytes;+ }++ if (batch.length > 0) {+ results.push(...(await this.addJobsBatch(batch, ownerId, gate)));+ }+ return results;+ }++ private prepareAddJob(+ job: AddJobInput<JobData>,+ ownerId: string | null,+ ): PreparedAddJob<JobData> {+ const dataBuf = Buffer.from(JSON.stringify(job.data ?? null), "utf8");+ const dataChunks = chunkBuffer(dataBuf, DATA_CHUNK_BYTES);+ const estimatedAffectedBytes = this.estimateEnqueueAffectedBytes(+ job,+ ownerId,+ dataChunks,+ );+ return {+ ...job,+ dataBuf,+ dataChunks,+ estimatedAffectedBytes,+ };+ }++ private estimateEnqueueAffectedBytes(+ job: AddJobInput<JobData>,+ ownerId: string | null,+ dataChunks: Buffer[],+ ): number {+ const ks = this.ks;+ const now = Date.now();+ const priority = job.options.priority ?? 0;+ const gid = job.options.groupId;+ const owner = ownerId ?? "";+ const entry: QueueEntry = {+ i: job.id,+ o: owner,+ g: gid,+ p: priority,+ f: F_GATED | F_CRAWL_GATED | F_LISTENABLE | F_ZDR | F_COUNTABLE | F_GACC,+ c: now,+ to: job.options.timesOutAt?.getTime(),+ };+ const meta: JobMeta = {+ c: now,+ p: priority,+ o: owner,+ g: gid,+ f: entry.f,+ to: entry.to,+ dc: dataChunks.length,+ };++ let bytes =+ ks.jobMeta(job.id).length ++ encodedJsonBytes(meta) ++ ks.jobStatus(job.id).length ++ encodedJsonBytes({ s: "pending", st: 0 }) ++ ks.readyPrefix(READY_SHARDS - 1, priority).length ++ encodedJsonBytes(entry) ++ ks.readyShardCount(READY_SHARDS - 1).length ++ 8;++ for (let i = 0; i < dataChunks.length; i++) {+ bytes += ks.jobData(job.id, i).length + dataChunks[i].length;+ }++ if (owner) {+ bytes +=+ ks.teamLimit(owner).length ++ 8 ++ ks.teamActive(owner).length ++ 8 ++ ks.teamActiveIndex(owner).length ++ 8 ++ ks.teamPendingCount(owner).length ++ 8 ++ ks.teamShardCount(owner, 0).length ++ 8 ++ ks.teamPendingKey(owner, 0, priority, now, job.id).length ++ encodedJsonBytes(entry);+ }++ if (gid) {+ bytes +=+ ks.groupMeta(gid).length ++ ks.groupRemaining(gid).length ++ 8 ++ ks.groupCrawlActive(gid).length ++ 8 ++ ks.groupStatusCount(gid, "pending").length ++ 8 ++ ks.groupPendingCount(gid).length ++ 8 ++ ks.groupPendingKey(gid, priority, now, job.id).length ++ encodedJsonBytes(entry) ++ ks.groupJob(gid, job.id).length ++ encodedJsonBytes({ m: 1, s: "pending" });+ }++ if (entry.to !== undefined) {+ bytes += ks.backlogTimeout(timeBucket(job.id), entry.to, job.id).length;+ }++ // Account for versionstamp suffixes, conflict ranges, tuple overhead, and+ // status loc variants. This intentionally overestimates so the batching+ // boundary stays conservative as placement changes inside the transaction.+ return bytes + 4096;+ }++ private async addJobsBatch(+ jobs: PreparedAddJob<JobData>[],+ ownerId: string | null,+ gate: NuQFdbGate,+ ): Promise<NuQFdbJob<JobData, JobReturnValue>[]> {+ const ks = this.ks;+ return await this.db.doTn(async tn => {+ const txc = newTxContext();+ const now = Date.now();+ const out: NuQFdbJob<JobData, JobReturnValue>[] = [];++ let free = Infinity;+ if (gate.teamLimit !== null && ownerId !== null) {+ const storedBuf = await tn.get(ks.teamLimit(ownerId));+ const stored = storedBuf ? decodeI64(storedBuf) : null;+ if (stored !== gate.teamLimit) {+ tn.set(ks.teamLimit(ownerId), encodeI64(gate.teamLimit));+ if (stored !== null && gate.teamLimit > stored) {+ tn.set(ks.taskTeamRaise(ownerId), EMPTY);+ }+ }++ const pend = decodeI64(+ await tn.snapshot().get(ks.teamPendingCount(ownerId)),+ );+ if (pend + jobs.length > gate.queueCap) {+ throw new QueueFullError(pend, gate.queueCap);+ }++ // big-limit teams tolerate a small admission overshoot in exchange for+ // a conflict-free read; small-limit teams get the strict read+ const active =+ gate.teamLimit >= 256+ ? decodeI64(await tn.snapshot().get(ks.teamActive(ownerId)))+ : decodeI64(await tn.get(ks.teamActive(ownerId)));+ free = Math.max(0, gate.teamLimit - active);+ }++ // crawl gate state per distinct live group+ const groupMetas = new Map<string, GroupMeta | null>();+ const crawlFree = new Map<string, number>();+ if (this.options.hasGroups) {+ for (const j of jobs) {+ const gid = j.options.groupId;+ if (!gid || groupMetas.has(gid)) continue;+ const gMeta = decodeJson<GroupMeta>(await tn.get(ks.groupMeta(gid)));+ groupMetas.set(gid, gMeta);+ if (gMeta && gMeta.s === "active" && gate.teamLimit !== null) {+ const effM = (gMeta.d ?? 0) > 0 ? 1 : gMeta.m;+ if (effM !== undefined) {+ const cact = decodeI64(await tn.get(ks.groupCrawlActive(gid)));+ crawlFree.set(gid, effM - cact);+ }+ }+ }+ }++ let granted = 0;+ const crawlAcquired = new Map<string, number>();++ for (const j of jobs) {+ const gid = j.options.groupId;+ const gMeta = gid ? groupMetas.get(gid) : null;+ const groupLive = !!gMeta && gMeta.s === "active";+ const gated = gate.teamLimit !== null && !j.options.bypassGate;+ const crawlGated = gated && !!gid && groupLive && crawlFree.has(gid!);+ const countable =+ this.options.hasGroups &&+ groupLive &&+ (j.data as any)?.mode === "single_urls";++ let flags = 0;+ if (gated) flags |= F_GATED;+ if (crawlGated) flags |= F_CRAWL_GATED;+ if (j.options.listenable) flags |= F_LISTENABLE;+ if ((j.data as any)?.zeroDataRetention) flags |= F_ZDR;+ if (countable) flags |= F_COUNTABLE;+ if (groupLive) flags |= F_GACC;++ const timesOutAt = gated ? j.options.timesOutAt?.getTime() : undefined;+ const entry: QueueEntry = {+ i: j.id,+ o: ownerId ?? "",+ g: gid,+ p: j.options.priority ?? 0,+ f: flags,+ c: now,+ to: timesOutAt,+ };++ const meta: JobMeta = {+ c: now,+ p: entry.p,+ o: entry.o,+ g: gid,+ f: flags,+ to: timesOutAt,+ dc: j.dataChunks.length,+ };+ tn.set(ks.jobMeta(j.id), encodeJson(meta));+ j.dataChunks.forEach((chunk, ci) =>+ tn.set(ks.jobData(j.id, ci), chunk),+ );++ let placedStatus: NuqFdbJobStatus;+ if (!gated) {+ pushReady(tn, ks, entry, txc);+ setStatusQueued(tn, ks, j.id);+ placedStatus = "queued";+ } else if (crawlGated && crawlFree.get(gid!)! <= 0) {+ const loc = appendCrawlPending(tn, ks, entry);+ setStatusPending(tn, ks, j.id, loc);+ placedStatus = "pending";+ } else {+ if (crawlGated) {+ crawlFree.set(gid!, crawlFree.get(gid!)! - 1);+ crawlAcquired.set(gid!, (crawlAcquired.get(gid!) ?? 0) + 1);+ }+ if (free > 0) {+ free--;+ granted++;+ pushReady(tn, ks, entry, txc);+ setStatusQueued(tn, ks, j.id);+ placedStatus = "queued";+ } else {+ const loc = appendTeamPending(tn, ks, entry);+ setStatusPending(tn, ks, j.id, loc);+ placedStatus = "pending";+ }+ }++ if (groupLive && gid) {+ tn.add(ks.groupRemaining(gid), ONE);+ setGroupJobIndex(tn, ks, gid, j.id, countable, placedStatus);+ if (countable) bumpGroupStatusCount(tn, ks, gid, placedStatus, 1);+ }++ out.push({+ id: j.id,+ status: placedStatus === "pending" ? "backlog" : "queued",+ createdAt: new Date(now),+ priority: entry.p,+ data: j.data,+ ownerId: entry.o || undefined,+ groupId: gid,+ });+ }++ if (granted > 0 && ownerId !== null) {+ bumpTeamActive(tn, ks, ownerId, granted);+ }+ for (const [gid, n] of crawlAcquired) {+ tn.add(ks.groupCrawlActive(gid), encodeI64(n));+ }++ return out;+ });+ }++ // === Take (worker dequeue)++ public async getJobToProcess(+ logger: Logger = _logger,+ ): Promise<NuQFdbJob<JobData, JobReturnValue> | null> {+ const startedAt = Date.now();+ // blind random probes win at high occupancy (no coordination, conflicts+ // spread across shards); the occupancy scan below covers the sparse case+ const PROBES = 4;+ const tried = new Set<number>();+ let randomDropped = 0;+ while (tried.size < PROBES) {+ const shard = Math.floor(Math.random() * READY_SHARDS);+ if (tried.has(shard)) continue;+ const result = await this.takeFromShard(shard);+ if (result === "empty") {+ tried.add(shard);+ continue;+ }+ if (result === "dropped") {+ // tombstone or cancelled-group divert consumed an entry; same shard+ // may hold live work, try it again without burning a probe+ randomDropped++;+ continue;+ }+ this.leaseExps.set(result.id, result.leaseExpiresAt!.getTime());+ logger.debug("NuQ FDB dequeue attempt", {+ canonicalLog: "nuq-fdb/dequeue",+ queueName: this.queueName,+ result: "job",+ path: "random_probe",+ durationMs: Date.now() - startedAt,+ readyShards: READY_SHARDS,+ randomProbeCount: tried.size + 1,+ randomEmptyCount: tried.size,+ randomDroppedCount: randomDropped,+ });+ return result;+ }++ // sparse queue: find non-empty shards via their occupancy counters+ const candidates = await this.db.doTn(async tn => {+ const r = this.ks.readyShardCountRange();+ const counts = await tn.snapshot().getRangeAll(r.begin, r.end);+ const nonEmpty: number[] = [];+ for (const [key, value] of counts) {+ if (decodeI64(value as Buffer) > 0) {+ nonEmpty.push(Number(this.ks.unpackId(key as Buffer)));+ }+ }+ return nonEmpty;+ });+ for (let i = candidates.length - 1; i > 0; i--) {+ const j = Math.floor(Math.random() * (i + 1));+ [candidates[i], candidates[j]] = [candidates[j], candidates[i]];+ }+ let fallbackDropped = 0;+ let fallbackEmpty = 0;+ let fallbackAttempts = 0;+ for (const shard of candidates.slice(0, 8)) {+ if (tried.has(shard)) continue;+ for (let attempt = 0; attempt < 4; attempt++) {+ fallbackAttempts++;+ const result = await this.takeFromShard(shard);+ if (result === "empty") {+ fallbackEmpty++;+ break;+ }+ if (result === "dropped") {+ fallbackDropped++;+ continue;+ }+ this.leaseExps.set(result.id, result.leaseExpiresAt!.getTime());+ logger.debug("NuQ FDB dequeue attempt", {+ canonicalLog: "nuq-fdb/dequeue",+ queueName: this.queueName,+ result: "job",+ path: "sparse_scan",+ durationMs: Date.now() - startedAt,+ readyShards: READY_SHARDS,+ randomProbeCount: tried.size,+ randomEmptyCount: tried.size,+ randomDroppedCount: randomDropped,+ nonEmptyCandidateCount: candidates.length,+ fallbackAttemptCount: fallbackAttempts,+ fallbackEmptyCount: fallbackEmpty,+ fallbackDroppedCount: fallbackDropped,+ });+ return result;+ }+ }+ logger.debug("NuQ FDB dequeue attempt", {+ canonicalLog: "nuq-fdb/dequeue",+ queueName: this.queueName,+ result: "empty",+ path: "sparse_scan",+ durationMs: Date.now() - startedAt,+ readyShards: READY_SHARDS,+ randomProbeCount: tried.size,+ randomEmptyCount: tried.size,+ randomDroppedCount: randomDropped,+ nonEmptyCandidateCount: candidates.length,+ fallbackAttemptCount: fallbackAttempts,+ fallbackEmptyCount: fallbackEmpty,+ fallbackDroppedCount: fallbackDropped,+ });+ return null;+ }++ private async takeFromShard(+ shard: number,+ ): Promise<NuQFdbJob<JobData, JobReturnValue> | "empty" | "dropped"> {+ const ks = this.ks;+ return await this.db.doTn(async tn => {+ const txc = newTxContext();+ const now = Date.now();+ const range = ks.readyShardRange(shard);+ const head = await tn.getRangeAll(range.begin, range.end, { limit: 1 });+ if (head.length === 0) return "empty";+ const [key, value] = head[0];+ const e = decodeJson<QueueEntry>(value as Buffer)!;+ tn.clear(key as Buffer);+ tn.add(ks.readyShardCount(shard), MINUS_ONE);++ const st = decodeJson<JobStatusRecord>(await tn.get(ks.jobStatus(e.i)));+ if (!st || st.s !== "queued") {+ // tombstone (removed job) -- slots were released by the remover+ return "dropped";+ }++ if (e.g && this.options.hasGroups) {+ const gMeta = decodeJson<GroupMeta>(+ await tn.snapshot().get(ks.groupMeta(e.g)),+ );+ if (gMeta && gMeta.s === "cancelled") {+ // lazy cancellation: this job dies at take time+ if (e.f & F_GACC) {+ const gj = decodeJson<GroupJobIndexValue>(+ await tn.get(ks.groupJob(e.g, e.i)),+ );+ if (gj) {+ tn.clear(ks.groupJob(e.g, e.i));+ tn.add(ks.groupRemaining(e.g), MINUS_ONE);+ if (e.f & F_COUNTABLE)+ bumpGroupStatusCount(tn, ks, e.g, "queued", -1);+ tn.set(ks.taskGroupFinish(e.g), EMPTY);+ }+ }+ deleteJobRecords(tn, ks, e.i);+ await releaseSlotsAndPromote(+ tn,+ ks,+ e,+ { team: true, crawl: true },+ now,+ txc,+ );+ return "dropped";+ }+ }++ const meta = decodeJson<JobMeta>(+ await tn.snapshot().get(ks.jobMeta(e.i)),+ );+ if (!meta) return "dropped";+ const dataRange = ks.jobDataRange(e.i);+ const dataParts = await tn+ .snapshot()+ .getRangeAll(dataRange.begin, dataRange.end);+ const data = JSON.parse(+ Buffer.concat(dataParts.map(([, v]) => v as Buffer)).toString("utf8"),+ );++ const lock = randomUUID();+ const exp = now + (this.options.leaseMs ?? LEASE_MS);+ const rec: JobStatusRecord = { s: "active", l: lock, e: exp, st: st.st };+ tn.set(ks.jobStatus(e.i), encodeJson(rec));+ tn.set(ks.lease(timeBucket(e.i), exp, e.i), encodeJson({ l: lock }));+ if (e.g && e.f & F_COUNTABLE) {+ bumpGroupStatusCount(tn, ks, e.g, "queued", -1);+ bumpGroupStatusCount(tn, ks, e.g, "active", 1);+ setGroupJobIndex(tn, ks, e.g, e.i, true, "active");+ }++ return {+ id: e.i,+ status: "active" as const,+ createdAt: new Date(meta.c),+ priority: meta.p,+ data,+ lock,+ leaseExpiresAt: new Date(exp),+ ownerId: meta.o || undefined,+ groupId: meta.g,+ };+ });+ }++ // === Leases++ public async renewLock(+ id: string,+ lock: string,+ logger: Logger = _logger,+ ): Promise<boolean> {+ const oldExp = this.leaseExps.get(id);+ if (oldExp === undefined) return false;+ const ks = this.ks;+ const newExp = Date.now() + (this.options.leaseMs ?? LEASE_MS);+ try {+ await this.db.doTn(async tn => {+ // blind writes only; the sweeper validates the lock before reaping+ tn.clear(ks.lease(timeBucket(id), oldExp, id));+ tn.set(ks.lease(timeBucket(id), newExp, id), encodeJson({ l: lock }));+ // the lease index moved, keep the status record's expiry in sync for+ // observability; only the worker holding the lock writes this+ const st = decodeJson<JobStatusRecord>(+ await tn.snapshot().get(ks.jobStatus(id)),+ );+ if (!st || st.s !== "active" || st.l !== lock) {+ throw new LockLostError();+ }+ tn.set(ks.jobStatus(id), encodeJson({ ...st, e: newExp }));+ });+ } catch (e) {+ if (e instanceof LockLostError) {+ this.leaseExps.delete(id);+ return false;+ }+ throw e;+ }+ this.leaseExps.set(id, newExp);+ return true;+ }++ // === Finish / fail++ public async jobFinish(+ id: string,+ lock: string,+ returnvalue: JobReturnValue | null,+ logger: Logger = _logger,+ ): Promise<boolean> {+ return this.finishOrFail(id, lock, "completed", returnvalue ?? null, null);+ }++ public async jobFail(+ id: string,+ lock: string,+ failedReason: string,+ logger: Logger = _logger,+ ): Promise<boolean> {+ return this.finishOrFail(id, lock, "failed", null, failedReason);+ }++ private async finishOrFail(+ id: string,+ lock: string,+ outcome: "completed" | "failed",+ returnvalue: any,+ failedReason: string | null,+ ): Promise<boolean> {+ const ks = this.ks;+ const ok = await this.db.doTn(async tn => {+ const txc = newTxContext();+ const now = Date.now();+ const st = decodeJson<JobStatusRecord>(await tn.get(ks.jobStatus(id)));+ if (!st) return false;+ // idempotency: a commit_unknown_result retry lands here+ if (st.s === outcome && st.l === lock) return true;+ if (st.s !== "active" || st.l !== lock) return false;+ const meta = decodeJson<JobMeta>(await tn.get(ks.jobMeta(id)));+ if (!meta) return false;++ const rec: JobStatusRecord = {+ s: outcome,+ l: lock,+ st: st.st,+ fa: now,+ };+ tn.set(ks.jobStatus(id), encodeJson(rec));+ if (st.e !== undefined) tn.clear(ks.lease(timeBucket(id), st.e, id));++ if (outcome === "completed") {+ if (+ returnvalue !== null &&+ this.storeReturnvalueInline() &&+ !(meta.f & F_ZDR)+ ) {+ const buf = Buffer.from(JSON.stringify(returnvalue), "utf8");+ chunkBuffer(buf, DATA_CHUNK_BYTES).forEach((chunk, ci) =>+ tn.set(ks.jobReturnvalue(id, ci), chunk),+ );+ }+ } else {+ tn.set(+ ks.jobFailedReason(id),+ Buffer.from((failedReason ?? "").slice(0, DATA_CHUNK_BYTES), "utf8"),+ );+ }++ // shed job data early on cloud (results live in GCS) and for ZDR+ if (config.GCS_BUCKET_NAME || meta.f & F_ZDR) {+ const r = ks.jobDataRange(id);+ tn.clearRange(r.begin, r.end);+ }++ if (meta.g && meta.f & F_GACC && this.groupOps) {+ await this.groupOps.terminalAccounting(+ tn,+ meta.g,+ id,+ "active",+ outcome,+ !!(meta.f & F_COUNTABLE),+ now,+ txc,+ );+ }++ const entry: QueueEntry = {+ i: id,+ o: meta.o,+ g: meta.g,+ p: meta.p,+ f: meta.f,+ c: meta.c,+ to: meta.to,+ };+ await releaseSlotsAndPromote(+ tn,+ ks,+ entry,+ { team: true, crawl: true },+ now,+ txc,+ );++ if (!meta.g) {+ const retention =+ outcome === "completed"+ ? COMPLETED_STANDALONE_RETENTION_MS+ : FAILED_STANDALONE_RETENTION_MS;+ tn.set(ks.jobExpiry(timeBucket(id), now + retention, id), EMPTY);+ }+ return true;+ });+ if (ok) this.leaseExps.delete(id);+ return ok;+ }++ // === Reads++ private async readJob(+ tn: Transaction,+ id: string,+ ): Promise<NuQFdbJob<JobData, JobReturnValue> | null> {+ const ks = this.ks;+ const snap = tn.snapshot();+ const [metaBuf, stBuf] = await Promise.all([+ snap.get(ks.jobMeta(id)),+ snap.get(ks.jobStatus(id)),+ ]);+ const meta = decodeJson<JobMeta>(metaBuf);+ const st = decodeJson<JobStatusRecord>(stBuf);+ if (!meta || !st) return null;+ const status = externalStatus(st.s);+ if (status === null) return null;++ const dataRange = ks.jobDataRange(id);+ const dataParts = await snap.getRangeAll(dataRange.begin, dataRange.end);+ const data =+ dataParts.length > 0+ ? JSON.parse(+ Buffer.concat(dataParts.map(([, v]) => v as Buffer)).toString(+ "utf8",+ ),+ )+ : null;++ let returnvalue: any = undefined;+ if (st.s === "completed") {+ const rvRange = ks.jobReturnvalueRange(id);+ const rvParts = await snap.getRangeAll(rvRange.begin, rvRange.end);+ if (rvParts.length > 0) {+ returnvalue = JSON.parse(+ Buffer.concat(rvParts.map(([, v]) => v as Buffer)).toString("utf8"),+ );+ } else {+ returnvalue = null;+ }+ }+ let failedReason: string | undefined = undefined;+ if (st.s === "failed") {+ const frBuf = await snap.get(ks.jobFailedReason(id));+ failedReason = frBuf ? frBuf.toString("utf8") : undefined;+ }++ return {+ id,+ status,+ createdAt: new Date(meta.c),+ priority: meta.p,+ data,+ finishedAt: st.fa !== undefined ? new Date(st.fa) : undefined,+ returnvalue,+ failedReason,+ lock: st.s === "active" ? st.l : undefined,+ ownerId: meta.o || undefined,+ groupId: meta.g,+ };+ }++ public async getJob(+ id: string,+ logger: Logger = _logger,+ ): Promise<NuQFdbJob<JobData, JobReturnValue> | null> {+ return await this.db.doTn(async tn => this.readJob(tn, id));+ }++ // cheap existence probe used by the dual-backend router+ public async hasJob(id: string): Promise<boolean> {+ return await this.db.doTn(async tn => {+ const st = await tn.snapshot().get(this.ks.jobStatus(id));+ return st !== undefined && st !== null;+ });+ }++ public async getJobs(+ ids: string[],+ logger: Logger = _logger,+ ): Promise<NuQFdbJob<JobData, JobReturnValue>[]> {+ if (ids.length === 0) return [];+ const out: NuQFdbJob<JobData, JobReturnValue>[] = [];+ const BATCH = 100;+ for (let i = 0; i < ids.length; i += BATCH) {+ const batch = ids.slice(i, i + BATCH);+ const jobs = await this.db.doTn(async tn =>+ Promise.all(batch.map(id => this.readJob(tn, id))),+ );+ out.push(+ ...jobs.filter(+ (j): j is NuQFdbJob<JobData, JobReturnValue> => j !== null,+ ),+ );+ }+ return out;+ }++ public async getJobsWithStatus(+ ids: string[],+ status: NuQJobStatusCompat,+ logger: Logger = _logger,+ ): Promise<NuQFdbJob<JobData, JobReturnValue>[]> {+ return (await this.getJobs(ids, logger)).filter(j => j.status === status);+ }++ public async getJobsWithStatuses(+ ids: string[],+ statuses: NuQJobStatusCompat[],+ logger: Logger = _logger,+ ): Promise<NuQFdbJob<JobData, JobReturnValue>[]> {+ const set = new Set(statuses);+ return (await this.getJobs(ids, logger)).filter(j => set.has(j.status));+ }++ // === Remove++ public async removeJob(+ id: string,+ logger: Logger = _logger,+ ): Promise<boolean> {+ const ks = this.ks;+ return await this.db.doTn(async tn => {+ const txc = newTxContext();+ const now = Date.now();+ const st = decodeJson<JobStatusRecord>(await tn.get(ks.jobStatus(id)));+ if (!st || st.s === "cancelled") return false;+ const meta = decodeJson<JobMeta>(await tn.get(ks.jobMeta(id)));+ if (!meta) return false;+ const entry: QueueEntry = {+ i: id,+ o: meta.o,+ g: meta.g,+ p: meta.p,+ f: meta.f,+ c: meta.c,+ to: meta.to,+ };+ const countable = !!(meta.f & F_COUNTABLE);+ const accounted = !!(meta.f & F_GACC) && !!meta.g && !!this.groupOps;++ if (st.s === "pending") {+ clearPendingPlacement(tn, ks, id, meta.o, meta.g, st.loc!, meta.to);+ // team-pending and delayed jobs hold a crawl slot+ if (st.loc!.k !== "gq") {+ await releaseSlotsAndPromote(+ tn,+ ks,+ entry,+ { team: false, crawl: true },+ now,+ txc,+ );+ }+ if (accounted) {+ tn.clear(ks.groupJob(meta.g!, id));+ tn.add(ks.groupRemaining(meta.g!), MINUS_ONE);+ if (countable) bumpGroupStatusCount(tn, ks, meta.g!, "pending", -1);+ tn.set(ks.taskGroupFinish(meta.g!), EMPTY);+ }+ deleteJobRecords(tn, ks, id);+ } else if (st.s === "queued" || st.s === "active") {+ tn.set(+ ks.jobStatus(id),+ encodeJson({ s: "cancelled", st: st.st } satisfies JobStatusRecord),+ );+ if (st.s === "active" && st.e !== undefined) {+ tn.clear(ks.lease(timeBucket(id), st.e, id));+ }+ await releaseSlotsAndPromote(+ tn,+ ks,+ entry,+ { team: true, crawl: true },+ now,+ txc,+ );+ if (accounted) {+ tn.clear(ks.groupJob(meta.g!, id));+ tn.add(ks.groupRemaining(meta.g!), MINUS_ONE);+ if (countable) bumpGroupStatusCount(tn, ks, meta.g!, st.s, -1);+ tn.set(ks.taskGroupFinish(meta.g!), EMPTY);+ }+ // status tombstone stays for take-side dedupe; sweeper GCs the records+ tn.set(+ ks.jobExpiry(+ timeBucket(id),+ now + COMPLETED_STANDALONE_RETENTION_MS,+ id,+ ),+ EMPTY,+ );+ } else {+ // terminal: drop the records, like the PG row delete+ if (accounted) {+ tn.clear(ks.groupJob(meta.g!, id));+ if (countable) bumpGroupStatusCount(tn, ks, meta.g!, st.s, -1);+ }+ deleteJobRecords(tn, ks, id);+ }+ return true;+ });+ }++ public async removeJobs(+ ids: string[],+ logger: Logger = _logger,+ ): Promise<void> {+ for (const id of ids) {+ await this.removeJob(id, logger);+ }+ }++ // === waitForJob++ public async waitForJob(+ id: string,+ timeout: number | null,+ logger: Logger = _logger,+ ): Promise<JobReturnValue> {+ const ks = this.ks;+ const deadline = timeout !== null ? Date.now() + timeout : null;+ while (true) {+ const { st, watch } = await this.db.doTn(async tn => {+ const stBuf = await tn.get(ks.jobStatus(id));+ const st = decodeJson<JobStatusRecord>(stBuf);+ if (+ st &&+ (st.s === "completed" || st.s === "failed" || st.s === "cancelled")+ ) {+ return { st, watch: null };+ }+ return { st, watch: tn.watch(ks.jobStatus(id)) };+ });++ if (!st) {+ if (watch) watch.cancel();+ throw new Error("Job raced out while waiting for it");+ }++ if (st.s === "completed") {+ if (watch) watch.cancel();+ const job = await this.getJob(id, logger);+ if (!job) throw new Error("Job raced out while waiting for it");+ return job.returnvalue!;+ }+ if (st.s === "failed") {+ if (watch) watch.cancel();+ const job = await this.getJob(id, logger);+ throw new Error(job?.failedReason ?? "Job failed");+ }+ if (st.s === "cancelled") {+ if (watch) watch.cancel();+ throw new Error("Job raced out while waiting for it");+ }++ const remaining = deadline !== null ? deadline - Date.now() : null;+ if (remaining !== null && remaining <= 0) {+ watch!.cancel();+ throw new Error("Timed out");+ }++ const fired = await Promise.race([+ watch!.promise.then(() => true),+ new Promise<false>(resolve =>+ setTimeout(+ () => resolve(false),+ remaining !== null ? Math.min(remaining, 30_000) : 30_000,+ ),+ ),+ ]);+ if (!fired) {+ watch!.cancel();+ if (deadline !== null && Date.now() >= deadline) {+ throw new Error("Timed out");+ }+ }+ }+ }++ // === Group-scoped reads++ public async getGroupNumericStats(+ groupId: string,+ logger: Logger = _logger,+ ): Promise<Record<NuQJobStatusCompat, number>> {+ const ks = this.ks;+ return await this.db.doTn(async tn => {+ const r = ks.groupStatusCountRange(groupId);+ const counts = await tn.snapshot().getRangeAll(r.begin, r.end);+ const out: Record<NuQJobStatusCompat, number> = {+ queued: 0,+ active: 0,+ completed: 0,+ failed: 0,+ backlog: 0,+ };+ for (const [key, value] of counts) {+ const status = ks.unpackId(key as Buffer);+ const n = Math.max(0, decodeI64(value as Buffer));+ if (status === "pending") out.backlog += n;+ else if (status in out) out[status as NuQJobStatusCompat] += n;+ }+ return out;+ });+ }++ public async getGroupAnyJob(+ groupId: string,+ ownerId: string,+ logger: Logger = _logger,+ ): Promise<NuQFdbJob<JobData, JobReturnValue> | null> {+ const ks = this.ks;+ const owner = normalizeOwnerId(ownerId);+ const candidateId = await this.db.doTn(async tn => {+ const r = ks.groupJobRange(groupId);+ let begin: Buffer | FdbKeySelector = r.begin;+ // scan for the first countable (single_urls) member+ for (let scanned = 0; scanned < 2000; ) {+ const batch = await tn+ .snapshot()+ .getRangeAll(begin as any, r.end, { limit: 200 });+ if (batch.length === 0) return null;+ for (const [key, value] of batch) {+ const gj = decodeJson<GroupJobIndexValue>(value as Buffer);+ if (gj?.m === 1) return ks.unpackId(key as Buffer);+ }+ scanned += batch.length;+ const lastKey = batch[batch.length - 1][0] as Buffer;+ begin = {+ key: lastKey,+ orEqual: true,+ offset: 1,+ _isKeySelector: true,+ };+ }+ return null;+ });+ if (!candidateId) return null;+ const job = await this.getJob(candidateId, logger);+ if (!job) return null;+ if (owner !== null && job.ownerId !== owner) return null;+ return job;+ }++ public async getCrawlJobsForListing(+ groupId: string,+ limit: number,+ offset: number,+ logger: Logger = _logger,+ ): Promise<NuQFdbJob<JobData, JobReturnValue>[]> {+ const ks = this.ks;+ const ids = await this.db.doTn(async tn => {+ const r = ks.groupDoneRange(groupId);+ const rows = await tn+ .snapshot()+ .getRangeAll(r.begin, r.end, { limit: offset + limit });+ return rows+ .slice(offset)+ .map(([, value]) => (value as Buffer).toString("utf8"));+ });+ const jobs = await this.getJobs(ids, logger);+ const byId = new Map(jobs.map(j => [j.id, j]));+ return ids+ .map(id => byId.get(id))+ .filter(+ (j): j is NuQFdbJob<JobData, JobReturnValue> =>+ !!j && j.status === "completed",+ );+ }++ // === Introspection used by status/admin endpoints++ public async getTeamActiveCount(teamId: string): Promise<number> {+ const owner = normalizeOwnerId(teamId);+ if (owner === null) return 0;+ return await this.db.doTn(async tn =>+ Math.max(+ 0,+ decodeI64(await tn.snapshot().get(this.ks.teamActive(owner))),+ ),+ );+ }++ public async getTeamActiveCounts(): Promise<Map<string, number>> {+ const ks = this.ks;+ return await this.db.doTn(async tn => {+ const r = ks.teamActiveIndexRange();+ const rows = await tn.snapshot().getRangeAll(r.begin, r.end);+ const counts = new Map<string, number>();+ for (const [key, value] of rows) {+ const parts = ks.unpack(key as Buffer);+ const teamId = parts[2];+ if (typeof teamId !== "string") continue;+ const count = Math.max(0, decodeI64(value as Buffer));+ if (count > 0) counts.set(teamId, count);+ }+ return counts;+ });+ }++ public async getTeamPendingCount(teamId: string): Promise<number> {+ const owner = normalizeOwnerId(teamId);+ if (owner === null) return 0;+ return await this.db.doTn(async tn =>+ Math.max(+ 0,+ decodeI64(await tn.snapshot().get(this.ks.teamPendingCount(owner))),+ ),+ );+ }+}++class LockLostError extends Error {}diff --git a/apps/api/src/services/worker/nuq-fdb/slots.ts b/apps/api/src/services/worker/nuq-fdb/slots.tsnew file mode 100644index 000000000..f85d30a5e--- /dev/null+++ b/apps/api/src/services/worker/nuq-fdb/slots.ts@@ -0,0 +1,136 @@+import type { Transaction } from "foundationdb";+import { getNuqFdbDatabase } from "./client";+import {+ NuqFdbKeyspace,+ QueueEntry,+ decodeJson,+ encodeJson,+ timeBucket,+ F_GATED,+ normalizeOwnerId,+} from "./keyspace";+import { bumpTeamActive, newTxContext, releaseSlotsAndPromote } from "./ops";++// External slots: capacity consumed by things that are not queue jobs (sync+// scrapes via the team semaphore, browser sessions). They unconditionally bump+// the team active counter -- possibly past the limit, matching the old Redis+// behavior where sync holders were mirrored into the same ZSET -- and hand+// their slot through the normal promotion chain on release.++type ExternalSlotRecord = {+ e: number; // expiry ms+};++export class NuqFdbExternalSlots {+ constructor(public readonly ks: NuqFdbKeyspace) {}++ private get db() {+ return getNuqFdbDatabase();+ }++ private key(teamId: string, holderId: string): Buffer {+ return this.ks.pack(["xs", teamId, holderId]);+ }++ private expiryKey(bucket: number, expMs: number, holderId: string): Buffer {+ return this.ks.pack(["xsexp", bucket, expMs, holderId]);+ }++ public expiryScanRange(bucket: number, untilMs: number) {+ return {+ begin: this.ks.pack(["xsexp", bucket]),+ end: this.ks.pack(["xsexp", bucket, untilMs]),+ };+ }++ // Acquires (or renews) an external slot. Unconditional: never blocks on the+ // team limit; the caller's own gate (Lua semaphore, session limits) decides+ // admission. Re-acquiring an existing holder just extends its expiry.+ public async acquire(+ teamId: string,+ holderId: string,+ ttlMs: number,+ ): Promise<void> {+ const owner = normalizeOwnerId(teamId);+ if (owner === null) return;+ const now = Date.now();+ const exp = now + ttlMs;+ await this.db.doTn(async tn => {+ const existing = decodeJson<ExternalSlotRecord>(+ await tn.get(this.key(owner, holderId)),+ );+ if (existing) {+ tn.clear(this.expiryKey(timeBucket(holderId), existing.e, holderId));+ } else {+ bumpTeamActive(tn, this.ks, owner, 1);+ }+ tn.set(+ this.key(owner, holderId),+ encodeJson({ e: exp } satisfies ExternalSlotRecord),+ );+ tn.set(+ this.expiryKey(timeBucket(holderId), exp, holderId),+ encodeJson({ t: owner }),+ );+ });+ }++ // Releases the slot, handing it to a pending job when one exists.+ public async release(teamId: string, holderId: string): Promise<void> {+ const owner = normalizeOwnerId(teamId);+ if (owner === null) return;+ await this.db.doTn(async tn => {+ await this.releaseInTxn(tn, owner, holderId);+ });+ }++ public async releaseInTxn(+ tn: Transaction,+ owner: string,+ holderId: string,+ ): Promise<boolean> {+ const existing = decodeJson<ExternalSlotRecord>(+ await tn.get(this.key(owner, holderId)),+ );+ if (!existing) return false;+ tn.clear(this.key(owner, holderId));+ tn.clear(this.expiryKey(timeBucket(holderId), existing.e, holderId));+ const entry: QueueEntry = {+ i: holderId,+ o: owner,+ p: 0,+ f: F_GATED,+ c: 0,+ };+ await releaseSlotsAndPromote(+ tn,+ this.ks,+ entry,+ { team: true, crawl: false },+ Date.now(),+ newTxContext(),+ );+ return true;+ }++ // Sweeper hook: releases slots whose holders stopped renewing.+ public async sweepExpired(now: number, buckets: number): Promise<void> {+ for (let b = 0; b < buckets; b++) {+ const r = this.expiryScanRange(b, now);+ const due = await this.db.doTn(async tn =>+ tn.snapshot().getRangeAll(r.begin, r.end, { limit: 50 }),+ );+ for (const [key, value] of due) {+ const holderId = this.ks.unpackId(key as Buffer);+ const rec = decodeJson<{ t: string }>(value as Buffer);+ if (!rec) continue;+ await this.db.doTn(async tn => {+ // releaseInTxn validates the record still exists (and clears this+ // index entry); if the holder renewed, just drop the stale entry+ const released = await this.releaseInTxn(tn, rec.t, holderId);+ if (!released) tn.clear(key as Buffer);+ });+ }+ }+ }+}diff --git a/apps/api/src/services/worker/nuq-fdb/sweeper.ts b/apps/api/src/services/worker/nuq-fdb/sweeper.tsnew file mode 100644index 000000000..803064a9a--- /dev/null+++ b/apps/api/src/services/worker/nuq-fdb/sweeper.ts@@ -0,0 +1,611 @@+import { randomUUID } from "crypto";+import { Logger } from "winston";+import { logger as _logger } from "../../../lib/logger";+import { getNuqFdbDatabase } from "./client";+import {+ NuqFdbKeyspace,+ JobMeta,+ JobStatusRecord,+ GroupMeta,+ QueueEntry,+ decodeI64,+ decodeJson,+ encodeJson,+ timeBucket,+ TIME_BUCKETS,+ F_GATED,+ F_CRAWL_GATED,+ F_COUNTABLE,+ F_GACC,+} from "./keyspace";+import {+ ONE,+ MINUS_ONE,+ EMPTY,+ MAX_STALLS,+ FAILED_STANDALONE_RETENTION_MS,+ newTxContext,+ pushReady,+ setStatusQueued,+ promoteEntryToReady,+ clearPendingPlacement,+ releaseSlotsAndPromote,+ admitThroughTeamGate,+ deleteJobRecords,+ popTeamPending,+ setGroupJobIndex,+ bumpGroupStatusCount,+ bumpTeamActive,+ GroupJobIndexValue,+} from "./ops";+import { NuQFdbQueue } from "./queue";+import { NuqFdbExternalSlots } from "./slots";++const SWEEP_LOCK_TTL_MS = 15_000;+const SWEEP_BATCH = 50;+const STALL_FAILED_REASON = "Job stalled too many times";++type SweepLagStats = {+ dueCount: number;+ processedCount: number;+ oldestOverdueAgeMs: number;+ saturatedBucketCount: number;+ durationMs: number;+};++function keyAfter(key: Buffer): Buffer {+ return Buffer.concat([key, Buffer.from([0])]);+}++function entryFromMeta(id: string, meta: JobMeta): QueueEntry {+ return {+ i: id,+ o: meta.o,+ g: meta.g,+ p: meta.p,+ f: meta.f,+ c: meta.c,+ to: meta.to,+ };+}++function emptySweepLagStats(): SweepLagStats {+ return {+ dueCount: 0,+ processedCount: 0,+ oldestOverdueAgeMs: 0,+ saturatedBucketCount: 0,+ durationMs: 0,+ };+}++function addDueKeysToLagStats(+ stats: SweepLagStats,+ ks: NuqFdbKeyspace,+ due: [unknown, unknown][],+ now: number,+): void {+ stats.dueCount += due.length;+ if (due.length >= SWEEP_BATCH) stats.saturatedBucketCount++;+ for (const [key] of due) {+ const dueAt = Number(ks.unpackId(key as Buffer, 1));+ if (Number.isFinite(dueAt)) {+ stats.oldestOverdueAgeMs = Math.max(+ stats.oldestOverdueAgeMs,+ now - dueAt,+ );+ }+ }+}++function logSweepLag(+ logger: Logger,+ queue: NuQFdbQueue,+ index: "lease" | "backlog_timeout" | "delay",+ stats: SweepLagStats,+): void {+ if (stats.dueCount === 0 && stats.saturatedBucketCount === 0) return;+ logger[stats.saturatedBucketCount > 0 ? "warn" : "debug"](+ "NuQ FDB sweeper lag",+ {+ canonicalLog: "nuq-fdb/sweeper_lag",+ queueName: queue.queueName,+ index,+ timeBuckets: TIME_BUCKETS,+ sweepBatch: SWEEP_BATCH,+ ...stats,+ },+ );+}++// One sweeper services all queues against the same FDB cluster. Each queue+// gets its own pass; a leased singleton lock (held on the first queue's+// keyspace) keeps multiple candidate processes from sweeping concurrently.+export class NuqFdbSweeper {+ private readonly sweeperId = randomUUID();+ private loop: NodeJS.Timeout | null = null;+ private running = false;++ constructor(+ public readonly queues: NuQFdbQueue[],+ public readonly externalSlots: NuqFdbExternalSlots[] = [],+ ) {}++ private get db() {+ return getNuqFdbDatabase();+ }++ private get lockKs(): NuqFdbKeyspace {+ return this.queues[0].ks;+ }++ public async tryAcquireLock(now: number = Date.now()): Promise<boolean> {+ return await this.db.doTn(async tn => {+ const rec = decodeJson<{ w: string; x: number }>(+ await tn.get(this.lockKs.sweeperLock()),+ );+ if (rec && rec.x > now && rec.w !== this.sweeperId) return false;+ tn.set(+ this.lockKs.sweeperLock(),+ encodeJson({ w: this.sweeperId, x: now + SWEEP_LOCK_TTL_MS }),+ );+ return true;+ });+ }++ // Runs one full sweep over all queues. Exposed for tests; production uses+ // start(), which wraps this in the singleton lock loop.+ public async sweepOnce(logger: Logger = _logger): Promise<void> {+ const now = Date.now();+ for (const queue of this.queues) {+ await this.sweepLeases(queue, now, logger);+ await this.sweepBacklogTimeouts(queue, now, logger);+ await this.sweepDelayed(queue, now, logger);+ await this.sweepGroupFinishTasks(queue, now, logger);+ await this.sweepGroupCancelTasks(queue, now, logger);+ await this.sweepTeamRaiseTasks(queue, now, logger);+ await this.sweepJobExpiry(queue, now, logger);+ await this.sweepGroupExpiry(queue, now, logger);+ }+ for (const slots of this.externalSlots) {+ await slots.sweepExpired(now, TIME_BUCKETS);+ }+ }++ public start(intervalMs: number = 1000, logger: Logger = _logger): void {+ if (this.loop) return;+ this.loop = setInterval(async () => {+ if (this.running) return;+ this.running = true;+ try {+ if (await this.tryAcquireLock()) {+ await this.sweepOnce(logger);+ }+ } catch (error) {+ logger.warn("NuQ FDB sweeper tick failed", {+ module: "nuq-fdb/sweeper",+ error,+ });+ } finally {+ this.running = false;+ }+ }, intervalMs);+ }++ public stop(): void {+ if (this.loop) {+ clearInterval(this.loop);+ this.loop = null;+ }+ }++ // === Lease expiry: requeue stalled jobs, fail them after MAX_STALLS++ private async sweepLeases(+ queue: NuQFdbQueue,+ now: number,+ logger: Logger,+ ): Promise<void> {+ const startedAt = Date.now();+ const ks = queue.ks;+ const stats = emptySweepLagStats();+ for (let b = 0; b < TIME_BUCKETS; b++) {+ const r = ks.leaseScanRange(b, now);+ const due = await this.db.doTn(async tn =>+ tn.snapshot().getRangeAll(r.begin, r.end, { limit: SWEEP_BATCH }),+ );+ addDueKeysToLagStats(stats, ks, due, now);+ for (const [key, value] of due) {+ const id = ks.unpackId(key as Buffer);+ const lease = decodeJson<{ l: string }>(value as Buffer);+ await this.db.doTn(async tn => {+ const txc = newTxContext();+ const st = decodeJson<JobStatusRecord>(+ await tn.get(ks.jobStatus(id)),+ );+ // stale entries: job moved on (renewal, finish) or was reaped already+ if (!st || st.s !== "active" || st.l !== lease?.l) {+ tn.clear(key as Buffer);+ return;+ }+ if (st.e !== undefined && st.e > now) {+ // renewed after our snapshot; the old index entry is what expired+ tn.clear(key as Buffer);+ return;+ }+ const meta = decodeJson<JobMeta>(await tn.get(ks.jobMeta(id)));+ tn.clear(key as Buffer);+ if (!meta) return;+ const entry = entryFromMeta(id, meta);++ if (st.st < MAX_STALLS) {+ // requeue directly to ready -- the job retains its slots+ pushReady(tn, ks, entry, txc);+ setStatusQueued(tn, ks, id, st.st + 1);+ if (meta.g && meta.f & F_COUNTABLE) {+ bumpGroupStatusCount(tn, ks, meta.g, "active", -1);+ bumpGroupStatusCount(tn, ks, meta.g, "queued", 1);+ }+ } else {+ tn.set(+ ks.jobStatus(id),+ encodeJson({+ s: "failed",+ st: st.st,+ fa: now,+ } satisfies JobStatusRecord),+ );+ tn.set(+ ks.jobFailedReason(id),+ Buffer.from(STALL_FAILED_REASON, "utf8"),+ );+ if (meta.g && meta.f & F_GACC && queue.groupOps) {+ await queue.groupOps.terminalAccounting(+ tn,+ meta.g,+ id,+ "active",+ "failed",+ !!(meta.f & F_COUNTABLE),+ now,+ txc,+ );+ }+ await releaseSlotsAndPromote(+ tn,+ ks,+ entry,+ { team: true, crawl: true },+ now,+ txc,+ );+ if (!meta.g) {+ tn.set(+ ks.jobExpiry(+ timeBucket(id),+ now + FAILED_STANDALONE_RETENTION_MS,+ id,+ ),+ EMPTY,+ );+ }+ }+ });+ stats.processedCount++;+ }+ }+ stats.durationMs = Date.now() - startedAt;+ logSweepLag(logger, queue, "lease", stats);+ }++ // === Backlog timeouts: silently drop pending jobs past their deadline++ private async sweepBacklogTimeouts(+ queue: NuQFdbQueue,+ now: number,+ logger: Logger,+ ): Promise<void> {+ const startedAt = Date.now();+ const ks = queue.ks;+ const stats = emptySweepLagStats();+ for (let b = 0; b < TIME_BUCKETS; b++) {+ const r = ks.backlogTimeoutScanRange(b, now);+ const due = await this.db.doTn(async tn =>+ tn.snapshot().getRangeAll(r.begin, r.end, { limit: SWEEP_BATCH }),+ );+ addDueKeysToLagStats(stats, ks, due, now);+ for (const [key] of due) {+ const id = ks.unpackId(key as Buffer);+ await this.db.doTn(async tn => {+ const txc = newTxContext();+ tn.clear(key as Buffer);+ const st = decodeJson<JobStatusRecord>(+ await tn.get(ks.jobStatus(id)),+ );+ if (!st || st.s !== "pending" || !st.loc) return;+ const meta = decodeJson<JobMeta>(await tn.get(ks.jobMeta(id)));+ if (!meta) return;+ clearPendingPlacement(tn, ks, id, meta.o, meta.g, st.loc, meta.to);+ if (st.loc.k !== "gq" && meta.f & F_CRAWL_GATED) {+ await releaseSlotsAndPromote(+ tn,+ ks,+ entryFromMeta(id, meta),+ { team: false, crawl: true },+ now,+ txc,+ );+ }+ if (meta.g && meta.f & F_GACC && queue.groupOps) {+ tn.clear(ks.groupJob(meta.g, id));+ tn.add(ks.groupRemaining(meta.g), MINUS_ONE);+ if (meta.f & F_COUNTABLE)+ bumpGroupStatusCount(tn, ks, meta.g, "pending", -1);+ tn.set(ks.taskGroupFinish(meta.g), EMPTY);+ }+ deleteJobRecords(tn, ks, id);+ });+ stats.processedCount++;+ }+ }+ stats.durationMs = Date.now() - startedAt;+ logSweepLag(logger, queue, "backlog_timeout", stats);+ }++ // === Delayed (crawl delay) promotions++ private async sweepDelayed(+ queue: NuQFdbQueue,+ now: number,+ logger: Logger,+ ): Promise<void> {+ const startedAt = Date.now();+ const ks = queue.ks;+ const stats = emptySweepLagStats();+ for (let b = 0; b < TIME_BUCKETS; b++) {+ const r = ks.delayedScanRange(b, now);+ const due = await this.db.doTn(async tn =>+ tn.snapshot().getRangeAll(r.begin, r.end, { limit: SWEEP_BATCH }),+ );+ addDueKeysToLagStats(stats, ks, due, now);+ for (const [key, value] of due) {+ const e = decodeJson<QueueEntry>(value as Buffer);+ if (!e) continue;+ await this.db.doTn(async tn => {+ const txc = newTxContext();+ const st = decodeJson<JobStatusRecord>(+ await tn.get(ks.jobStatus(e.i)),+ );+ tn.clear(key as Buffer);+ if (!st || st.s !== "pending" || st.loc?.k !== "dl") return;+ // the job already holds its crawl slot; admit through the team gate+ await admitThroughTeamGate(tn, ks, e, txc);+ });+ stats.processedCount++;+ }+ }+ stats.durationMs = Date.now() - startedAt;+ logSweepLag(logger, queue, "delay", stats);+ }++ // === Group finish detection (backstop for the inline path)++ private async sweepGroupFinishTasks(+ queue: NuQFdbQueue,+ now: number,+ logger: Logger,+ ): Promise<void> {+ if (!queue.groupOps) return;+ const ks = queue.ks;+ const r = ks.taskGroupFinishRange();+ const tasks = await this.db.doTn(async tn =>+ tn.snapshot().getRangeAll(r.begin, r.end, { limit: 200 }),+ );+ for (const [key] of tasks) {+ const gid = ks.unpackId(key as Buffer);+ await this.db.doTn(async tn => {+ const txc = newTxContext();+ // normal read so a concurrent finisher's decrement forces a retry --+ // clearing the task may not race with the group draining to zero+ const rem = decodeI64(await tn.get(ks.groupRemaining(gid)));+ if (rem > 0) {+ tn.clear(key as Buffer);+ return;+ }+ await queue.groupOps!.tryCompleteGroup(tn, gid, now, txc);+ });+ }+ }++ // === Lazy group cancellation cleanup++ private async sweepGroupCancelTasks(+ queue: NuQFdbQueue,+ now: number,+ logger: Logger,+ ): Promise<void> {+ if (!queue.groupOps) return;+ const ks = queue.ks;+ const r = ks.taskGroupCancelRange();+ const tasks = await this.db.doTn(async tn =>+ tn.snapshot().getRangeAll(r.begin, r.end, { limit: 20 }),+ );+ for (const [key] of tasks) {+ const gid = ks.unpackId(key as Buffer);+ let exhausted = false;+ let begin: Buffer | null = null;+ // clean pending members in batches until none remain+ for (let rounds = 0; rounds < 50 && !exhausted; rounds++) {+ const result = await this.db.doTn(async tn => {+ const jr = ks.groupJobRange(gid);+ const rangeBegin = begin ?? jr.begin;+ const members = await tn+ .snapshot()+ .getRangeAll(rangeBegin, jr.end, { limit: 500 });+ let cleaned = 0;+ for (const [mKey, mValue] of members) {+ const gj = decodeJson<GroupJobIndexValue>(mValue as Buffer);+ if (!gj || gj.s !== "pending") continue;+ if (cleaned >= SWEEP_BATCH) {+ return { exhausted: false, nextBegin: rangeBegin };+ }+ const id = ks.unpackId(mKey as Buffer);+ const st = decodeJson<JobStatusRecord>(+ await tn.get(ks.jobStatus(id)),+ );+ if (!st || st.s !== "pending" || !st.loc) {+ // moved on; fix the index lazily+ continue;+ }+ const meta = decodeJson<JobMeta>(await tn.get(ks.jobMeta(id)));+ if (!meta) continue;+ clearPendingPlacement(tn, ks, id, meta.o, meta.g, st.loc, meta.to);+ // team-pending/delayed members hold a crawl slot; the group is+ // cancelled so there is nothing to promote -- just release it+ if (st.loc.k !== "gq" && meta.f & F_CRAWL_GATED) {+ tn.add(ks.groupCrawlActive(gid), MINUS_ONE);+ }+ tn.clear(mKey as Buffer);+ tn.add(ks.groupRemaining(gid), MINUS_ONE);+ if (meta.f & F_COUNTABLE)+ bumpGroupStatusCount(tn, ks, gid, "pending", -1);+ deleteJobRecords(tn, ks, id);+ cleaned++;+ }+ const lastKey = members[members.length - 1]?.[0] as+ | Buffer+ | undefined;+ return {+ exhausted: members.length < 500,+ nextBegin: lastKey ? keyAfter(lastKey) : jr.end,+ };+ });+ exhausted = result.exhausted;+ begin = result.nextBegin;+ }+ if (exhausted) {+ await this.db.doTn(async tn => {+ tn.clear(key as Buffer);+ tn.set(ks.taskGroupFinish(gid), EMPTY);+ });+ }+ }+ }++ // === Limit raises: drain newly-available slots++ private async sweepTeamRaiseTasks(+ queue: NuQFdbQueue,+ now: number,+ logger: Logger,+ ): Promise<void> {+ const ks = queue.ks;+ const r = ks.taskTeamRaiseRange();+ const tasks = await this.db.doTn(async tn =>+ tn.snapshot().getRangeAll(r.begin, r.end, { limit: 50 }),+ );+ for (const [key] of tasks) {+ const tid = ks.unpackId(key as Buffer);+ const done = await this.db.doTn(async tn => {+ const txc = newTxContext();+ const limitBuf = await tn.get(ks.teamLimit(tid));+ const limit = limitBuf ? decodeI64(limitBuf) : Infinity;+ const active = decodeI64(await tn.get(ks.teamActive(tid)));+ let free = Math.min(Math.max(0, limit - active), 32);+ let promoted = 0;+ while (free > 0) {+ const e = await popTeamPending(tn, ks, tid);+ if (!e) break;+ promoteEntryToReady(tn, ks, e, txc);+ promoted++;+ free--;+ }+ if (promoted > 0) bumpTeamActive(tn, ks, tid, promoted);+ // done when no free slots remain or the pending queue is drained+ return free > 0 || limit - active <= 0;+ });+ if (done) {+ await this.db.doTn(async tn => tn.clear(key as Buffer));+ }+ }+ }++ // === Record GC++ private async sweepJobExpiry(+ queue: NuQFdbQueue,+ now: number,+ logger: Logger,+ ): Promise<void> {+ const ks = queue.ks;+ for (let b = 0; b < TIME_BUCKETS; b++) {+ const r = ks.jobExpiryScanRange(b, now);+ const due = await this.db.doTn(async tn =>+ tn.snapshot().getRangeAll(r.begin, r.end, { limit: SWEEP_BATCH * 2 }),+ );+ if (due.length === 0) continue;+ await this.db.doTn(async tn => {+ for (const [key] of due) {+ const id = ks.unpackId(key as Buffer);+ tn.clear(key as Buffer);+ const st = decodeJson<JobStatusRecord>(+ await tn.get(ks.jobStatus(id)),+ );+ if (+ st &&+ (st.s === "completed" || st.s === "failed" || st.s === "cancelled")+ ) {+ deleteJobRecords(tn, ks, id);+ }+ }+ });+ }+ }++ private async sweepGroupExpiry(+ queue: NuQFdbQueue,+ now: number,+ logger: Logger,+ ): Promise<void> {+ if (!queue.groupOps) return;+ const ks = queue.ks;+ const r = ks.groupExpiryScanRange(now);+ const due = await this.db.doTn(async tn =>+ tn.snapshot().getRangeAll(r.begin, r.end, { limit: 20 }),+ );+ for (const [key] of due) {+ const gid = ks.unpackId(key as Buffer);+ // delete member job records in batches, then the group's own keyspace+ let drained = false;+ for (let rounds = 0; rounds < 200 && !drained; rounds++) {+ drained = await this.db.doTn(async tn => {+ const jr = ks.groupJobRange(gid);+ const members = await tn+ .snapshot()+ .getRangeAll(jr.begin, jr.end, { limit: 200 });+ for (const [mKey] of members) {+ const id = ks.unpackId(mKey as Buffer);+ deleteJobRecords(tn, ks, id);+ tn.clear(mKey as Buffer);+ }+ return members.length < 200;+ });+ }+ await this.db.doTn(async tn => {+ const g = decodeJson<GroupMeta>(await tn.get(ks.groupMeta(gid)));+ // the crawl-finished job for this group lives in the finished queue+ const fjobBuf = await tn.get(ks.groupFinishedJob(gid));+ if (fjobBuf && queue.groupOps!.finishedKs) {+ const fid = fjobBuf.toString("utf8");+ deleteJobRecords(tn, queue.groupOps!.finishedKs as any, fid);+ }+ const gr = ks.groupRange(gid);+ tn.clearRange(gr.begin, gr.end);+ if (g) tn.clear(ks.ongoingGroup(g.o, gid));+ tn.clear(ks.taskGroupFinish(gid));+ tn.clear(ks.taskGroupCancel(gid));+ tn.clear(key as Buffer);+ });+ }+ }+}diff --git a/apps/api/src/services/worker/nuq-router.ts b/apps/api/src/services/worker/nuq-router.tsnew file mode 100644index 000000000..92d8716a7--- /dev/null+++ b/apps/api/src/services/worker/nuq-router.ts@@ -0,0 +1,645 @@+import { Logger } from "winston";+import { logger as _logger } from "../../lib/logger";+import { config } from "../../config";+import { RateLimiterMode, ScrapeJobData } from "../../types";+import { getACUCTeam } from "../../controllers/auth";+import { redisEvictConnection } from "../../services/redis";+import { isSelfHosted } from "../../lib/deployment";+import {+ getTeamQueueLimit,+ getConcurrencyLimitActiveJobsCount,+ pushConcurrencyLimitActiveJob,+ removeConcurrencyLimitActiveJob,+} from "../../lib/concurrency-redis";+import {+ NuQJob,+ NuQJobStatus,+ NuQGroupStatus,+ NuQJobGroupInstance,+ scrapeQueue as scrapeQueuePg,+ crawlFinishedQueue as crawlFinishedQueuePg,+ crawlGroup as crawlGroupPg,+} from "./nuq";+import {+ scrapeQueueFdb,+ crawlFinishedQueueFdb,+ crawlGroupFdb,+ externalSlotsFdb,+ isFdbConfigured,+ NuQFdbQueue,+ NuQFdbJob,+} from "./nuq-fdb";++// Dual-backend router for the NuQ migration to FoundationDB. Exports the same+// `scrapeQueue` / `crawlFinishedQueue` / `crawlGroup` names as ./nuq so call+// sites only swap their import path. Routing rules:+// - new crawls: team flag (TeamFlags.nuqFdb) or NUQ_BACKEND=fdb decides; the+// choice is pinned in StoredCrawl.queueBackend so a crawl never spans+// backends+// - reads: probe FDB first (cheap point reads), fall back to PG+// - workers: dual-poll FDB first; in-flight jobs are tagged with their+// backend so renew/finish/fail route correctly++export type QueueBackend = "pg" | "fdb";++export type { NuQJob, NuQJobStatus, NuQGroupStatus, NuQJobGroupInstance };++export function fdbQueueEnabled(): boolean {+ return isFdbConfigured();+}++function fdbForced(): boolean {+ return config.NUQ_BACKEND === "fdb";+}++const fdbFallbackLastWarn = new Map<string, number>();++function logFdbFallback(+ logger: Logger,+ operation: string,+ error: unknown,+): void {+ const now = Date.now();+ const lastWarn = fdbFallbackLastWarn.get(operation) ?? 0;+ if (now - lastWarn < 60_000) return;+ fdbFallbackLastWarn.set(operation, now);+ logger.warn("FDB queue operation failed, falling back to PG", {+ module: "nuq-router",+ operation,+ error,+ });+}++// Whether NEW work for this team should go to FDB. Existing crawls follow+// their StoredCrawl.queueBackend marker instead.+export async function isFdbTeam(teamId: string | undefined): Promise<boolean> {+ if (!fdbQueueEnabled()) return false;+ if (fdbForced()) return true;+ if (!teamId) return false;+ try {+ const acuc = await getACUCTeam(teamId, false, true, RateLimiterMode.Crawl);+ return acuc?.flags?.nuqFdb === true;+ } catch (error) {+ _logger.warn("Failed to resolve nuqFdb team flag, defaulting to pg", {+ module: "nuq-router",+ teamId,+ error,+ });+ return false;+ }+}++export async function resolveNewGroupBackend(+ teamId: string,+): Promise<QueueBackend> {+ return (await isFdbTeam(teamId)) ? "fdb" : "pg";+}++// Reads only the queueBackend marker off the stored crawl. Deliberately not+// getCrawl() -- importing crawl-redis pulls the whole scraper tree in.+async function getCrawlQueueBackend(+ crawlId: string,+): Promise<QueueBackend | null> {+ const raw = await redisEvictConnection.get("crawl:" + crawlId);+ if (!raw) return null;+ try {+ const sc = JSON.parse(raw);+ return sc?.queueBackend === "fdb" ? "fdb" : sc ? "pg" : null;+ } catch {+ return null;+ }+}++// Which backend a job belongs to at enqueue time. Crawl jobs follow their+// crawl's pinned backend; standalone jobs follow the team flag.+export async function resolveJobBackend(+ data: ScrapeJobData,+): Promise<QueueBackend> {+ if (!fdbQueueEnabled()) return "pg";+ if (fdbForced()) return "fdb";+ if (data.crawl_id) {+ return (await getCrawlQueueBackend(data.crawl_id)) ?? "pg";+ }+ return (await isFdbTeam(data.team_id)) ? "fdb" : "pg";+}++function tagFdbJob<T extends object>(job: T): T & { backend: "fdb" } {+ (job as any).backend = "fdb";+ return job as T & { backend: "fdb" };+}++// === External capacity holders (browser sessions, sync scrapes)+//+// Non-queue work that occupies team capacity mirrors itself into whichever+// ledger the team runs on. Mismatched acquire/release pairs (flag flipped+// mid-hold) self-heal: Redis entries expire by score, FDB external slots are+// reaped by the sweeper.++export async function mirrorExternalSlotAcquire(+ teamId: string,+ holderId: string,+ ttlMs: number,+): Promise<void> {+ if (await isFdbTeam(teamId)) {+ await externalSlotsFdb.acquire(teamId, holderId, ttlMs);+ } else {+ await pushConcurrencyLimitActiveJob(teamId, holderId, ttlMs);+ }+}++export async function mirrorExternalSlotRelease(+ teamId: string,+ holderId: string,+): Promise<void> {+ if (await isFdbTeam(teamId)) {+ await externalSlotsFdb.release(teamId, holderId);+ } else {+ await removeConcurrencyLimitActiveJob(teamId, holderId);+ }+}++// Active count across both ledgers; a migrating team has load on both while+// its old PG crawls drain.+export async function getCombinedTeamActiveCount(+ teamId: string,+): Promise<number> {+ const redisCount = await getConcurrencyLimitActiveJobsCount(teamId);+ if (!fdbQueueEnabled()) return redisCount;+ try {+ return redisCount + (await scrapeQueueFdb.getTeamActiveCount(teamId));+ } catch (error) {+ if (fdbForced()) throw error;+ logFdbFallback(_logger, "getCombinedTeamActiveCount", error);+ return redisCount;+ }+}++// === FDB enqueue (the whole gating block of queue-jobs collapses into this)++export function backlogTimeoutMsForGate(timeoutMs: number): Date {+ return new Date(Date.now() + timeoutMs);+}++export async function fdbEnqueueScrapeJobs(+ jobs: {+ jobId: string;+ data: ScrapeJobData;+ priority: number;+ listenable?: boolean;+ backlogTimeoutMs: number;+ }[],+ teamId: string,+ options?: { bypassGate?: boolean },+): Promise<{+ jobs: (NuQJob<ScrapeJobData> & { backend: "fdb" })[];+ backloggedCount: number;+ teamLimit: number | null;+}> {+ let teamLimit: number | null = null;+ if (!isSelfHosted() && !fdbForced()) {+ const acuc = await getACUCTeam(teamId, false, true, RateLimiterMode.Crawl);+ teamLimit = acuc?.concurrency ?? 2;+ } else if (!isSelfHosted()) {+ const acuc = await getACUCTeam(teamId, false, true, RateLimiterMode.Crawl);+ teamLimit = acuc?.concurrency ?? null;+ }++ const queueCap =+ teamLimit === null ? Number.MAX_SAFE_INTEGER : getTeamQueueLimit(teamLimit);++ const results = await scrapeQueueFdb.addJobs(+ jobs.map(j => ({+ id: j.jobId,+ data: j.data,+ options: {+ priority: j.priority,+ listenable: j.listenable ?? false,+ ownerId: j.data.team_id ?? undefined,+ groupId: j.data.crawl_id ?? undefined,+ bypassGate:+ options?.bypassGate ||+ j.data.mode === "kickoff" ||+ j.data.mode === "kickoff_sitemap",+ timesOutAt: new Date(Date.now() + j.backlogTimeoutMs),+ },+ })),+ { teamLimit, queueCap },+ );++ const tagged = results.map(r => tagFdbJob(r as NuQJob<ScrapeJobData>));+ return {+ jobs: tagged,+ backloggedCount: tagged.filter(j => j.status === "backlog").length,+ teamLimit,+ };+}++// === Routed scrape queue++class RoutedScrapeQueue {+ // in-flight jobs taken by THIS process, so renew/finish/fail can route+ private inflightBackend = new Map<string, QueueBackend>();++ private backendFor(id: string): QueueBackend {+ return this.inflightBackend.get(id) ?? "pg";+ }++ public async getJobToProcess(+ logger: Logger = _logger,+ ): Promise<NuQJob<ScrapeJobData> | null> {+ if (fdbQueueEnabled()) {+ try {+ const job = await scrapeQueueFdb.getJobToProcess(logger);+ if (job) {+ this.inflightBackend.set(job.id, "fdb");+ return tagFdbJob(job as NuQJob<ScrapeJobData>);+ }+ } catch (error) {+ if (fdbForced()) throw error;+ logFdbFallback(logger, "scrape.getJobToProcess", error);+ }+ }+ const job = await scrapeQueuePg.getJobToProcess();+ if (job) this.inflightBackend.set(job.id, "pg");+ return job;+ }++ public async renewLock(+ id: string,+ lock: string,+ logger: Logger = _logger,+ ): Promise<boolean> {+ if (this.backendFor(id) === "fdb") {+ return scrapeQueueFdb.renewLock(id, lock, logger);+ }+ return scrapeQueuePg.renewLock(id, lock, logger);+ }++ public async jobFinish(+ id: string,+ lock: string,+ returnvalue: any | null,+ logger: Logger = _logger,+ ): Promise<boolean> {+ const backend = this.backendFor(id);+ this.inflightBackend.delete(id);+ if (backend === "fdb") {+ return scrapeQueueFdb.jobFinish(id, lock, returnvalue, logger);+ }+ return scrapeQueuePg.jobFinish(id, lock, returnvalue, logger);+ }++ public async jobFail(+ id: string,+ lock: string,+ failedReason: string,+ logger: Logger = _logger,+ ): Promise<boolean> {+ const backend = this.backendFor(id);+ this.inflightBackend.delete(id);+ if (backend === "fdb") {+ return scrapeQueueFdb.jobFail(id, lock, failedReason, logger);+ }+ return scrapeQueuePg.jobFail(id, lock, failedReason, logger);+ }++ public async getJob(+ id: string,+ logger: Logger = _logger,+ ): Promise<NuQJob<ScrapeJobData> | null> {+ if (fdbQueueEnabled()) {+ try {+ const job = await scrapeQueueFdb.getJob(id, logger);+ if (job) return tagFdbJob(job as NuQJob<ScrapeJobData>);+ if (fdbForced()) return null;+ } catch (error) {+ if (fdbForced()) throw error;+ logFdbFallback(logger, "scrape.getJob", error);+ }+ }+ return scrapeQueuePg.getJob(id, logger);+ }++ public async getJobs(+ ids: string[],+ logger: Logger = _logger,+ ): Promise<NuQJob<ScrapeJobData>[]> {+ if (!fdbQueueEnabled()) return scrapeQueuePg.getJobs(ids, logger);+ let fdbJobs: NuQFdbJob<ScrapeJobData>[] = [];+ try {+ fdbJobs = await scrapeQueueFdb.getJobs(ids, logger);+ } catch (error) {+ if (fdbForced()) throw error;+ logFdbFallback(logger, "scrape.getJobs", error);+ return scrapeQueuePg.getJobs(ids, logger);+ }+ if (fdbForced())+ return fdbJobs.map(j => tagFdbJob(j as NuQJob<ScrapeJobData>));+ const found = new Set(fdbJobs.map(j => j.id));+ const missing = ids.filter(id => !found.has(id));+ const pgJobs =+ missing.length > 0 ? await scrapeQueuePg.getJobs(missing, logger) : [];+ const byId = new Map<string, NuQJob<ScrapeJobData>>();+ for (const j of fdbJobs)+ byId.set(j.id, tagFdbJob(j as NuQJob<ScrapeJobData>));+ for (const j of pgJobs) byId.set(j.id, j);+ return ids+ .map(id => byId.get(id))+ .filter((j): j is NuQJob<ScrapeJobData> => j !== undefined);+ }++ public async getJobsWithStatus(+ ids: string[],+ status: NuQJobStatus,+ logger: Logger = _logger,+ ): Promise<NuQJob<ScrapeJobData>[]> {+ return (await this.getJobs(ids, logger)).filter(j => j.status === status);+ }++ public async getJobsWithStatuses(+ ids: string[],+ statuses: NuQJobStatus[],+ logger: Logger = _logger,+ ): Promise<NuQJob<ScrapeJobData>[]> {+ const set = new Set(statuses);+ return (await this.getJobs(ids, logger)).filter(j => set.has(j.status));+ }++ private async isFdbGroup(groupId: string): Promise<boolean> {+ if (!fdbQueueEnabled()) return false;+ try {+ return (await crawlGroupFdb.getGroup(groupId)) !== null;+ } catch (error) {+ if (fdbForced()) throw error;+ logFdbFallback(_logger, "scrape.isFdbGroup", error);+ return false;+ }+ }++ public async getGroupAnyJob(+ groupId: string,+ ownerId: string,+ logger: Logger = _logger,+ ): Promise<NuQJob<ScrapeJobData> | null> {+ if (await this.isFdbGroup(groupId)) {+ const job = await scrapeQueueFdb.getGroupAnyJob(groupId, ownerId, logger);+ return job ? tagFdbJob(job as NuQJob<ScrapeJobData>) : null;+ }+ return scrapeQueuePg.getGroupAnyJob(groupId, ownerId);+ }++ public async getGroupNumericStats(+ groupId: string,+ logger: Logger = _logger,+ ): Promise<Record<NuQJobStatus, number>> {+ if (await this.isFdbGroup(groupId)) {+ return scrapeQueueFdb.getGroupNumericStats(groupId, logger) as Promise<+ Record<NuQJobStatus, number>+ >;+ }+ return scrapeQueuePg.getGroupNumericStats(groupId, logger);+ }++ public async getCrawlJobsForListing(+ groupId: string,+ limit: number,+ offset: number,+ logger: Logger = _logger,+ ): Promise<NuQJob<ScrapeJobData>[]> {+ if (await this.isFdbGroup(groupId)) {+ const jobs = await scrapeQueueFdb.getCrawlJobsForListing(+ groupId,+ limit,+ offset,+ logger,+ );+ return jobs.map(j => tagFdbJob(j as NuQJob<ScrapeJobData>));+ }+ return scrapeQueuePg.getCrawlJobsForListing(groupId, limit, offset, logger);+ }++ public async removeJob(id: string, logger: Logger = _logger): Promise<void> {+ if (fdbQueueEnabled()) {+ try {+ if (await scrapeQueueFdb.hasJob(id)) {+ await scrapeQueueFdb.removeJob(id, logger);+ return;+ }+ } catch (error) {+ if (fdbForced()) throw error;+ logFdbFallback(logger, "scrape.removeJob", error);+ }+ }+ if (fdbForced()) return;+ await scrapeQueuePg.removeJob(id, logger);+ }++ public async removeJobs(+ ids: string[],+ logger: Logger = _logger,+ ): Promise<void> {+ for (const id of ids) {+ await this.removeJob(id, logger);+ }+ }++ public async waitForJob<T = any>(+ id: string,+ timeout: number | null,+ logger: Logger = _logger,+ ): Promise<T> {+ if (fdbQueueEnabled()) {+ try {+ if (await scrapeQueueFdb.hasJob(id)) {+ return scrapeQueueFdb.waitForJob(id, timeout, logger);+ }+ } catch (error) {+ if (fdbForced()) throw error;+ logFdbFallback(logger, "scrape.waitForJob", error);+ }+ }+ if (fdbForced()) throw new Error("Job not found");+ return scrapeQueuePg.waitForJob(id, timeout, logger) as Promise<T>;+ }++ public async getMetrics(logger: Logger = _logger) {+ return scrapeQueuePg.getMetrics();+ }+}++// === Routed crawl-finished queue (worker consumer + reads)++class RoutedCrawlFinishedQueue {+ private inflightBackend = new Map<string, QueueBackend>();++ public async getJobToProcess(+ logger: Logger = _logger,+ ): Promise<NuQJob<any> | null> {+ if (fdbQueueEnabled()) {+ try {+ const job = await crawlFinishedQueueFdb.getJobToProcess(logger);+ if (job) {+ this.inflightBackend.set(job.id, "fdb");+ return tagFdbJob(job as NuQJob<any>);+ }+ } catch (error) {+ if (fdbForced()) throw error;+ logFdbFallback(logger, "crawlFinished.getJobToProcess", error);+ }+ }+ const job = await crawlFinishedQueuePg.getJobToProcess();+ if (job) this.inflightBackend.set(job.id, "pg");+ return job;+ }++ public async renewLock(+ id: string,+ lock: string,+ logger: Logger = _logger,+ ): Promise<boolean> {+ if (this.inflightBackend.get(id) === "fdb") {+ return crawlFinishedQueueFdb.renewLock(id, lock, logger);+ }+ return crawlFinishedQueuePg.renewLock(id, lock, logger);+ }++ public async jobFinish(+ id: string,+ lock: string,+ returnvalue: any | null,+ logger: Logger = _logger,+ ): Promise<boolean> {+ const backend = this.inflightBackend.get(id) ?? "pg";+ this.inflightBackend.delete(id);+ if (backend === "fdb") {+ return crawlFinishedQueueFdb.jobFinish(id, lock, returnvalue, logger);+ }+ return crawlFinishedQueuePg.jobFinish(id, lock, returnvalue, logger);+ }++ public async jobFail(+ id: string,+ lock: string,+ failedReason: string,+ logger: Logger = _logger,+ ): Promise<boolean> {+ const backend = this.inflightBackend.get(id) ?? "pg";+ this.inflightBackend.delete(id);+ if (backend === "fdb") {+ return crawlFinishedQueueFdb.jobFail(id, lock, failedReason, logger);+ }+ return crawlFinishedQueuePg.jobFail(id, lock, failedReason, logger);+ }++ public async getJob(+ id: string,+ logger: Logger = _logger,+ ): Promise<NuQJob<any> | null> {+ if (fdbQueueEnabled()) {+ try {+ const job = await crawlFinishedQueueFdb.getJob(id, logger);+ if (job) return tagFdbJob(job as NuQJob<any>);+ if (fdbForced()) return null;+ } catch (error) {+ if (fdbForced()) throw error;+ logFdbFallback(logger, "crawlFinished.getJob", error);+ }+ }+ return crawlFinishedQueuePg.getJob(id, logger);+ }+}++// === Routed crawl group++class RoutedCrawlGroup {+ public async addGroup(+ id: string,+ ownerId: string,+ ttl?: number,+ opts?: {+ backend?: QueueBackend;+ maxConcurrency?: number;+ delaySeconds?: number;+ },+ logger: Logger = _logger,+ ): Promise<NuQJobGroupInstance> {+ if (opts?.backend === "fdb") {+ const g = await crawlGroupFdb.addGroup(+ id,+ ownerId,+ ttl,+ {+ maxConcurrency: opts.maxConcurrency,+ delaySeconds: opts.delaySeconds,+ },+ logger,+ );+ return g as NuQJobGroupInstance;+ }+ return crawlGroupPg.addGroup(id, ownerId, ttl, logger);+ }++ public async getGroup(+ id: string,+ logger: Logger = _logger,+ ): Promise<NuQJobGroupInstance | null> {+ if (fdbQueueEnabled()) {+ try {+ const g = await crawlGroupFdb.getGroup(id, logger);+ if (g) return g as NuQJobGroupInstance;+ if (fdbForced()) return null;+ } catch (error) {+ if (fdbForced()) throw error;+ logFdbFallback(logger, "crawlGroup.getGroup", error);+ }+ }+ return crawlGroupPg.getGroup(id, logger);+ }++ public async getOngoingByOwner(+ ownerId: string,+ logger: Logger = _logger,+ ): Promise<NuQJobGroupInstance[]> {+ if (!fdbQueueEnabled()) {+ return crawlGroupPg.getOngoingByOwner(ownerId, logger);+ }+ let fdb: NuQJobGroupInstance[] = [];+ try {+ fdb = (await crawlGroupFdb.getOngoingByOwner(+ ownerId,+ logger,+ )) as NuQJobGroupInstance[];+ } catch (error) {+ if (fdbForced()) throw error;+ logFdbFallback(logger, "crawlGroup.getOngoingByOwner", error);+ return crawlGroupPg.getOngoingByOwner(ownerId, logger);+ }+ if (fdbForced()) return fdb as NuQJobGroupInstance[];+ const pg = await crawlGroupPg.getOngoingByOwner(ownerId, logger);+ const seen = new Set(fdb.map(g => g.id));+ return [+ ...(fdb as NuQJobGroupInstance[]),+ ...pg.filter(g => !seen.has(g.id)),+ ];+ }++ // O(1) cancel; only meaningful for FDB groups. PG crawls keep their+ // existing Redis-based cancellation path.+ public async cancelGroup(+ id: string,+ logger: Logger = _logger,+ ): Promise<boolean> {+ if (!fdbQueueEnabled()) return false;+ try {+ return crawlGroupFdb.cancelGroup(id, logger);+ } catch (error) {+ if (fdbForced()) throw error;+ logFdbFallback(logger, "crawlGroup.cancelGroup", error);+ return false;+ }+ }+}++export const scrapeQueue = new RoutedScrapeQueue();+export const crawlFinishedQueue = new RoutedCrawlFinishedQueue();+export const crawlGroup = new RoutedCrawlGroup();diff --git a/apps/api/src/services/worker/nuq-worker.ts b/apps/api/src/services/worker/nuq-worker.tsindex 3f41a0881..aa5982eae 100644--- a/apps/api/src/services/worker/nuq-worker.ts+++ b/apps/api/src/services/worker/nuq-worker.ts@@ -4,7 +4,13 @@ import "../sentry";import { setSentryServiceTag } from "../sentry";import { logger as _logger } from "../../lib/logger";import { processJobInternal } from "./scrape-worker";-import { scrapeQueue, nuqGetLocalMetrics, nuqHealthCheck } from "./nuq";+import {+ nuqGetLocalMetrics,+ nuqHealthCheck,+ scrapeQueue as scrapeQueuePg,+} from "./nuq";+import { scrapeQueue, fdbQueueEnabled } from "./nuq-router";+import { getNuqFdbSweeper } from "./nuq-fdb";import { jobDurationSeconds } from "../../lib/job-metrics";import { register } from "prom-client";import Express from "express";@@ -57,6 +63,20 @@ import { initializeEngineForcing } from "../../scraper/WebScraper/utils/engine-flet noJobTimeout = 1500;+ // the FDB backend has no pg_cron: one worker at a time holds the sweeper+ // lease and runs lease/timeout/group sweeps for everyone+ if (fdbQueueEnabled()) {+ try {+ getNuqFdbSweeper().start();+ } catch (error) {+ if (config.NUQ_BACKEND === "fdb") throw error;+ _logger.warn("Failed to start FDB sweeper, continuing with PG", {+ module: "nuq-worker",+ error,+ });+ }+ }+while (!isShuttingDown) {const job = await scrapeQueue.getJobToProcess();@@ -137,7 +157,17 @@ import { initializeEngineForcing } from "../../scraper/WebScraper/utils/engine-f_logger.info("NuQ worker shutting down");server.close(async () => {- await scrapeQueue.shutdown();+ if (fdbQueueEnabled()) {+ try {+ getNuqFdbSweeper().stop();+ } catch (error) {+ _logger.warn("Failed to stop FDB sweeper", {+ module: "nuq-worker",+ error,+ });+ }+ }+ await scrapeQueuePg.shutdown();_logger.info("NuQ worker shut down");process.exit(0);});diff --git a/apps/api/src/services/worker/nuq.ts b/apps/api/src/services/worker/nuq.tsindex f891103af..627196e20 100644--- a/apps/api/src/services/worker/nuq.ts+++ b/apps/api/src/services/worker/nuq.ts@@ -4,7 +4,7 @@ import { Client, Pool } from "pg";import { type ScrapeJobData } from "../../types";import { withSpan, setSpanAttributes } from "../../lib/otel-tracer";import amqp from "amqplib";-import { v5 as uuidv5, validate as isUUID } from "uuid";+import { normalizeOwnerId } from "../../lib/owner-id";import { config } from "../../config";import { nuqRedis } from "./redis";@@ -53,16 +53,6 @@ type NuQOptions = {backlog?: boolean;};-// owner IDs can sometimes be non-UUID, so let's normalize it to avoid query breakage - mogery-const normalizedUUIDNamespace = "0f38e00e-d7ee-4b77-8a7a-a787a3537ca2";-export function normalizeOwnerId(- ownerId: string | undefined | null,-): string | null {- if (typeof ownerId !== "string") return null;- if (isUUID(ownerId)) return ownerId;- return uuidv5(ownerId, normalizedUUIDNamespace);-}-function isExpectedAmqpCloseError(error: unknown): boolean {const message =error instanceof Errordiff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex fa084dc01..9c1bc0cad 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -520,6 +520,9 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {[doc.metadata.url ?? doc.metadata.sourceURL!],1,sc.crawlerOptions?.maxDepth ?? 10,+ false,+ false,+ true,);if (filterResult.links.length === 0) {const url = doc.metadata.url ?? doc.metadata.sourceURL!;@@ -1307,10 +1310,14 @@ export const processJobInternal = async (job: NuQJob<ScrapeJobData>) => {};async function processJobWithTracing(job: NuQJob<ScrapeJobData>, logger: any) {+ // FDB-backed jobs hold their concurrency slot through the queue lease; the+ // Redis slot mirror and promotion-on-done below are PG-backend machinery+ const isFdbJob = (job as any).backend === "fdb";try {try {let extendLockInterval: NodeJS.Timeout | null = null;if (+ !isFdbJob &&job.data?.mode !== "kickoff" &&job.data?.team_id &&!job.data.skipNuq@@ -1380,7 +1387,7 @@ async function processJobWithTracing(job: NuQJob<ScrapeJobData>, logger: any) {}}} finally {- if (!job.data.skipNuq) {+ if (!job.data.skipNuq && !isFdbJob) {await concurrentJobDone(job);}}diff --git a/apps/api/src/services/worker/team-semaphore.ts b/apps/api/src/services/worker/team-semaphore.tsindex 10eaf3b4b..7b3c9954f 100644--- a/apps/api/src/services/worker/team-semaphore.ts+++ b/apps/api/src/services/worker/team-semaphore.ts@@ -2,6 +2,8 @@ import {pushConcurrencyLimitActiveJob,removeConcurrencyLimitActiveJob,} from "../../lib/concurrency-limit";+import { isFdbTeam } from "./nuq-router";+import { externalSlotsFdb } from "./nuq-fdb";import { isSelfHosted } from "../../lib/deployment";import { ScrapeJobTimeoutError, TransportableError } from "../../lib/error";import { logger as _logger } from "../../lib/logger";@@ -30,6 +32,8 @@ const semaphoreHoldDuration = new Histogram({const { scripts, runScript, ensure } = nuqRedis;const SEMAPHORE_TTL = 30 * 1000;+type MirrorBackend = "pg" | "fdb";+type MirrorState = { backend?: MirrorBackend; touched: Set<MirrorBackend> };async function acquire(teamId: string,@@ -137,29 +141,50 @@ async function count(teamId: string): Promise<number> {return count;}-function startHeartbeat(teamId: string, holderId: string, intervalMs: number) {+function startHeartbeat(+ teamId: string,+ holderId: string,+ intervalMs: number,+ mirrorState: MirrorState,+) {let stopped = false;+ let wake: (() => void) | null = null;++ const sleep = (ms: number) =>+ new Promise<void>(resolve => {+ const timer = setTimeout(() => {+ wake = null;+ resolve();+ }, ms);+ wake = () => {+ clearTimeout(timer);+ wake = null;+ resolve();+ };+ });const promise = (async () => {try {while (!stopped) {- await pushConcurrencyLimitActiveJob(teamId, holderId, 60 * 1000).catch(- () => {- _logger.warn("Failed to update concurrency limit active job", {- teamId,- jobId: holderId,- });- },- );+ await mirrorSlotAcquire(teamId, holderId, mirrorState).catch(() => {+ _logger.warn("Failed to update concurrency limit active job", {+ teamId,+ jobId: holderId,+ });+ });+ if (stopped) break;const ok = await heartbeat(teamId, holderId);if (!ok) {throw new TransportableError("SCRAPE_TIMEOUT", "heartbeat_failed");}- await new Promise(r => setTimeout(r, intervalMs));+ if (stopped) break;+ await sleep(intervalMs);}} catch (error) {- _logger.error("Error in semaphore heartbeat loop", { error });+ if (!stopped) {+ _logger.error("Error in semaphore heartbeat loop", { error });+ }}return Promise.reject(@@ -169,12 +194,72 @@ function startHeartbeat(teamId: string, holderId: string, intervalMs: number) {return {promise,- stop() {+ async stop() {stopped = true;+ wake?.();+ await promise.catch(() => {});},};}+// Sync scrapes occupy queue capacity so async jobs see the team's real load.+// PG-backed teams mirror into the Redis ZSET; FDB-backed teams consume an+// external slot on the FDB ledger.+async function resolveMirrorBackend(teamId: string): Promise<MirrorBackend> {+ return (await isFdbTeam(teamId)) ? "fdb" : "pg";+}++async function releaseMirrorBackend(+ teamId: string,+ holderId: string,+ backend: MirrorBackend,+): Promise<void> {+ if (backend === "fdb") {+ await externalSlotsFdb.release(teamId, holderId);+ } else {+ await removeConcurrencyLimitActiveJob(teamId, holderId);+ }+}++async function mirrorSlotAcquire(+ teamId: string,+ holderId: string,+ state: MirrorState,+): Promise<void> {+ const backend = await resolveMirrorBackend(teamId);+ if (backend === "fdb") {+ await externalSlotsFdb.acquire(teamId, holderId, 60 * 1000);+ } else {+ await pushConcurrencyLimitActiveJob(teamId, holderId, 60 * 1000);+ }+ const previous = state.backend;+ state.backend = backend;+ state.touched.add(backend);+ if (previous && previous !== backend) {+ await releaseMirrorBackend(teamId, holderId, previous);+ state.touched.delete(previous);+ }+}++async function mirrorSlotRelease(+ teamId: string,+ holderId: string,+ state: MirrorState,+): Promise<void> {+ const backends = Array.from(state.touched);+ const results = await Promise.allSettled(+ backends.map(backend => releaseMirrorBackend(teamId, holderId, backend)),+ );+ const failed = results.find(+ (result): result is PromiseRejectedResult => result.status === "rejected",+ );+ if (failed) {+ throw failed.reason;+ }+ state.backend = undefined;+ state.touched.clear();+}+async function withSemaphore<T>(teamId: string,holderId: string,@@ -200,16 +285,20 @@ async function withSemaphore<T>(});const endTimer = semaphoreHoldDuration.startTimer();- const hb = startHeartbeat(teamId, holderId, SEMAPHORE_TTL / 2);+ const mirrorState: MirrorState = { touched: new Set() };+ let hb: ReturnType<typeof startHeartbeat> | null = null;activeSemaphores.inc();try {- await pushConcurrencyLimitActiveJob(teamId, holderId, 60 * 1000);+ await mirrorSlotAcquire(teamId, holderId, mirrorState);+ hb = startHeartbeat(teamId, holderId, SEMAPHORE_TTL / 2, mirrorState);const result = await Promise.race([func(limited), hb.promise]);return result;} finally {- await removeConcurrencyLimitActiveJob(teamId, holderId).catch(() => {+ await hb?.stop();++ await mirrorSlotRelease(teamId, holderId, mirrorState).catch(() => {_logger.warn("Failed to remove concurrency limit active job", {teamId,jobId: holderId,@@ -217,7 +306,6 @@ async function withSemaphore<T>(});activeSemaphores.dec();- hb.stop();endTimer();await release(teamId, holderId).catch(() => {});diff --git a/docker-compose.yaml b/docker-compose.yamlindex b6a1778e3..abf6980e7 100644--- a/docker-compose.yaml+++ b/docker-compose.yaml@@ -38,8 +38,8 @@ x-common-env: &common-envOPENAI_API_KEY: ${OPENAI_API_KEY}OPENAI_BASE_URL: ${OPENAI_BASE_URL}MODEL_NAME: ${MODEL_NAME}- MODEL_EMBEDDING_NAME: ${MODEL_EMBEDDING_NAME}- OLLAMA_BASE_URL: ${OLLAMA_BASE_URL}+ MODEL_EMBEDDING_NAME: ${MODEL_EMBEDDING_NAME}+ OLLAMA_BASE_URL: ${OLLAMA_BASE_URL}AUTUMN_SECRET_KEY: ${AUTUMN_SECRET_KEY}SLACK_WEBHOOK_URL: ${SLACK_WEBHOOK_URL}BULL_AUTH_KEY: ${BULL_AUTH_KEY}@@ -55,6 +55,9 @@ x-common-env: &common-envSEARXNG_ENDPOINT: ${SEARXNG_ENDPOINT}SEARXNG_ENGINES: ${SEARXNG_ENGINES}SEARXNG_CATEGORIES: ${SEARXNG_CATEGORIES}+ # Set NUQ_BACKEND=fdb to run the queue on FoundationDB instead of Postgres+ NUQ_BACKEND: ${NUQ_BACKEND}+ FDB_CLUSTER_FILE: ${NUQ_BACKEND:+/var/fdb/fdb.cluster}services:playwright-service:@@ -106,6 +109,8 @@ services:condition: service_healthyports:- "${PORT:-3002}:${INTERNAL_PORT:-3002}"+ volumes:+ - fdb-cluster-file:/var/fdb:rocommand: node dist/src/harness.js --start-docker# Resource limits for Docker Compose (not Swarm)# Increase if you have more CPU cores/RAM available@@ -129,7 +134,7 @@ services:max-size: "5m"max-file: "2"compress: "true"-+rabbitmq:image: rabbitmq:3-managementnetworks:@@ -147,7 +152,7 @@ services:max-size: "5m"max-file: "2"compress: "true"-+nuq-postgres:# NOTE: If you don't want to build the image locally,# comment out the build: statement and uncomment the image: statement@@ -166,6 +171,45 @@ services:max-file: "3"compress: "true"+ # FoundationDB queue backend (experimental). Enable by setting+ # NUQ_BACKEND=fdb; the api container reads the shared cluster file from the+ # fdb-cluster-file volume. Replaces nuq-postgres once stable.+ foundationdb:+ image: foundationdb/foundationdb:7.3.63+ environment:+ FDB_NETWORKING_MODE: container+ FDB_COORDINATOR_PORT: 4500+ networks:+ - backend+ volumes:+ - fdb-data:/var/fdb/data+ - fdb-cluster-file:/var/fdb+ logging:+ driver: "json-file"+ options:+ max-size: "10m"+ max-file: "3"+ compress: "true"++ # one-shot: creates the database on first boot (no-op afterwards)+ foundationdb-init:+ image: foundationdb/foundationdb:7.3.63+ depends_on:+ - foundationdb+ entrypoint:+ - /bin/bash+ - -c+ - "sleep 5 && out=$(fdbcli -C /var/fdb/fdb.cluster --exec 'configure new single ssd' 2>&1); status=$$?; printf '%s\n' \"$$out\"; if [ \"$$status\" -eq 0 ]; then exit 0; fi; printf '%s\n' \"$$out\" | grep -Eiq 'already.*configured|database.*configured'"+ volumes:+ - fdb-cluster-file:/var/fdb+ networks:+ - backend+ restart: "no"+networks:backend:driver: bridge++volumes:+ fdb-data:+ fdb-cluster-file: