Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(nuq): API-key-scoped concurrency limits on the FDB queue (#3961)
Gergő Móricz8671d1501f00
diff --git a/apps/api/src/__tests__/nuq-fdb/key-gate.test.ts b/apps/api/src/__tests__/nuq-fdb/key-gate.test.tsnew file mode 100644index 000000000..39ff53632--- /dev/null+++ b/apps/api/src/__tests__/nuq-fdb/key-gate.test.ts@@ -0,0 +1,360 @@+import { randomUUID } from "crypto";+import { config } from "../../config";+import {+ NuQFdbQueue,+ NuQFdbJobGroup,+ NuqFdbSweeper,+} from "../../services/worker/nuq-fdb";+import {+ getNuqFdbDatabase,+ getFdb,+} from "../../services/worker/nuq-fdb/client";+import { decodeI64 } from "../../services/worker/nuq-fdb/keyspace";++// API-key concurrency gate tests, run directly against a real FoundationDB+// cluster like core.test.ts. Skipped when FDB is not configured.+const describeIf = config.FDB_CLUSTER_FILE ? describe : describe.skip;++const RUN = randomUUID().slice(0, 8);+const TEST_LEASE_MS = 1500;++const createdQueueNames: string[] = [];++type Ctx = {+ queue: NuQFdbQueue;+ group: NuQFdbJobGroup;+ sweeper: NuqFdbSweeper;+};++async function makeCtx(name: string): Promise<Ctx> {+ const scrapeName = `tk-${RUN}-${name}`;+ const finishedName = `tk-${RUN}-${name}-fin`;+ createdQueueNames.push(scrapeName, finishedName);+ const queue = new NuQFdbQueue(scrapeName, {+ hasGroups: true,+ finishedQueueName: finishedName,+ leaseMs: TEST_LEASE_MS,+ });+ const finishedQueue = new NuQFdbQueue(finishedName, { hasGroups: false });+ const group = new NuQFdbJobGroup(queue.ks, queue.groupOps!);+ const sweeper = new NuqFdbSweeper([queue, finishedQueue]);+ return { queue, group, sweeper };+}++async function takeAll(+ queue: NuQFdbQueue,+ maxJobs: number = 50,+): Promise<any[]> {+ const out: any[] = [];+ while (out.length < maxJobs) {+ const job = await queue.getJobToProcess();+ if (job === null) break;+ out.push(job);+ }+ return out;+}++function freshOwner(): string {+ return randomUUID();+}++function scrapeData(): any {+ return { mode: "single_urls", url: "https://example.com" };+}++const kgate = (+ teamLimit: number,+ keyId: string,+ keyLimit: number,+ cap: number = 1_000_000,+) => ({+ teamLimit,+ queueCap: cap,+ key: { id: keyId, limit: keyLimit },+});++async function keyActiveCount(queue: NuQFdbQueue, kid: string) {+ return await getNuqFdbDatabase().doTn(async tn =>+ decodeI64(await tn.get(queue.ks.keyActive(kid))),+ );+}++function jobInput(owner: string, groupId?: string) {+ return {+ id: randomUUID(),+ data: scrapeData(),+ options: {+ ownerId: owner,+ groupId,+ timesOutAt: new Date(Date.now() + 60_000),+ },+ };+}++describeIf("NuQ FDB API-key gate", () => {+ afterAll(async () => {+ const fdb = getFdb();+ const db = getNuqFdbDatabase();+ for (const name of createdQueueNames) {+ const r = fdb.tuple.range(["nuq", name]);+ await db.doTn(async tn =>+ tn.clearRange(r.begin as Buffer, r.end as Buffer),+ );+ }+ });++ test("key limit 2 admits 2, backlogs the rest, hands off on finish", async () => {+ const { queue } = await makeCtx("basic");+ const owner = freshOwner();+ const kid = randomUUID();+ const jobs = await queue.addJobs(+ Array.from({ length: 5 }, () => jobInput(owner)),+ kgate(10, kid, 2),+ );++ // the team would admit all 5; the key admits only 2+ expect(jobs.filter(j => j.status === "queued").length).toBe(2);+ expect(jobs.filter(j => j.status === "backlog").length).toBe(3);+ expect(await queue.getTeamActiveCount(owner)).toBe(2);+ expect(await queue.getTeamPendingCount(owner)).toBe(3);+ expect(await keyActiveCount(queue, kid)).toBe(2);++ const taken = await takeAll(queue, 5);+ expect(taken.length).toBe(2);++ // finishing one job hands its key slot to exactly one backlogged job+ await queue.jobFinish(taken[0].id, taken[0].lock!, null);+ expect(await keyActiveCount(queue, kid)).toBe(2);+ expect(await queue.getTeamActiveCount(owner)).toBe(2);+ expect(await queue.getTeamPendingCount(owner)).toBe(2);++ const next = await takeAll(queue, 5);+ expect(next.length).toBe(1);++ // drain everything; counters return to zero+ let inflight = [taken[1], ...next];+ while (inflight.length > 0) {+ for (const j of inflight) await queue.jobFinish(j.id, j.lock!, null);+ inflight = await takeAll(queue, 5);+ }+ expect(await queue.getTeamActiveCount(owner)).toBe(0);+ expect(await queue.getTeamPendingCount(owner)).toBe(0);+ expect(await keyActiveCount(queue, kid)).toBe(0);+ });++ test("key gate cascades inside the crawl gate", async () => {+ const { queue, group } = await makeCtx("cascade");+ const owner = freshOwner();+ const kid = randomUUID();+ const gid = randomUUID();+ await group.addGroup(gid, owner, undefined, { maxConcurrency: 3 });++ await queue.addJobs(+ Array.from({ length: 6 }, () => jobInput(owner, gid)),+ kgate(10, kid, 2),+ );++ // crawl admits 3, key admits 2 of those: 2 ready, 1 key-pending,+ // 3 crawl-pending+ expect(await queue.getTeamActiveCount(owner)).toBe(2);+ expect(await keyActiveCount(queue, kid)).toBe(2);++ // the key limit stays the ceiling while all 6 jobs drain through+ let finished = 0;+ let inflight = await takeAll(queue, 6);+ expect(inflight.length).toBe(2);+ while (inflight.length > 0) {+ expect(inflight.length).toBeLessThanOrEqual(2);+ for (const j of inflight) {+ await queue.jobFinish(j.id, j.lock!, null);+ finished++;+ }+ expect(await keyActiveCount(queue, kid)).toBeLessThanOrEqual(2);+ inflight = await takeAll(queue, 6);+ }+ expect(finished).toBe(6);++ const g = await group.getGroup(gid);+ expect(g?.status).toBe("completed");+ expect(await queue.getTeamActiveCount(owner)).toBe(0);+ expect(await keyActiveCount(queue, kid)).toBe(0);+ });++ test("key limit raise promotes backlogged jobs via the sweeper", async () => {+ const { queue, sweeper } = await makeCtx("raise");+ const owner = freshOwner();+ const kid = randomUUID();+ await queue.addJobs(+ Array.from({ length: 3 }, () => jobInput(owner)),+ kgate(10, kid, 1),+ );+ expect(await queue.getTeamActiveCount(owner)).toBe(1);+ expect(await queue.getTeamPendingCount(owner)).toBe(2);++ // a later enqueue arrives with a raised key limit+ await queue.addJob(+ randomUUID(),+ scrapeData(),+ { ownerId: owner, timesOutAt: new Date(Date.now() + 60_000) },+ kgate(10, kid, 3),+ );+ await sweeper.sweepOnce();++ expect(await queue.getTeamActiveCount(owner)).toBe(3);+ expect(await queue.getTeamPendingCount(owner)).toBe(1);+ expect(await keyActiveCount(queue, kid)).toBe(3);++ let inflight = await takeAll(queue, 4);+ expect(inflight.length).toBe(3);+ let total = 0;+ while (inflight.length > 0) {+ for (const j of inflight) {+ await queue.jobFinish(j.id, j.lock!, null);+ total++;+ }+ inflight = await takeAll(queue, 4);+ }+ expect(total).toBe(4);+ expect(await keyActiveCount(queue, kid)).toBe(0);+ });++ test("two keys gate independently under one team", async () => {+ const { queue } = await makeCtx("twokeys");+ const owner = freshOwner();+ const kid1 = randomUUID();+ const kid2 = randomUUID();++ await queue.addJobs(+ Array.from({ length: 2 }, () => jobInput(owner)),+ kgate(10, kid1, 1),+ );+ await queue.addJobs(+ Array.from({ length: 2 }, () => jobInput(owner)),+ kgate(10, kid2, 1),+ );++ expect(await queue.getTeamActiveCount(owner)).toBe(2);+ expect(await keyActiveCount(queue, kid1)).toBe(1);+ expect(await keyActiveCount(queue, kid2)).toBe(1);++ const taken = await takeAll(queue, 4);+ expect(taken.length).toBe(2);++ // finishing one key's job promotes that key's backlog only+ await queue.jobFinish(taken[0].id, taken[0].lock!, null);+ expect(await keyActiveCount(queue, kid1)).toBe(1);+ expect(await keyActiveCount(queue, kid2)).toBe(1);+ expect(await queue.getTeamPendingCount(owner)).toBe(1);++ await queue.jobFinish(taken[1].id, taken[1].lock!, null);+ let inflight = await takeAll(queue, 4);+ let total = 2; // 2 taken initially+ while (inflight.length > 0) {+ total += inflight.length;+ for (const j of inflight) await queue.jobFinish(j.id, j.lock!, null);+ inflight = await takeAll(queue, 4);+ }+ expect(total).toBe(4);+ expect(await keyActiveCount(queue, kid1)).toBe(0);+ expect(await keyActiveCount(queue, kid2)).toBe(0);+ });++ test("removing a team-pending job hands its key slot to the key backlog", async () => {+ const { queue } = await makeCtx("remove");+ const owner = freshOwner();+ const kid = randomUUID();+ // team 1, key 2: A ready (team+key), B team-pending (holds key),+ // C key-pending+ const inputs = Array.from({ length: 3 }, () => jobInput(owner));+ const jobs = await queue.addJobs(inputs, kgate(1, kid, 2));+ const a = jobs.find(j => j.status === "queued")!;+ expect(await keyActiveCount(queue, kid)).toBe(2);+ expect(await queue.getTeamPendingCount(owner)).toBe(2);++ // jobs are placed in input order: jobs[1] is the team-pending one (it+ // won the second key slot), jobs[2] waits in the key gate+ const b = jobs[1];+ expect(b.status).toBe("backlog");+ await queue.removeJob(b.id);+ // its key slot moved to the key-pending job; nothing ran yet, so the+ // key active count is unchanged and one job still waits for the team+ expect(await keyActiveCount(queue, kid)).toBe(2);+ expect(await queue.getTeamPendingCount(owner)).toBe(1);++ const [takenA] = await takeAll(queue, 2);+ expect(takenA.id).toBe(a.id);+ await queue.jobFinish(takenA.id, takenA.lock!, null);++ const [takenC] = await takeAll(queue, 2);+ expect(takenC).toBeDefined();+ await queue.jobFinish(takenC.id, takenC.lock!, null);++ expect(await queue.getTeamActiveCount(owner)).toBe(0);+ expect(await queue.getTeamPendingCount(owner)).toBe(0);+ expect(await keyActiveCount(queue, kid)).toBe(0);+ });++ test("key-pending jobs are silently dropped at their backlog deadline", async () => {+ const { queue, sweeper } = await makeCtx("bto");+ const owner = freshOwner();+ const kid = randomUUID();+ const inputs = Array.from({ length: 3 }, () => ({+ id: randomUUID(),+ data: scrapeData(),+ options: {+ ownerId: owner,+ timesOutAt: new Date(Date.now() + 1000),+ },+ }));+ const jobs = await queue.addJobs(inputs, kgate(10, kid, 1));+ expect(jobs.filter(j => j.status === "backlog").length).toBe(2);++ await new Promise(resolve => setTimeout(resolve, 1200));+ await sweeper.sweepOnce();++ // the two key-pending jobs are gone; the running one is unaffected+ expect(await queue.getTeamPendingCount(owner)).toBe(0);+ expect(await keyActiveCount(queue, kid)).toBe(1);+ for (const j of jobs.filter(j => j.status === "backlog")) {+ expect(await queue.getJob(j.id)).toBeNull();+ }++ const taken = await takeAll(queue, 3);+ expect(taken.length).toBe(1);+ await queue.jobFinish(taken[0].id, taken[0].lock!, null);+ expect(await keyActiveCount(queue, kid)).toBe(0);+ });++ test("crawl delay wake-up passes through the key gate", async () => {+ const { queue, group, sweeper } = await makeCtx("delay");+ const owner = freshOwner();+ const kid = randomUUID();+ const gid = randomUUID();+ await group.addGroup(gid, owner, undefined, { delaySeconds: 1 });++ const inputs = [jobInput(owner, gid), jobInput(owner, gid)];+ await queue.addJobs(inputs, kgate(10, kid, 1));++ const [first] = await takeAll(queue, 2);+ expect(first).toBeDefined();+ await queue.jobFinish(first.id, first.lock!, null);+ // the second job is parked in the delay index holding only its crawl+ // slot; the finished job's key slot was released+ expect(await keyActiveCount(queue, kid)).toBe(0);++ await sweeper.sweepOnce();+ expect(await queue.getJobToProcess()).toBeNull();++ await new Promise(resolve => setTimeout(resolve, 1200));+ await sweeper.sweepOnce();+ // the wake-up acquired the key slot again+ expect(await keyActiveCount(queue, kid)).toBe(1);+ const [second] = await takeAll(queue, 1);+ expect(second).toBeDefined();+ await queue.jobFinish(second.id, second.lock!, null);+ expect(await keyActiveCount(queue, kid)).toBe(0);++ const g = await group.getGroup(gid);+ expect(g?.status).toBe("completed");+ });+});diff --git a/apps/api/src/db/schema/public.ts b/apps/api/src/db/schema/public.tsindex fa29eb7ae..9af4190d5 100644--- a/apps/api/src/db/schema/public.ts+++ b/apps/api/src/db/schema/public.ts@@ -73,6 +73,8 @@ export const api_keys = pgTable(team_id: uuid("team_id"),owner_id: uuid("owner_id"),agent_provisioned: boolean("agent_provisioned").default(false),+ // API-key-scoped concurrency limit; null = key inherits the team limit only+ concurrency: integer("concurrency"),},table => [// Target of key_restriction_config's composite FK, which pins adiff --git a/apps/api/src/lib/api-key-concurrency.ts b/apps/api/src/lib/api-key-concurrency.tsnew file mode 100644index 000000000..7128d11ba--- /dev/null+++ b/apps/api/src/lib/api-key-concurrency.ts@@ -0,0 +1,74 @@+import { eq } from "drizzle-orm";+import { dbRr } from "../db/connection";+import * as schema from "../db/schema";+import { getValue, setValue } from "../services/redis";+import { logger } from "./logger";++// Propagation delay for edits to api_keys.concurrency.+const LIMIT_CACHE_TTL_SECONDS = 60;++const limitCacheKey = (apiKeyId: number) => `api-key-concurrency:${apiKeyId}`;++/**+ * Returns the API-key-scoped concurrency limit (api_keys.concurrency), or+ * null when the key has no limit of its own. Cached for a minute per key.+ *+ * Fails open (null): unlike the IP allowlist this is a throttle, not a+ * security boundary, and a transient DB/cache error must not stall enqueues.+ */+export async function getApiKeyConcurrencyLimit(+ apiKeyId: number,+): Promise<number | null> {+ const cacheKey = limitCacheKey(apiKeyId);++ try {+ const cached = await getValue(cacheKey);+ if (cached !== null) {+ // only the explicit negative-cache sentinel means "no limit"; anything+ // else malformed is a cache miss so a corrupted entry cannot silently+ // disable the key's gate+ if (cached === "none") return null;+ const parsed = Number(cached);+ if (Number.isInteger(parsed) && parsed > 0) return parsed;+ logger.warn("Ignoring malformed API key concurrency cache entry", {+ apiKeyId,+ });+ }+ } catch (error) {+ logger.warn("Failed to read API key concurrency cache", {+ apiKeyId,+ error,+ });+ }++ let limit: number | null = null;+ try {+ const [row] = await dbRr+ .select({ concurrency: schema.api_keys.concurrency })+ .from(schema.api_keys)+ .where(eq(schema.api_keys.id, apiKeyId))+ .limit(1);+ limit =+ typeof row?.concurrency === "number" && row.concurrency > 0+ ? row.concurrency+ : null;+ } catch (error) {+ logger.warn("Failed to load API key concurrency limit", {+ apiKeyId,+ error,+ });+ return null;+ }++ try {+ // "none" is a cached negative so unlimited keys skip the DB read too+ await setValue(cacheKey, String(limit ?? "none"), LIMIT_CACHE_TTL_SECONDS);+ } catch (error) {+ logger.warn("Failed to cache API key concurrency limit", {+ apiKeyId,+ error,+ });+ }++ return limit;+}diff --git a/apps/api/src/services/worker/nuq-fdb/keyspace.ts b/apps/api/src/services/worker/nuq-fdb/keyspace.tsindex e84622ea9..38e95f6fb 100644--- a/apps/api/src/services/worker/nuq-fdb/keyspace.ts+++ b/apps/api/src/services/worker/nuq-fdb/keyspace.ts@@ -25,11 +25,13 @@ export const F_LISTENABLE = 4;export const F_ZDR = 8;export const F_COUNTABLE = 16; // mode === "single_urls": counted in group numeric statsexport const F_GACC = 32; // participates in group remaining-count accounting+export const F_KEY_GATED = 64; // holds an API-key slot while team-pending/queued/active// Where a pending job's queue entry lives, stored on its status record so// sweepers and removers can clear the exact key without scanning.export type PendingLoc =| { k: "tq"; s: number; p: number; c: number } // team-pending: shard, priority, createdAtMs+ | { k: "kq"; p: number; c: number } // key-pending: priority, createdAtMs| { k: "gq"; p: number; c: number } // crawl-pending: priority, createdAtMs| { k: "dl"; at: number }; // delay index: notBeforeMs@@ -38,6 +40,7 @@ export type JobMeta = {p: number; // priorityo: string; // ownerId (normalized uuid)g?: string; // groupId+ k?: string; // API key id (set only when key-gated)f: number; // flagsto?: number; // backlog timesOutAt msdc: number; // data chunk count@@ -57,6 +60,7 @@ export type QueueEntry = {i: string; // jobIdo: string; // ownerIdg?: string; // groupId+ k?: string; // API key id (set only when key-gated)p: number; // priorityf: number; // flagsc: number; // createdAt ms@@ -202,6 +206,28 @@ export class NuqFdbKeyspace {return this.packRange(["t", tid, "q", shard]);}+ // === API-key gate (sits between the crawl gate and the team gate)+ keyLimit(kid: string): Buffer {+ return this.pack(["k", kid, "limit"]);+ }+ keyActive(kid: string): Buffer {+ return this.pack(["k", kid, "active"]);+ }+ keyPendingCount(kid: string): Buffer {+ return this.pack(["k", kid, "qn"]);+ }+ keyPendingKey(+ kid: string,+ priority: number,+ createdAtMs: number,+ id: string,+ ): Buffer {+ return this.pack(["k", kid, "q", priority, createdAtMs, id]);+ }+ keyPendingRange(kid: string) {+ return this.packRange(["k", kid, "q"]);+ }+// === Group (crawl) recordsgroupMeta(gid: string): Buffer {return this.pack(["g", gid, "meta"]);@@ -340,6 +366,12 @@ export class NuqFdbKeyspace {taskTeamRaiseRange() {return this.packRange(["task", "traise"]);}+ taskKeyRaise(kid: string): Buffer {+ return this.pack(["task", "kraise", kid]);+ }+ taskKeyRaiseRange() {+ return this.packRange(["task", "kraise"]);+ }sweeperLock(): Buffer {return this.pack(["sweep", "lock"]);}diff --git a/apps/api/src/services/worker/nuq-fdb/ops.ts b/apps/api/src/services/worker/nuq-fdb/ops.tsindex 24526174d..130570078 100644--- a/apps/api/src/services/worker/nuq-fdb/ops.ts+++ b/apps/api/src/services/worker/nuq-fdb/ops.ts@@ -17,6 +17,7 @@ import {F_GATED,F_CRAWL_GATED,F_COUNTABLE,+ F_KEY_GATED,} from "./keyspace";export const ONE = encodeI64(1);@@ -112,6 +113,20 @@ export function appendTeamPending(return { k: "tq", s: shard, p: e.p, c: e.c };}+export function appendKeyPending(+ tn: Transaction,+ ks: NuqFdbKeyspace,+ e: QueueEntry,+): PendingLoc {+ tn.set(ks.keyPendingKey(e.k!, e.p, e.c, e.i), encodeJson(e));+ tn.add(ks.keyPendingCount(e.k!), ONE);+ tn.add(ks.teamPendingCount(e.o), ONE);+ if (e.to !== undefined) {+ tn.set(ks.backlogTimeout(timeBucket(e.i), e.to, e.i), EMPTY);+ }+ return { k: "kq", p: e.p, c: e.c };+}+export function appendCrawlPending(tn: Transaction,ks: NuqFdbKeyspace,@@ -158,10 +173,12 @@ export function pendingKeyFromLoc(id: string,ownerId: string,groupId: string | undefined,+ keyId: string | undefined,loc: PendingLoc,): Buffer {if (loc.k === "tq")return ks.teamPendingKey(ownerId, loc.s, loc.p, loc.c, id);+ if (loc.k === "kq") return ks.keyPendingKey(keyId!, loc.p, loc.c, id);if (loc.k === "gq") return ks.groupPendingKey(groupId!, loc.p, loc.c, id);return ks.delayed(timeBucket(id), loc.at, id);}@@ -174,13 +191,17 @@ export function clearPendingPlacement(id: string,ownerId: string,groupId: string | undefined,+ keyId: string | undefined,loc: PendingLoc,timesOutAt: number | undefined,): void {- tn.clear(pendingKeyFromLoc(ks, id, ownerId, groupId, loc));+ tn.clear(pendingKeyFromLoc(ks, id, ownerId, groupId, keyId, loc));if (loc.k === "tq") {tn.add(ks.teamShardCount(ownerId, loc.s), MINUS_ONE);tn.add(ks.teamPendingCount(ownerId), MINUS_ONE);+ } else if (loc.k === "kq") {+ tn.add(ks.keyPendingCount(keyId!), MINUS_ONE);+ tn.add(ks.teamPendingCount(ownerId), MINUS_ONE);} else if (loc.k === "gq") {tn.add(ks.groupPendingCount(groupId!), MINUS_ONE);tn.add(ks.teamPendingCount(ownerId), MINUS_ONE);@@ -255,6 +276,26 @@ export async function popTeamPending(return null;}+// Like popCrawlPending, key-pending is a single range: concurrent finishers of+// the same key conflict on the head, but per-key concurrency is small by+// definition (it is the key's limit), so the retry pressure stays bounded.+export async function popKeyPending(+ tn: Transaction,+ ks: NuqFdbKeyspace,+ kid: string,+): Promise<QueueEntry | null> {+ const r = ks.keyPendingRange(kid);+ const head = await tn.getRangeAll(r.begin, r.end, { limit: 1 });+ if (head.length === 0) return null;+ const [key, value] = head[0];+ const e = decodeJson<QueueEntry>(value as Buffer)!;+ tn.clear(key as Buffer);+ tn.add(ks.keyPendingCount(kid), MINUS_ONE);+ tn.add(ks.teamPendingCount(e.o), MINUS_ONE);+ clearBacklogTimeout(tn, ks, e);+ return e;+}+export async function popCrawlPending(tn: Transaction,ks: NuqFdbKeyspace,@@ -278,24 +319,29 @@ export async function popCrawlPending(// the same transaction whenever possible, so this function never does a// conflicting read of the team active counter on the handoff path.//+// The gate chain is crawl -> key -> team -> ready. A job entering an inner+// gate keeps the outer slots it already won.+//// `held` describes which slots the leaving job actually holds:-// - active / ready jobs: team + crawl (if crawl-gated)-// - team-pending / delayed jobs: crawl only (if crawl-gated)+// - active / ready jobs: team + key (if key-gated) + crawl (if crawl-gated)+// - team-pending jobs: key + crawl only+// - key-pending / delayed jobs: crawl only// - crawl-pending jobs: noneexport async function releaseSlotsAndPromote(tn: Transaction,ks: NuqFdbKeyspace,e: QueueEntry,- held: { team: boolean; crawl: boolean },+ held: { team: boolean; key: boolean; crawl: boolean },now: number,txc: TxContext,): Promise<void> {if (!(e.f & F_GATED)) return;const tid = e.o;+ const holdsKey = held.key && !!e.k && !!(e.f & F_KEY_GATED);const holdsCrawl = held.crawl && !!e.g && !!(e.f & F_CRAWL_GATED);- if (!held.team && !holdsCrawl) return;+ if (!held.team && !holdsKey && !holdsCrawl) return;// Limit-lowering convergence: if the team is over its limit, don't hand off.let overLimit = false;@@ -313,6 +359,41 @@ export async function releaseSlotsAndPromote(if (held.team && !overLimit) {teamHead = await popTeamPending(tn, ks, tid);}+ // whether the freed team slot is still unassigned after the pops so far+ const teamSlotFree = () => held.team && !overLimit && teamHead === null;++ // Key slot handoff: promote the key-pending head into the team gate. Runs+ // before the crawl handoff so the longest-gated job gets the slot first.+ let keyHandedOff = false;+ if (holdsKey) {+ let keyOverLimit = false;+ const kLimitBuf = await tn.get(ks.keyLimit(e.k!)); // cold key+ if (kLimitBuf) {+ const kLimit = decodeI64(kLimitBuf);+ const kActive = decodeI64(await tn.snapshot().get(ks.keyActive(e.k!)));+ keyOverLimit = kActive > kLimit;+ }+ if (keyOverLimit) {+ // limit-lowering convergence: swallow the slot instead of handing off+ tn.add(ks.keyActive(e.k!), MINUS_ONE);+ keyHandedOff = true; // accounted for; don't decrement again below+ } else {+ const keyHead = await popKeyPending(tn, ks, e.k!);+ if (keyHead) {+ keyHandedOff = true;+ // the key head now owns the freed key slot; it still needs a team slot+ if (teamSlotFree()) {+ promoteEntryToReady(tn, ks, keyHead, txc);+ teamHead = "consumed";+ } else if (!held.team) {+ await admitThroughTeamGate(tn, ks, keyHead, txc);+ } else {+ const loc = appendTeamPending(tn, ks, keyHead);+ setStatusPending(tn, ks, keyHead.i, loc);+ }+ }+ }+ }// Crawl slot handoff: promote the crawl-pending head out of the crawl gate.let crawlPromoted: QueueEntry | null = null;@@ -336,28 +417,58 @@ export async function releaseSlotsAndPromote(if (crawlPromoted) {const j2 = crawlPromoted;if (crawlDelaySeconds > 0) {- // crawl delay: park with a not-before timestamp; keeps the crawl slot.+ // crawl delay: park with a not-before timestamp; keeps the crawl slot+ // only (the key gate is passed on wake-up).const notBefore = now + crawlDelaySeconds * 1000;tn.set(ks.delayed(timeBucket(j2.i), notBefore, j2.i), encodeJson(j2));setStatusPending(tn, ks, j2.i, { k: "dl", at: notBefore });- } else if (held.team && !overLimit && teamHead === null) {- // the freed team slot goes directly to the crawl-promoted job- promoteEntryToReady(tn, ks, j2, txc);- teamHead = "consumed";- } else if (!held.team) {- // no team slot was freed here (job removal/cancellation paths), so the- // promoted job must be admitted through the gate on its own- await admitThroughTeamGate(tn, ks, j2, txc);} else {- // waits in the team gate; keeps the crawl slot- const loc = appendTeamPending(tn, ks, j2);- setStatusPending(tn, ks, j2.i, loc);+ // key gate for the crawl-promoted job+ const j2KeyGated = !!j2.k && !!(j2.f & F_KEY_GATED);+ let hasKeySlot = !j2KeyGated;+ if (!hasKeySlot) {+ if (holdsKey && !keyHandedOff && j2.k === e.k) {+ // the freed key slot goes directly to the crawl-promoted job+ keyHandedOff = true;+ hasKeySlot = true;+ } else {+ // rare path: acquire on its own (conflicting read of keyActive)+ const kLimitBuf = await tn.get(ks.keyLimit(j2.k!));+ const kLimit = kLimitBuf ? decodeI64(kLimitBuf) : Infinity;+ const kActive = decodeI64(await tn.get(ks.keyActive(j2.k!)));+ if (kActive < kLimit) {+ tn.add(ks.keyActive(j2.k!), ONE);+ hasKeySlot = true;+ }+ }+ }+ if (!hasKeySlot) {+ // waits in the key gate; keeps the crawl slot+ const loc = appendKeyPending(tn, ks, j2);+ setStatusPending(tn, ks, j2.i, loc);+ } else if (teamSlotFree()) {+ // the freed team slot goes directly to the crawl-promoted job+ promoteEntryToReady(tn, ks, j2, txc);+ teamHead = "consumed";+ } else if (!held.team) {+ // no team slot was freed here (job removal/cancellation paths), so the+ // promoted job must be admitted through the gate on its own+ await admitThroughTeamGate(tn, ks, j2, txc);+ } else {+ // waits in the team gate; keeps the crawl + key slots+ const loc = appendTeamPending(tn, ks, j2);+ setStatusPending(tn, ks, j2.i, loc);+ }}}+ if (holdsKey && !keyHandedOff) {+ tn.add(ks.keyActive(e.k!), MINUS_ONE);+ }+if (held.team) {if (teamHead === "consumed") {- // slot handed to crawlPromoted above+ // slot handed to the key head or crawlPromoted above} else if (teamHead !== null) {promoteEntryToReady(tn, ks, teamHead, txc);} else {@@ -366,9 +477,10 @@ export async function releaseSlotsAndPromote(}}-// Admits a slotless entry through the team gate, used when a job enters the-// team gate outside of a one-for-one handoff (delayed promotion, limit raise).-// Does a conflicting read of the team active counter -- callers are rare paths.+// Admits an entry holding no team slot through the team gate, used when a job+// enters the team gate outside of a one-for-one handoff (delayed promotion,+// limit raise). Does a conflicting read of the team active counter -- callers+// are rare paths.export async function admitThroughTeamGate(tn: Transaction,ks: NuqFdbKeyspace,@@ -387,6 +499,28 @@ export async function admitThroughTeamGate(}}+// Admits an entry holding only its crawl slot through the key gate and then+// the team gate (delayed promotion wake-up). Conflicting reads; rare path.+export async function admitThroughGates(+ tn: Transaction,+ ks: NuqFdbKeyspace,+ e: QueueEntry,+ txc: TxContext,+): Promise<void> {+ if (e.k && e.f & F_KEY_GATED) {+ const kLimitBuf = await tn.get(ks.keyLimit(e.k));+ const kLimit = kLimitBuf ? decodeI64(kLimitBuf) : Infinity;+ const kActive = decodeI64(await tn.get(ks.keyActive(e.k)));+ if (kActive >= kLimit) {+ const loc = appendKeyPending(tn, ks, e);+ setStatusPending(tn, ks, e.i, loc);+ return;+ }+ tn.add(ks.keyActive(e.k), ONE);+ }+ await admitThroughTeamGate(tn, ks, e, txc);+}+// === Job record cleanupexport function deleteJobRecords(diff --git a/apps/api/src/services/worker/nuq-fdb/queue.ts b/apps/api/src/services/worker/nuq-fdb/queue.tsindex e5a137b15..36d7c9ac5 100644--- a/apps/api/src/services/worker/nuq-fdb/queue.ts+++ b/apps/api/src/services/worker/nuq-fdb/queue.ts@@ -24,6 +24,7 @@ import {F_ZDR,F_COUNTABLE,F_GACC,+ F_KEY_GATED,normalizeOwnerId,} from "./keyspace";@@ -40,6 +41,7 @@ import {newTxContext,pushReady,appendTeamPending,+ appendKeyPending,appendCrawlPending,popTeamPending,setStatusQueued,@@ -110,6 +112,9 @@ export type NuQFdbGate = {// null = unlimited (self-hosted)teamLimit: number | null;queueCap: number;+ // API-key-scoped concurrency limit; null/absent = the key is unlimited.+ // Only applies when the batch is team-gated (teamLimit !== null).+ key?: { id: string; limit: number } | null;};type AddJobInput<Data> = {@@ -202,7 +207,9 @@ export class NuQFdbQueue<JobData = any, JobReturnValue = any> {throw new Error("addJobs requires all jobs to share an owner");}- const prepared = jobs.map(j => this.prepareAddJob(j, ownerId));+ const prepared = jobs.map(j =>+ this.prepareAddJob(j, ownerId, gate.key?.id ?? null),+ );const results: NuQFdbJob<JobData, JobReturnValue>[] = [];let batch: PreparedAddJob<JobData>[] = [];let batchBytes = 0;@@ -248,12 +255,14 @@ export class NuQFdbQueue<JobData = any, JobReturnValue = any> {private prepareAddJob(job: AddJobInput<JobData>,ownerId: string | null,+ keyId: string | null,): PreparedAddJob<JobData> {const dataBuf = Buffer.from(JSON.stringify(job.data ?? null), "utf8");const dataChunks = chunkBuffer(dataBuf, DATA_CHUNK_BYTES);const estimatedAffectedBytes = this.estimateEnqueueAffectedBytes(job,ownerId,+ keyId,dataChunks,);return {@@ -267,6 +276,7 @@ export class NuQFdbQueue<JobData = any, JobReturnValue = any> {private estimateEnqueueAffectedBytes(job: AddJobInput<JobData>,ownerId: string | null,+ keyId: string | null,dataChunks: Buffer[],): number {const ks = this.ks;@@ -278,8 +288,16 @@ export class NuQFdbQueue<JobData = any, JobReturnValue = any> {i: job.id,o: owner,g: gid,+ k: keyId ?? undefined,p: priority,- f: F_GATED | F_CRAWL_GATED | F_LISTENABLE | F_ZDR | F_COUNTABLE | F_GACC,+ f:+ F_GATED |+ F_CRAWL_GATED |+ F_KEY_GATED |+ F_LISTENABLE |+ F_ZDR |+ F_COUNTABLE |+ F_GACC,c: now,to: job.options.timesOutAt?.getTime(),};@@ -288,6 +306,7 @@ export class NuQFdbQueue<JobData = any, JobReturnValue = any> {p: priority,o: owner,g: gid,+ k: entry.k,f: entry.f,to: entry.to,dc: dataChunks.length,@@ -323,6 +342,18 @@ export class NuQFdbQueue<JobData = any, JobReturnValue = any> {encodedJsonBytes(entry);}+ if (keyId) {+ bytes +=+ ks.keyLimit(keyId).length ++ 8 ++ ks.keyActive(keyId).length ++ 8 ++ ks.keyPendingCount(keyId).length ++ 8 ++ ks.keyPendingKey(keyId, priority, now, job.id).length ++ encodedJsonBytes(entry);+ }+if (gid) {bytes +=ks.groupMeta(gid).length +@@ -388,6 +419,23 @@ export class NuQFdbQueue<JobData = any, JobReturnValue = any> {free = Math.max(0, gate.teamLimit - active);}+ // API-key gate state; only meaningful inside the team-gated world+ let keyFree = Infinity;+ const keyId = gate.teamLimit !== null && gate.key ? gate.key.id : null;+ if (keyId !== null && gate.key) {+ const storedBuf = await tn.get(ks.keyLimit(keyId));+ const stored = storedBuf ? decodeI64(storedBuf) : null;+ if (stored !== gate.key.limit) {+ tn.set(ks.keyLimit(keyId), encodeI64(gate.key.limit));+ if (stored !== null && gate.key.limit > stored) {+ tn.set(ks.taskKeyRaise(keyId), EMPTY);+ }+ }+ // key limits are small by definition: always the strict read+ const kActive = decodeI64(await tn.get(ks.keyActive(keyId)));+ keyFree = Math.max(0, gate.key.limit - kActive);+ }+// crawl gate state per distinct live groupconst groupMetas = new Map<string, GroupMeta | null>();const crawlFree = new Map<string, number>();@@ -408,6 +456,7 @@ export class NuQFdbQueue<JobData = any, JobReturnValue = any> {}let granted = 0;+ let keyGranted = 0;const crawlAcquired = new Map<string, number>();for (const j of jobs) {@@ -416,6 +465,7 @@ export class NuQFdbQueue<JobData = any, JobReturnValue = any> {const groupLive = !!gMeta && gMeta.s === "active";const gated = gate.teamLimit !== null && !j.options.bypassGate;const crawlGated = gated && !!gid && groupLive && crawlFree.has(gid!);+ const keyGated = gated && keyId !== null;const countable =this.options.hasGroups &&groupLive &&@@ -424,6 +474,7 @@ export class NuQFdbQueue<JobData = any, JobReturnValue = any> {let flags = 0;if (gated) flags |= F_GATED;if (crawlGated) flags |= F_CRAWL_GATED;+ if (keyGated) flags |= F_KEY_GATED;if (j.options.listenable) flags |= F_LISTENABLE;if ((j.data as any)?.zeroDataRetention) flags |= F_ZDR;if (countable) flags |= F_COUNTABLE;@@ -434,6 +485,7 @@ export class NuQFdbQueue<JobData = any, JobReturnValue = any> {i: j.id,o: ownerId ?? "",g: gid,+ k: keyGated ? keyId! : undefined,p: j.options.priority ?? 0,f: flags,c: now,@@ -445,6 +497,7 @@ export class NuQFdbQueue<JobData = any, JobReturnValue = any> {p: entry.p,o: entry.o,g: gid,+ k: entry.k,f: flags,to: timesOutAt,dc: j.dataChunks.length,@@ -463,11 +516,24 @@ export class NuQFdbQueue<JobData = any, JobReturnValue = any> {const loc = appendCrawlPending(tn, ks, entry);setStatusPending(tn, ks, j.id, loc);placedStatus = "pending";+ } else if (keyGated && keyFree <= 0) {+ // holds the crawl slot (if any) while waiting in the key gate+ if (crawlGated) {+ crawlFree.set(gid!, crawlFree.get(gid!)! - 1);+ crawlAcquired.set(gid!, (crawlAcquired.get(gid!) ?? 0) + 1);+ }+ const loc = appendKeyPending(tn, ks, entry);+ setStatusPending(tn, ks, j.id, loc);+ placedStatus = "pending";} else {if (crawlGated) {crawlFree.set(gid!, crawlFree.get(gid!)! - 1);crawlAcquired.set(gid!, (crawlAcquired.get(gid!) ?? 0) + 1);}+ if (keyGated) {+ keyFree--;+ keyGranted++;+ }if (free > 0) {free--;granted++;@@ -501,6 +567,9 @@ export class NuQFdbQueue<JobData = any, JobReturnValue = any> {if (granted > 0 && ownerId !== null) {bumpTeamActive(tn, ks, ownerId, granted);}+ if (keyGranted > 0 && keyId !== null) {+ tn.add(ks.keyActive(keyId), encodeI64(keyGranted));+ }for (const [gid, n] of crawlAcquired) {tn.add(ks.groupCrawlActive(gid), encodeI64(n));}@@ -662,7 +731,7 @@ export class NuQFdbQueue<JobData = any, JobReturnValue = any> {tn,ks,e,- { team: true, crawl: true },+ { team: true, key: true, crawl: true },now,txc,);@@ -837,6 +906,7 @@ export class NuQFdbQueue<JobData = any, JobReturnValue = any> {i: id,o: meta.o,g: meta.g,+ k: meta.k,p: meta.p,f: meta.f,c: meta.c,@@ -846,7 +916,7 @@ export class NuQFdbQueue<JobData = any, JobReturnValue = any> {tn,ks,entry,- { team: true, crawl: true },+ { team: true, key: true, crawl: true },now,txc,);@@ -997,6 +1067,7 @@ export class NuQFdbQueue<JobData = any, JobReturnValue = any> {i: id,o: meta.o,g: meta.g,+ k: meta.k,p: meta.p,f: meta.f,c: meta.c,@@ -1006,14 +1077,24 @@ export class NuQFdbQueue<JobData = any, JobReturnValue = any> {const accounted = !!(meta.f & F_GACC) && !!meta.g && !!this.groupOps;if (st.s === "pending") {- clearPendingPlacement(tn, ks, id, meta.o, meta.g, st.loc!, meta.to);- // team-pending and delayed jobs hold a crawl slot+ clearPendingPlacement(+ tn,+ ks,+ id,+ meta.o,+ meta.g,+ meta.k,+ st.loc!,+ meta.to,+ );+ // key-pending and delayed jobs hold a crawl slot; team-pending jobs+ // hold a key slot on topif (st.loc!.k !== "gq") {await releaseSlotsAndPromote(tn,ks,entry,- { team: false, crawl: true },+ { team: false, key: st.loc!.k === "tq", crawl: true },now,txc,);@@ -1037,7 +1118,7 @@ export class NuQFdbQueue<JobData = any, JobReturnValue = any> {tn,ks,entry,- { team: true, crawl: true },+ { team: true, key: true, crawl: true },now,txc,);diff --git a/apps/api/src/services/worker/nuq-fdb/slots.ts b/apps/api/src/services/worker/nuq-fdb/slots.tsindex f85d30a5e..4e7f3c20e 100644--- a/apps/api/src/services/worker/nuq-fdb/slots.ts+++ b/apps/api/src/services/worker/nuq-fdb/slots.ts@@ -106,7 +106,7 @@ export class NuqFdbExternalSlots {tn,this.ks,entry,- { team: true, crawl: false },+ { team: true, key: false, crawl: false },Date.now(),newTxContext(),);diff --git a/apps/api/src/services/worker/nuq-fdb/sweeper.ts b/apps/api/src/services/worker/nuq-fdb/sweeper.tsindex 803064a9a..a49f5b751 100644--- a/apps/api/src/services/worker/nuq-fdb/sweeper.ts+++ b/apps/api/src/services/worker/nuq-fdb/sweeper.ts@@ -9,6 +9,7 @@ import {GroupMeta,QueueEntry,decodeI64,+ encodeI64,decodeJson,encodeJson,timeBucket,@@ -17,6 +18,7 @@ import {F_CRAWL_GATED,F_COUNTABLE,F_GACC,+ F_KEY_GATED,} from "./keyspace";import {ONE,@@ -31,8 +33,10 @@ import {clearPendingPlacement,releaseSlotsAndPromote,admitThroughTeamGate,+ admitThroughGates,deleteJobRecords,popTeamPending,+ popKeyPending,setGroupJobIndex,bumpGroupStatusCount,bumpTeamActive,@@ -62,6 +66,7 @@ function entryFromMeta(id: string, meta: JobMeta): QueueEntry {i: id,o: meta.o,g: meta.g,+ k: meta.k,p: meta.p,f: meta.f,c: meta.c,@@ -164,6 +169,7 @@ export class NuqFdbSweeper {await this.sweepGroupFinishTasks(queue, now, logger);await this.sweepGroupCancelTasks(queue, now, logger);await this.sweepTeamRaiseTasks(queue, now, logger);+ await this.sweepKeyRaiseTasks(queue, now, logger);await this.sweepJobExpiry(queue, now, logger);await this.sweepGroupExpiry(queue, now, logger);}@@ -275,7 +281,7 @@ export class NuqFdbSweeper {tn,ks,entry,- { team: true, crawl: true },+ { team: true, key: true, crawl: true },now,txc,);@@ -325,13 +331,22 @@ export class NuqFdbSweeper {if (!st || st.s !== "pending" || !st.loc) return;const meta = decodeJson<JobMeta>(await tn.get(ks.jobMeta(id)));if (!meta) return;- clearPendingPlacement(tn, ks, id, meta.o, meta.g, st.loc, meta.to);- if (st.loc.k !== "gq" && meta.f & F_CRAWL_GATED) {+ clearPendingPlacement(+ tn,+ ks,+ id,+ meta.o,+ meta.g,+ meta.k,+ st.loc,+ meta.to,+ );+ if (st.loc.k !== "gq") {await releaseSlotsAndPromote(tn,ks,entryFromMeta(id, meta),- { team: false, crawl: true },+ { team: false, key: st.loc.k === "tq", crawl: true },now,txc,);@@ -378,8 +393,9 @@ export class NuqFdbSweeper {);tn.clear(key as Buffer);if (!st || st.s !== "pending" || st.loc?.k !== "dl") return;- // the job already holds its crawl slot; admit through the team gate- await admitThroughTeamGate(tn, ks, e, txc);+ // the job already holds its crawl slot; admit through the key gate+ // and then the team gate+ await admitThroughGates(tn, ks, e, txc);});stats.processedCount++;}@@ -459,12 +475,27 @@ export class NuqFdbSweeper {}const meta = decodeJson<JobMeta>(await tn.get(ks.jobMeta(id)));if (!meta) continue;- clearPendingPlacement(tn, ks, id, meta.o, meta.g, st.loc, meta.to);- // team-pending/delayed members hold a crawl slot; the group is- // cancelled so there is nothing to promote -- just release it+ clearPendingPlacement(+ tn,+ ks,+ id,+ meta.o,+ meta.g,+ meta.k,+ st.loc,+ meta.to,+ );+ // team-/key-pending/delayed members hold a crawl slot; the group+ // is cancelled so there is nothing to promote -- just release itif (st.loc.k !== "gq" && meta.f & F_CRAWL_GATED) {tn.add(ks.groupCrawlActive(gid), MINUS_ONE);}+ // team-pending members also hold a key slot; release it and let+ // the raise task hand it to key-pending jobs outside this group+ if (st.loc.k === "tq" && meta.k && meta.f & F_KEY_GATED) {+ tn.add(ks.keyActive(meta.k), MINUS_ONE);+ tn.set(ks.taskKeyRaise(meta.k), EMPTY);+ }tn.clear(mKey as Buffer);tn.add(ks.groupRemaining(gid), MINUS_ONE);if (meta.f & F_COUNTABLE)@@ -530,6 +561,45 @@ export class NuqFdbSweeper {}}+ // Key raises admit key-pending heads through the team gate: each promoted+ // job acquires a key slot here and a team slot (or a team-pending place)+ // in admitThroughTeamGate.+ private async sweepKeyRaiseTasks(+ queue: NuQFdbQueue,+ now: number,+ logger: Logger,+ ): Promise<void> {+ const ks = queue.ks;+ const r = ks.taskKeyRaiseRange();+ const tasks = await this.db.doTn(async tn =>+ tn.snapshot().getRangeAll(r.begin, r.end, { limit: 50 }),+ );+ for (const [key] of tasks) {+ const kid = ks.unpackId(key as Buffer);+ const done = await this.db.doTn(async tn => {+ const txc = newTxContext();+ const limitBuf = await tn.get(ks.keyLimit(kid));+ const limit = limitBuf ? decodeI64(limitBuf) : Infinity;+ const active = decodeI64(await tn.get(ks.keyActive(kid)));+ let free = Math.min(Math.max(0, limit - active), 32);+ let promoted = 0;+ while (free > 0) {+ const e = await popKeyPending(tn, ks, kid);+ if (!e) break;+ await admitThroughTeamGate(tn, ks, e, txc);+ promoted++;+ free--;+ }+ if (promoted > 0) tn.add(ks.keyActive(kid), encodeI64(promoted));+ // done when no free slots remain or the pending queue is drained+ return free > 0 || limit - active <= 0;+ });+ if (done) {+ await this.db.doTn(async tn => tn.clear(key as Buffer));+ }+ }+ }+// === Record GCprivate async sweepJobExpiry(diff --git a/apps/api/src/services/worker/nuq-router.ts b/apps/api/src/services/worker/nuq-router.tsindex 8748faf14..d8e9d5f00 100644--- a/apps/api/src/services/worker/nuq-router.ts+++ b/apps/api/src/services/worker/nuq-router.ts@@ -5,6 +5,7 @@ import { RateLimiterMode, ScrapeJobData } from "../../types";import { getACUCTeam } from "../../controllers/auth";import { redisEvictConnection } from "../../services/redis";import { isSelfHosted } from "../../lib/deployment";+import { getApiKeyConcurrencyLimit } from "../../lib/api-key-concurrency";import {getTeamQueueLimit,getConcurrencyLimitActiveJobsCount,@@ -257,6 +258,21 @@ export async function fdbEnqueueScrapeJobs(const queueCap =teamLimit === null ? Number.MAX_SAFE_INTEGER : getTeamQueueLimit(teamLimit);+ // API-key-scoped concurrency: applies when every job in the batch was+ // requested with the same key (batches always are; child jobs inherit the+ // kickoff's apiKeyId) and that key has a limit configured.+ let keyGate: { id: string; limit: number } | null = null;+ if (teamLimit !== null) {+ const keyIds = new Set(jobs.map(j => j.data.apiKeyId ?? null));+ const apiKeyId = keyIds.size === 1 ? [...keyIds][0] : null;+ if (apiKeyId !== null) {+ const keyLimit = await getApiKeyConcurrencyLimit(apiKeyId);+ if (keyLimit !== null) {+ keyGate = { id: String(apiKeyId), limit: keyLimit };+ }+ }+ }+const results = await optionalFdb(() =>scrapeQueueFdb.addJobs(jobs.map(j => ({@@ -274,7 +290,7 @@ export async function fdbEnqueueScrapeJobs(timesOutAt: new Date(Date.now() + j.backlogTimeoutMs),},})),- { teamLimit, queueCap },+ { teamLimit, queueCap, key: keyGate },),);