Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(nuq): API-key-scoped concurrency limits on the FDB queue (#3961)

Gergő Móricz8671d1501f00
diff --git a/apps/api/src/__tests__/nuq-fdb/key-gate.test.ts b/apps/api/src/__tests__/nuq-fdb/key-gate.test.ts
new file mode 100644
index 000000000..39ff53632
--- /dev/null
+++ b/apps/api/src/__tests__/nuq-fdb/key-gate.test.ts
@@ -0,0 +1,360 @@
+import { randomUUID } from "crypto";
+import { config } from "../../config";
+import {
+ NuQFdbQueue,
+ NuQFdbJobGroup,
+ NuqFdbSweeper,
+} from "../../services/worker/nuq-fdb";
+import {
+ getNuqFdbDatabase,
+ getFdb,
+} from "../../services/worker/nuq-fdb/client";
+import { decodeI64 } from "../../services/worker/nuq-fdb/keyspace";
+
+// API-key concurrency gate tests, run directly against a real FoundationDB
+// cluster like core.test.ts. Skipped when FDB is not configured.
+const describeIf = config.FDB_CLUSTER_FILE ? describe : describe.skip;
+
+const RUN = randomUUID().slice(0, 8);
+const TEST_LEASE_MS = 1500;
+
+const createdQueueNames: string[] = [];
+
+type Ctx = {
+ queue: NuQFdbQueue;
+ group: NuQFdbJobGroup;
+ sweeper: NuqFdbSweeper;
+};
+
+async function makeCtx(name: string): Promise<Ctx> {
+ const scrapeName = `tk-${RUN}-${name}`;
+ const finishedName = `tk-${RUN}-${name}-fin`;
+ createdQueueNames.push(scrapeName, finishedName);
+ const queue = new NuQFdbQueue(scrapeName, {
+ hasGroups: true,
+ finishedQueueName: finishedName,
+ leaseMs: TEST_LEASE_MS,
+ });
+ const finishedQueue = new NuQFdbQueue(finishedName, { hasGroups: false });
+ const group = new NuQFdbJobGroup(queue.ks, queue.groupOps!);
+ const sweeper = new NuqFdbSweeper([queue, finishedQueue]);
+ return { queue, group, sweeper };
+}
+
+async function takeAll(
+ queue: NuQFdbQueue,
+ maxJobs: number = 50,
+): Promise<any[]> {
+ const out: any[] = [];
+ while (out.length < maxJobs) {
+ const job = await queue.getJobToProcess();
+ if (job === null) break;
+ out.push(job);
+ }
+ return out;
+}
+
+function freshOwner(): string {
+ return randomUUID();
+}
+
+function scrapeData(): any {
+ return { mode: "single_urls", url: "https://example.com" };
+}
+
+const kgate = (
+ teamLimit: number,
+ keyId: string,
+ keyLimit: number,
+ cap: number = 1_000_000,
+) => ({
+ teamLimit,
+ queueCap: cap,
+ key: { id: keyId, limit: keyLimit },
+});
+
+async function keyActiveCount(queue: NuQFdbQueue, kid: string) {
+ return await getNuqFdbDatabase().doTn(async tn =>
+ decodeI64(await tn.get(queue.ks.keyActive(kid))),
+ );
+}
+
+function jobInput(owner: string, groupId?: string) {
+ return {
+ id: randomUUID(),
+ data: scrapeData(),
+ options: {
+ ownerId: owner,
+ groupId,
+ timesOutAt: new Date(Date.now() + 60_000),
+ },
+ };
+}
+
+describeIf("NuQ FDB API-key gate", () => {
+ afterAll(async () => {
+ const fdb = getFdb();
+ const db = getNuqFdbDatabase();
+ for (const name of createdQueueNames) {
+ const r = fdb.tuple.range(["nuq", name]);
+ await db.doTn(async tn =>
+ tn.clearRange(r.begin as Buffer, r.end as Buffer),
+ );
+ }
+ });
+
+ test("key limit 2 admits 2, backlogs the rest, hands off on finish", async () => {
+ const { queue } = await makeCtx("basic");
+ const owner = freshOwner();
+ const kid = randomUUID();
+ const jobs = await queue.addJobs(
+ Array.from({ length: 5 }, () => jobInput(owner)),
+ kgate(10, kid, 2),
+ );
+
+ // the team would admit all 5; the key admits only 2
+ expect(jobs.filter(j => j.status === "queued").length).toBe(2);
+ expect(jobs.filter(j => j.status === "backlog").length).toBe(3);
+ expect(await queue.getTeamActiveCount(owner)).toBe(2);
+ expect(await queue.getTeamPendingCount(owner)).toBe(3);
+ expect(await keyActiveCount(queue, kid)).toBe(2);
+
+ const taken = await takeAll(queue, 5);
+ expect(taken.length).toBe(2);
+
+ // finishing one job hands its key slot to exactly one backlogged job
+ await queue.jobFinish(taken[0].id, taken[0].lock!, null);
+ expect(await keyActiveCount(queue, kid)).toBe(2);
+ expect(await queue.getTeamActiveCount(owner)).toBe(2);
+ expect(await queue.getTeamPendingCount(owner)).toBe(2);
+
+ const next = await takeAll(queue, 5);
+ expect(next.length).toBe(1);
+
+ // drain everything; counters return to zero
+ let inflight = [taken[1], ...next];
+ while (inflight.length > 0) {
+ for (const j of inflight) await queue.jobFinish(j.id, j.lock!, null);
+ inflight = await takeAll(queue, 5);
+ }
+ expect(await queue.getTeamActiveCount(owner)).toBe(0);
+ expect(await queue.getTeamPendingCount(owner)).toBe(0);
+ expect(await keyActiveCount(queue, kid)).toBe(0);
+ });
+
+ test("key gate cascades inside the crawl gate", async () => {
+ const { queue, group } = await makeCtx("cascade");
+ const owner = freshOwner();
+ const kid = randomUUID();
+ const gid = randomUUID();
+ await group.addGroup(gid, owner, undefined, { maxConcurrency: 3 });
+
+ await queue.addJobs(
+ Array.from({ length: 6 }, () => jobInput(owner, gid)),
+ kgate(10, kid, 2),
+ );
+
+ // crawl admits 3, key admits 2 of those: 2 ready, 1 key-pending,
+ // 3 crawl-pending
+ expect(await queue.getTeamActiveCount(owner)).toBe(2);
+ expect(await keyActiveCount(queue, kid)).toBe(2);
+
+ // the key limit stays the ceiling while all 6 jobs drain through
+ let finished = 0;
+ let inflight = await takeAll(queue, 6);
+ expect(inflight.length).toBe(2);
+ while (inflight.length > 0) {
+ expect(inflight.length).toBeLessThanOrEqual(2);
+ for (const j of inflight) {
+ await queue.jobFinish(j.id, j.lock!, null);
+ finished++;
+ }
+ expect(await keyActiveCount(queue, kid)).toBeLessThanOrEqual(2);
+ inflight = await takeAll(queue, 6);
+ }
+ expect(finished).toBe(6);
+
+ const g = await group.getGroup(gid);
+ expect(g?.status).toBe("completed");
+ expect(await queue.getTeamActiveCount(owner)).toBe(0);
+ expect(await keyActiveCount(queue, kid)).toBe(0);
+ });
+
+ test("key limit raise promotes backlogged jobs via the sweeper", async () => {
+ const { queue, sweeper } = await makeCtx("raise");
+ const owner = freshOwner();
+ const kid = randomUUID();
+ await queue.addJobs(
+ Array.from({ length: 3 }, () => jobInput(owner)),
+ kgate(10, kid, 1),
+ );
+ expect(await queue.getTeamActiveCount(owner)).toBe(1);
+ expect(await queue.getTeamPendingCount(owner)).toBe(2);
+
+ // a later enqueue arrives with a raised key limit
+ await queue.addJob(
+ randomUUID(),
+ scrapeData(),
+ { ownerId: owner, timesOutAt: new Date(Date.now() + 60_000) },
+ kgate(10, kid, 3),
+ );
+ await sweeper.sweepOnce();
+
+ expect(await queue.getTeamActiveCount(owner)).toBe(3);
+ expect(await queue.getTeamPendingCount(owner)).toBe(1);
+ expect(await keyActiveCount(queue, kid)).toBe(3);
+
+ let inflight = await takeAll(queue, 4);
+ expect(inflight.length).toBe(3);
+ let total = 0;
+ while (inflight.length > 0) {
+ for (const j of inflight) {
+ await queue.jobFinish(j.id, j.lock!, null);
+ total++;
+ }
+ inflight = await takeAll(queue, 4);
+ }
+ expect(total).toBe(4);
+ expect(await keyActiveCount(queue, kid)).toBe(0);
+ });
+
+ test("two keys gate independently under one team", async () => {
+ const { queue } = await makeCtx("twokeys");
+ const owner = freshOwner();
+ const kid1 = randomUUID();
+ const kid2 = randomUUID();
+
+ await queue.addJobs(
+ Array.from({ length: 2 }, () => jobInput(owner)),
+ kgate(10, kid1, 1),
+ );
+ await queue.addJobs(
+ Array.from({ length: 2 }, () => jobInput(owner)),
+ kgate(10, kid2, 1),
+ );
+
+ expect(await queue.getTeamActiveCount(owner)).toBe(2);
+ expect(await keyActiveCount(queue, kid1)).toBe(1);
+ expect(await keyActiveCount(queue, kid2)).toBe(1);
+
+ const taken = await takeAll(queue, 4);
+ expect(taken.length).toBe(2);
+
+ // finishing one key's job promotes that key's backlog only
+ await queue.jobFinish(taken[0].id, taken[0].lock!, null);
+ expect(await keyActiveCount(queue, kid1)).toBe(1);
+ expect(await keyActiveCount(queue, kid2)).toBe(1);
+ expect(await queue.getTeamPendingCount(owner)).toBe(1);
+
+ await queue.jobFinish(taken[1].id, taken[1].lock!, null);
+ let inflight = await takeAll(queue, 4);
+ let total = 2; // 2 taken initially
+ while (inflight.length > 0) {
+ total += inflight.length;
+ for (const j of inflight) await queue.jobFinish(j.id, j.lock!, null);
+ inflight = await takeAll(queue, 4);
+ }
+ expect(total).toBe(4);
+ expect(await keyActiveCount(queue, kid1)).toBe(0);
+ expect(await keyActiveCount(queue, kid2)).toBe(0);
+ });
+
+ test("removing a team-pending job hands its key slot to the key backlog", async () => {
+ const { queue } = await makeCtx("remove");
+ const owner = freshOwner();
+ const kid = randomUUID();
+ // team 1, key 2: A ready (team+key), B team-pending (holds key),
+ // C key-pending
+ const inputs = Array.from({ length: 3 }, () => jobInput(owner));
+ const jobs = await queue.addJobs(inputs, kgate(1, kid, 2));
+ const a = jobs.find(j => j.status === "queued")!;
+ expect(await keyActiveCount(queue, kid)).toBe(2);
+ expect(await queue.getTeamPendingCount(owner)).toBe(2);
+
+ // jobs are placed in input order: jobs[1] is the team-pending one (it
+ // won the second key slot), jobs[2] waits in the key gate
+ const b = jobs[1];
+ expect(b.status).toBe("backlog");
+ await queue.removeJob(b.id);
+ // its key slot moved to the key-pending job; nothing ran yet, so the
+ // key active count is unchanged and one job still waits for the team
+ expect(await keyActiveCount(queue, kid)).toBe(2);
+ expect(await queue.getTeamPendingCount(owner)).toBe(1);
+
+ const [takenA] = await takeAll(queue, 2);
+ expect(takenA.id).toBe(a.id);
+ await queue.jobFinish(takenA.id, takenA.lock!, null);
+
+ const [takenC] = await takeAll(queue, 2);
+ expect(takenC).toBeDefined();
+ await queue.jobFinish(takenC.id, takenC.lock!, null);
+
+ expect(await queue.getTeamActiveCount(owner)).toBe(0);
+ expect(await queue.getTeamPendingCount(owner)).toBe(0);
+ expect(await keyActiveCount(queue, kid)).toBe(0);
+ });
+
+ test("key-pending jobs are silently dropped at their backlog deadline", async () => {
+ const { queue, sweeper } = await makeCtx("bto");
+ const owner = freshOwner();
+ const kid = randomUUID();
+ const inputs = Array.from({ length: 3 }, () => ({
+ id: randomUUID(),
+ data: scrapeData(),
+ options: {
+ ownerId: owner,
+ timesOutAt: new Date(Date.now() + 1000),
+ },
+ }));
+ const jobs = await queue.addJobs(inputs, kgate(10, kid, 1));
+ expect(jobs.filter(j => j.status === "backlog").length).toBe(2);
+
+ await new Promise(resolve => setTimeout(resolve, 1200));
+ await sweeper.sweepOnce();
+
+ // the two key-pending jobs are gone; the running one is unaffected
+ expect(await queue.getTeamPendingCount(owner)).toBe(0);
+ expect(await keyActiveCount(queue, kid)).toBe(1);
+ for (const j of jobs.filter(j => j.status === "backlog")) {
+ expect(await queue.getJob(j.id)).toBeNull();
+ }
+
+ const taken = await takeAll(queue, 3);
+ expect(taken.length).toBe(1);
+ await queue.jobFinish(taken[0].id, taken[0].lock!, null);
+ expect(await keyActiveCount(queue, kid)).toBe(0);
+ });
+
+ test("crawl delay wake-up passes through the key gate", async () => {
+ const { queue, group, sweeper } = await makeCtx("delay");
+ const owner = freshOwner();
+ const kid = randomUUID();
+ const gid = randomUUID();
+ await group.addGroup(gid, owner, undefined, { delaySeconds: 1 });
+
+ const inputs = [jobInput(owner, gid), jobInput(owner, gid)];
+ await queue.addJobs(inputs, kgate(10, kid, 1));
+
+ const [first] = await takeAll(queue, 2);
+ expect(first).toBeDefined();
+ await queue.jobFinish(first.id, first.lock!, null);
+ // the second job is parked in the delay index holding only its crawl
+ // slot; the finished job's key slot was released
+ expect(await keyActiveCount(queue, kid)).toBe(0);
+
+ await sweeper.sweepOnce();
+ expect(await queue.getJobToProcess()).toBeNull();
+
+ await new Promise(resolve => setTimeout(resolve, 1200));
+ await sweeper.sweepOnce();
+ // the wake-up acquired the key slot again
+ expect(await keyActiveCount(queue, kid)).toBe(1);
+ const [second] = await takeAll(queue, 1);
+ expect(second).toBeDefined();
+ await queue.jobFinish(second.id, second.lock!, null);
+ expect(await keyActiveCount(queue, kid)).toBe(0);
+
+ const g = await group.getGroup(gid);
+ expect(g?.status).toBe("completed");
+ });
+});
diff --git a/apps/api/src/db/schema/public.ts b/apps/api/src/db/schema/public.ts
index fa29eb7ae..9af4190d5 100644
--- a/apps/api/src/db/schema/public.ts
+++ b/apps/api/src/db/schema/public.ts
@@ -73,6 +73,8 @@ export const api_keys = pgTable(
team_id: uuid("team_id"),
owner_id: uuid("owner_id"),
agent_provisioned: boolean("agent_provisioned").default(false),
+ // API-key-scoped concurrency limit; null = key inherits the team limit only
+ concurrency: integer("concurrency"),
},
table => [
// Target of key_restriction_config's composite FK, which pins a
diff --git a/apps/api/src/lib/api-key-concurrency.ts b/apps/api/src/lib/api-key-concurrency.ts
new file mode 100644
index 000000000..7128d11ba
--- /dev/null
+++ b/apps/api/src/lib/api-key-concurrency.ts
@@ -0,0 +1,74 @@
+import { eq } from "drizzle-orm";
+import { dbRr } from "../db/connection";
+import * as schema from "../db/schema";
+import { getValue, setValue } from "../services/redis";
+import { logger } from "./logger";
+
+// Propagation delay for edits to api_keys.concurrency.
+const LIMIT_CACHE_TTL_SECONDS = 60;
+
+const limitCacheKey = (apiKeyId: number) => `api-key-concurrency:${apiKeyId}`;
+
+/**
+ * Returns the API-key-scoped concurrency limit (api_keys.concurrency), or
+ * null when the key has no limit of its own. Cached for a minute per key.
+ *
+ * Fails open (null): unlike the IP allowlist this is a throttle, not a
+ * security boundary, and a transient DB/cache error must not stall enqueues.
+ */
+export async function getApiKeyConcurrencyLimit(
+ apiKeyId: number,
+): Promise<number | null> {
+ const cacheKey = limitCacheKey(apiKeyId);
+
+ try {
+ const cached = await getValue(cacheKey);
+ if (cached !== null) {
+ // only the explicit negative-cache sentinel means "no limit"; anything
+ // else malformed is a cache miss so a corrupted entry cannot silently
+ // disable the key's gate
+ if (cached === "none") return null;
+ const parsed = Number(cached);
+ if (Number.isInteger(parsed) && parsed > 0) return parsed;
+ logger.warn("Ignoring malformed API key concurrency cache entry", {
+ apiKeyId,
+ });
+ }
+ } catch (error) {
+ logger.warn("Failed to read API key concurrency cache", {
+ apiKeyId,
+ error,
+ });
+ }
+
+ let limit: number | null = null;
+ try {
+ const [row] = await dbRr
+ .select({ concurrency: schema.api_keys.concurrency })
+ .from(schema.api_keys)
+ .where(eq(schema.api_keys.id, apiKeyId))
+ .limit(1);
+ limit =
+ typeof row?.concurrency === "number" && row.concurrency > 0
+ ? row.concurrency
+ : null;
+ } catch (error) {
+ logger.warn("Failed to load API key concurrency limit", {
+ apiKeyId,
+ error,
+ });
+ return null;
+ }
+
+ try {
+ // "none" is a cached negative so unlimited keys skip the DB read too
+ await setValue(cacheKey, String(limit ?? "none"), LIMIT_CACHE_TTL_SECONDS);
+ } catch (error) {
+ logger.warn("Failed to cache API key concurrency limit", {
+ apiKeyId,
+ error,
+ });
+ }
+
+ return limit;
+}
diff --git a/apps/api/src/services/worker/nuq-fdb/keyspace.ts b/apps/api/src/services/worker/nuq-fdb/keyspace.ts
index e84622ea9..38e95f6fb 100644
--- a/apps/api/src/services/worker/nuq-fdb/keyspace.ts
+++ b/apps/api/src/services/worker/nuq-fdb/keyspace.ts
@@ -25,11 +25,13 @@ export const F_LISTENABLE = 4;
export const F_ZDR = 8;
export const F_COUNTABLE = 16; // mode === "single_urls": counted in group numeric stats
export const F_GACC = 32; // participates in group remaining-count accounting
+export const F_KEY_GATED = 64; // holds an API-key slot while team-pending/queued/active
// Where a pending job's queue entry lives, stored on its status record so
// sweepers and removers can clear the exact key without scanning.
export type PendingLoc =
| { k: "tq"; s: number; p: number; c: number } // team-pending: shard, priority, createdAtMs
+ | { k: "kq"; p: number; c: number } // key-pending: priority, createdAtMs
| { k: "gq"; p: number; c: number } // crawl-pending: priority, createdAtMs
| { k: "dl"; at: number }; // delay index: notBeforeMs
@@ -38,6 +40,7 @@ export type JobMeta = {
p: number; // priority
o: string; // ownerId (normalized uuid)
g?: string; // groupId
+ k?: string; // API key id (set only when key-gated)
f: number; // flags
to?: number; // backlog timesOutAt ms
dc: number; // data chunk count
@@ -57,6 +60,7 @@ export type QueueEntry = {
i: string; // jobId
o: string; // ownerId
g?: string; // groupId
+ k?: string; // API key id (set only when key-gated)
p: number; // priority
f: number; // flags
c: number; // createdAt ms
@@ -202,6 +206,28 @@ export class NuqFdbKeyspace {
return this.packRange(["t", tid, "q", shard]);
}
+ // === API-key gate (sits between the crawl gate and the team gate)
+ keyLimit(kid: string): Buffer {
+ return this.pack(["k", kid, "limit"]);
+ }
+ keyActive(kid: string): Buffer {
+ return this.pack(["k", kid, "active"]);
+ }
+ keyPendingCount(kid: string): Buffer {
+ return this.pack(["k", kid, "qn"]);
+ }
+ keyPendingKey(
+ kid: string,
+ priority: number,
+ createdAtMs: number,
+ id: string,
+ ): Buffer {
+ return this.pack(["k", kid, "q", priority, createdAtMs, id]);
+ }
+ keyPendingRange(kid: string) {
+ return this.packRange(["k", kid, "q"]);
+ }
+
// === Group (crawl) records
groupMeta(gid: string): Buffer {
return this.pack(["g", gid, "meta"]);
@@ -340,6 +366,12 @@ export class NuqFdbKeyspace {
taskTeamRaiseRange() {
return this.packRange(["task", "traise"]);
}
+ taskKeyRaise(kid: string): Buffer {
+ return this.pack(["task", "kraise", kid]);
+ }
+ taskKeyRaiseRange() {
+ return this.packRange(["task", "kraise"]);
+ }
sweeperLock(): Buffer {
return this.pack(["sweep", "lock"]);
}
diff --git a/apps/api/src/services/worker/nuq-fdb/ops.ts b/apps/api/src/services/worker/nuq-fdb/ops.ts
index 24526174d..130570078 100644
--- a/apps/api/src/services/worker/nuq-fdb/ops.ts
+++ b/apps/api/src/services/worker/nuq-fdb/ops.ts
@@ -17,6 +17,7 @@ import {
F_GATED,
F_CRAWL_GATED,
F_COUNTABLE,
+ F_KEY_GATED,
} from "./keyspace";
export const ONE = encodeI64(1);
@@ -112,6 +113,20 @@ export function appendTeamPending(
return { k: "tq", s: shard, p: e.p, c: e.c };
}
+export function appendKeyPending(
+ tn: Transaction,
+ ks: NuqFdbKeyspace,
+ e: QueueEntry,
+): PendingLoc {
+ tn.set(ks.keyPendingKey(e.k!, e.p, e.c, e.i), encodeJson(e));
+ tn.add(ks.keyPendingCount(e.k!), ONE);
+ tn.add(ks.teamPendingCount(e.o), ONE);
+ if (e.to !== undefined) {
+ tn.set(ks.backlogTimeout(timeBucket(e.i), e.to, e.i), EMPTY);
+ }
+ return { k: "kq", p: e.p, c: e.c };
+}
+
export function appendCrawlPending(
tn: Transaction,
ks: NuqFdbKeyspace,
@@ -158,10 +173,12 @@ export function pendingKeyFromLoc(
id: string,
ownerId: string,
groupId: string | undefined,
+ keyId: string | undefined,
loc: PendingLoc,
): Buffer {
if (loc.k === "tq")
return ks.teamPendingKey(ownerId, loc.s, loc.p, loc.c, id);
+ if (loc.k === "kq") return ks.keyPendingKey(keyId!, loc.p, loc.c, id);
if (loc.k === "gq") return ks.groupPendingKey(groupId!, loc.p, loc.c, id);
return ks.delayed(timeBucket(id), loc.at, id);
}
@@ -174,13 +191,17 @@ export function clearPendingPlacement(
id: string,
ownerId: string,
groupId: string | undefined,
+ keyId: string | undefined,
loc: PendingLoc,
timesOutAt: number | undefined,
): void {
- tn.clear(pendingKeyFromLoc(ks, id, ownerId, groupId, loc));
+ tn.clear(pendingKeyFromLoc(ks, id, ownerId, groupId, keyId, loc));
if (loc.k === "tq") {
tn.add(ks.teamShardCount(ownerId, loc.s), MINUS_ONE);
tn.add(ks.teamPendingCount(ownerId), MINUS_ONE);
+ } else if (loc.k === "kq") {
+ tn.add(ks.keyPendingCount(keyId!), MINUS_ONE);
+ tn.add(ks.teamPendingCount(ownerId), MINUS_ONE);
} else if (loc.k === "gq") {
tn.add(ks.groupPendingCount(groupId!), MINUS_ONE);
tn.add(ks.teamPendingCount(ownerId), MINUS_ONE);
@@ -255,6 +276,26 @@ export async function popTeamPending(
return null;
}
+// Like popCrawlPending, key-pending is a single range: concurrent finishers of
+// the same key conflict on the head, but per-key concurrency is small by
+// definition (it is the key's limit), so the retry pressure stays bounded.
+export async function popKeyPending(
+ tn: Transaction,
+ ks: NuqFdbKeyspace,
+ kid: string,
+): Promise<QueueEntry | null> {
+ const r = ks.keyPendingRange(kid);
+ const head = await tn.getRangeAll(r.begin, r.end, { limit: 1 });
+ if (head.length === 0) return null;
+ const [key, value] = head[0];
+ const e = decodeJson<QueueEntry>(value as Buffer)!;
+ tn.clear(key as Buffer);
+ tn.add(ks.keyPendingCount(kid), MINUS_ONE);
+ tn.add(ks.teamPendingCount(e.o), MINUS_ONE);
+ clearBacklogTimeout(tn, ks, e);
+ return e;
+}
+
export async function popCrawlPending(
tn: Transaction,
ks: NuqFdbKeyspace,
@@ -278,24 +319,29 @@ export async function popCrawlPending(
// the same transaction whenever possible, so this function never does a
// conflicting read of the team active counter on the handoff path.
//
+// The gate chain is crawl -> key -> team -> ready. A job entering an inner
+// gate keeps the outer slots it already won.
+//
// `held` describes which slots the leaving job actually holds:
-// - active / ready jobs: team + crawl (if crawl-gated)
-// - team-pending / delayed jobs: crawl only (if crawl-gated)
+// - active / ready jobs: team + key (if key-gated) + crawl (if crawl-gated)
+// - team-pending jobs: key + crawl only
+// - key-pending / delayed jobs: crawl only
// - crawl-pending jobs: none
export async function releaseSlotsAndPromote(
tn: Transaction,
ks: NuqFdbKeyspace,
e: QueueEntry,
- held: { team: boolean; crawl: boolean },
+ held: { team: boolean; key: boolean; crawl: boolean },
now: number,
txc: TxContext,
): Promise<void> {
if (!(e.f & F_GATED)) return;
const tid = e.o;
+ const holdsKey = held.key && !!e.k && !!(e.f & F_KEY_GATED);
const holdsCrawl = held.crawl && !!e.g && !!(e.f & F_CRAWL_GATED);
- if (!held.team && !holdsCrawl) return;
+ if (!held.team && !holdsKey && !holdsCrawl) return;
// Limit-lowering convergence: if the team is over its limit, don't hand off.
let overLimit = false;
@@ -313,6 +359,41 @@ export async function releaseSlotsAndPromote(
if (held.team && !overLimit) {
teamHead = await popTeamPending(tn, ks, tid);
}
+ // whether the freed team slot is still unassigned after the pops so far
+ const teamSlotFree = () => held.team && !overLimit && teamHead === null;
+
+ // Key slot handoff: promote the key-pending head into the team gate. Runs
+ // before the crawl handoff so the longest-gated job gets the slot first.
+ let keyHandedOff = false;
+ if (holdsKey) {
+ let keyOverLimit = false;
+ const kLimitBuf = await tn.get(ks.keyLimit(e.k!)); // cold key
+ if (kLimitBuf) {
+ const kLimit = decodeI64(kLimitBuf);
+ const kActive = decodeI64(await tn.snapshot().get(ks.keyActive(e.k!)));
+ keyOverLimit = kActive > kLimit;
+ }
+ if (keyOverLimit) {
+ // limit-lowering convergence: swallow the slot instead of handing off
+ tn.add(ks.keyActive(e.k!), MINUS_ONE);
+ keyHandedOff = true; // accounted for; don't decrement again below
+ } else {
+ const keyHead = await popKeyPending(tn, ks, e.k!);
+ if (keyHead) {
+ keyHandedOff = true;
+ // the key head now owns the freed key slot; it still needs a team slot
+ if (teamSlotFree()) {
+ promoteEntryToReady(tn, ks, keyHead, txc);
+ teamHead = "consumed";
+ } else if (!held.team) {
+ await admitThroughTeamGate(tn, ks, keyHead, txc);
+ } else {
+ const loc = appendTeamPending(tn, ks, keyHead);
+ setStatusPending(tn, ks, keyHead.i, loc);
+ }
+ }
+ }
+ }
// Crawl slot handoff: promote the crawl-pending head out of the crawl gate.
let crawlPromoted: QueueEntry | null = null;
@@ -336,28 +417,58 @@ export async function releaseSlotsAndPromote(
if (crawlPromoted) {
const j2 = crawlPromoted;
if (crawlDelaySeconds > 0) {
- // crawl delay: park with a not-before timestamp; keeps the crawl slot.
+ // crawl delay: park with a not-before timestamp; keeps the crawl slot
+ // only (the key gate is passed on wake-up).
const notBefore = now + crawlDelaySeconds * 1000;
tn.set(ks.delayed(timeBucket(j2.i), notBefore, j2.i), encodeJson(j2));
setStatusPending(tn, ks, j2.i, { k: "dl", at: notBefore });
- } else if (held.team && !overLimit && teamHead === null) {
- // the freed team slot goes directly to the crawl-promoted job
- promoteEntryToReady(tn, ks, j2, txc);
- teamHead = "consumed";
- } else if (!held.team) {
- // no team slot was freed here (job removal/cancellation paths), so the
- // promoted job must be admitted through the gate on its own
- await admitThroughTeamGate(tn, ks, j2, txc);
} else {
- // waits in the team gate; keeps the crawl slot
- const loc = appendTeamPending(tn, ks, j2);
- setStatusPending(tn, ks, j2.i, loc);
+ // key gate for the crawl-promoted job
+ const j2KeyGated = !!j2.k && !!(j2.f & F_KEY_GATED);
+ let hasKeySlot = !j2KeyGated;
+ if (!hasKeySlot) {
+ if (holdsKey && !keyHandedOff && j2.k === e.k) {
+ // the freed key slot goes directly to the crawl-promoted job
+ keyHandedOff = true;
+ hasKeySlot = true;
+ } else {
+ // rare path: acquire on its own (conflicting read of keyActive)
+ const kLimitBuf = await tn.get(ks.keyLimit(j2.k!));
+ const kLimit = kLimitBuf ? decodeI64(kLimitBuf) : Infinity;
+ const kActive = decodeI64(await tn.get(ks.keyActive(j2.k!)));
+ if (kActive < kLimit) {
+ tn.add(ks.keyActive(j2.k!), ONE);
+ hasKeySlot = true;
+ }
+ }
+ }
+ if (!hasKeySlot) {
+ // waits in the key gate; keeps the crawl slot
+ const loc = appendKeyPending(tn, ks, j2);
+ setStatusPending(tn, ks, j2.i, loc);
+ } else if (teamSlotFree()) {
+ // the freed team slot goes directly to the crawl-promoted job
+ promoteEntryToReady(tn, ks, j2, txc);
+ teamHead = "consumed";
+ } else if (!held.team) {
+ // no team slot was freed here (job removal/cancellation paths), so the
+ // promoted job must be admitted through the gate on its own
+ await admitThroughTeamGate(tn, ks, j2, txc);
+ } else {
+ // waits in the team gate; keeps the crawl + key slots
+ const loc = appendTeamPending(tn, ks, j2);
+ setStatusPending(tn, ks, j2.i, loc);
+ }
}
}
+ if (holdsKey && !keyHandedOff) {
+ tn.add(ks.keyActive(e.k!), MINUS_ONE);
+ }
+
if (held.team) {
if (teamHead === "consumed") {
- // slot handed to crawlPromoted above
+ // slot handed to the key head or crawlPromoted above
} else if (teamHead !== null) {
promoteEntryToReady(tn, ks, teamHead, txc);
} else {
@@ -366,9 +477,10 @@ export async function releaseSlotsAndPromote(
}
}
-// Admits a slotless entry through the team gate, used when a job enters the
-// team gate outside of a one-for-one handoff (delayed promotion, limit raise).
-// Does a conflicting read of the team active counter -- callers are rare paths.
+// Admits an entry holding no team slot through the team gate, used when a job
+// enters the team gate outside of a one-for-one handoff (delayed promotion,
+// limit raise). Does a conflicting read of the team active counter -- callers
+// are rare paths.
export async function admitThroughTeamGate(
tn: Transaction,
ks: NuqFdbKeyspace,
@@ -387,6 +499,28 @@ export async function admitThroughTeamGate(
}
}
+// Admits an entry holding only its crawl slot through the key gate and then
+// the team gate (delayed promotion wake-up). Conflicting reads; rare path.
+export async function admitThroughGates(
+ tn: Transaction,
+ ks: NuqFdbKeyspace,
+ e: QueueEntry,
+ txc: TxContext,
+): Promise<void> {
+ if (e.k && e.f & F_KEY_GATED) {
+ const kLimitBuf = await tn.get(ks.keyLimit(e.k));
+ const kLimit = kLimitBuf ? decodeI64(kLimitBuf) : Infinity;
+ const kActive = decodeI64(await tn.get(ks.keyActive(e.k)));
+ if (kActive >= kLimit) {
+ const loc = appendKeyPending(tn, ks, e);
+ setStatusPending(tn, ks, e.i, loc);
+ return;
+ }
+ tn.add(ks.keyActive(e.k), ONE);
+ }
+ await admitThroughTeamGate(tn, ks, e, txc);
+}
+
// === Job record cleanup
export function deleteJobRecords(
diff --git a/apps/api/src/services/worker/nuq-fdb/queue.ts b/apps/api/src/services/worker/nuq-fdb/queue.ts
index e5a137b15..36d7c9ac5 100644
--- a/apps/api/src/services/worker/nuq-fdb/queue.ts
+++ b/apps/api/src/services/worker/nuq-fdb/queue.ts
@@ -24,6 +24,7 @@ import {
F_ZDR,
F_COUNTABLE,
F_GACC,
+ F_KEY_GATED,
normalizeOwnerId,
} from "./keyspace";
@@ -40,6 +41,7 @@ import {
newTxContext,
pushReady,
appendTeamPending,
+ appendKeyPending,
appendCrawlPending,
popTeamPending,
setStatusQueued,
@@ -110,6 +112,9 @@ export type NuQFdbGate = {
// null = unlimited (self-hosted)
teamLimit: number | null;
queueCap: number;
+ // API-key-scoped concurrency limit; null/absent = the key is unlimited.
+ // Only applies when the batch is team-gated (teamLimit !== null).
+ key?: { id: string; limit: number } | null;
};
type AddJobInput<Data> = {
@@ -202,7 +207,9 @@ export class NuQFdbQueue<JobData = any, JobReturnValue = any> {
throw new Error("addJobs requires all jobs to share an owner");
}
- const prepared = jobs.map(j => this.prepareAddJob(j, ownerId));
+ const prepared = jobs.map(j =>
+ this.prepareAddJob(j, ownerId, gate.key?.id ?? null),
+ );
const results: NuQFdbJob<JobData, JobReturnValue>[] = [];
let batch: PreparedAddJob<JobData>[] = [];
let batchBytes = 0;
@@ -248,12 +255,14 @@ export class NuQFdbQueue<JobData = any, JobReturnValue = any> {
private prepareAddJob(
job: AddJobInput<JobData>,
ownerId: string | null,
+ keyId: string | null,
): PreparedAddJob<JobData> {
const dataBuf = Buffer.from(JSON.stringify(job.data ?? null), "utf8");
const dataChunks = chunkBuffer(dataBuf, DATA_CHUNK_BYTES);
const estimatedAffectedBytes = this.estimateEnqueueAffectedBytes(
job,
ownerId,
+ keyId,
dataChunks,
);
return {
@@ -267,6 +276,7 @@ export class NuQFdbQueue<JobData = any, JobReturnValue = any> {
private estimateEnqueueAffectedBytes(
job: AddJobInput<JobData>,
ownerId: string | null,
+ keyId: string | null,
dataChunks: Buffer[],
): number {
const ks = this.ks;
@@ -278,8 +288,16 @@ export class NuQFdbQueue<JobData = any, JobReturnValue = any> {
i: job.id,
o: owner,
g: gid,
+ k: keyId ?? undefined,
p: priority,
- f: F_GATED | F_CRAWL_GATED | F_LISTENABLE | F_ZDR | F_COUNTABLE | F_GACC,
+ f:
+ F_GATED |
+ F_CRAWL_GATED |
+ F_KEY_GATED |
+ F_LISTENABLE |
+ F_ZDR |
+ F_COUNTABLE |
+ F_GACC,
c: now,
to: job.options.timesOutAt?.getTime(),
};
@@ -288,6 +306,7 @@ export class NuQFdbQueue<JobData = any, JobReturnValue = any> {
p: priority,
o: owner,
g: gid,
+ k: entry.k,
f: entry.f,
to: entry.to,
dc: dataChunks.length,
@@ -323,6 +342,18 @@ export class NuQFdbQueue<JobData = any, JobReturnValue = any> {
encodedJsonBytes(entry);
}
+ if (keyId) {
+ bytes +=
+ ks.keyLimit(keyId).length +
+ 8 +
+ ks.keyActive(keyId).length +
+ 8 +
+ ks.keyPendingCount(keyId).length +
+ 8 +
+ ks.keyPendingKey(keyId, priority, now, job.id).length +
+ encodedJsonBytes(entry);
+ }
+
if (gid) {
bytes +=
ks.groupMeta(gid).length +
@@ -388,6 +419,23 @@ export class NuQFdbQueue<JobData = any, JobReturnValue = any> {
free = Math.max(0, gate.teamLimit - active);
}
+ // API-key gate state; only meaningful inside the team-gated world
+ let keyFree = Infinity;
+ const keyId = gate.teamLimit !== null && gate.key ? gate.key.id : null;
+ if (keyId !== null && gate.key) {
+ const storedBuf = await tn.get(ks.keyLimit(keyId));
+ const stored = storedBuf ? decodeI64(storedBuf) : null;
+ if (stored !== gate.key.limit) {
+ tn.set(ks.keyLimit(keyId), encodeI64(gate.key.limit));
+ if (stored !== null && gate.key.limit > stored) {
+ tn.set(ks.taskKeyRaise(keyId), EMPTY);
+ }
+ }
+ // key limits are small by definition: always the strict read
+ const kActive = decodeI64(await tn.get(ks.keyActive(keyId)));
+ keyFree = Math.max(0, gate.key.limit - kActive);
+ }
+
// crawl gate state per distinct live group
const groupMetas = new Map<string, GroupMeta | null>();
const crawlFree = new Map<string, number>();
@@ -408,6 +456,7 @@ export class NuQFdbQueue<JobData = any, JobReturnValue = any> {
}
let granted = 0;
+ let keyGranted = 0;
const crawlAcquired = new Map<string, number>();
for (const j of jobs) {
@@ -416,6 +465,7 @@ export class NuQFdbQueue<JobData = any, JobReturnValue = any> {
const groupLive = !!gMeta && gMeta.s === "active";
const gated = gate.teamLimit !== null && !j.options.bypassGate;
const crawlGated = gated && !!gid && groupLive && crawlFree.has(gid!);
+ const keyGated = gated && keyId !== null;
const countable =
this.options.hasGroups &&
groupLive &&
@@ -424,6 +474,7 @@ export class NuQFdbQueue<JobData = any, JobReturnValue = any> {
let flags = 0;
if (gated) flags |= F_GATED;
if (crawlGated) flags |= F_CRAWL_GATED;
+ if (keyGated) flags |= F_KEY_GATED;
if (j.options.listenable) flags |= F_LISTENABLE;
if ((j.data as any)?.zeroDataRetention) flags |= F_ZDR;
if (countable) flags |= F_COUNTABLE;
@@ -434,6 +485,7 @@ export class NuQFdbQueue<JobData = any, JobReturnValue = any> {
i: j.id,
o: ownerId ?? "",
g: gid,
+ k: keyGated ? keyId! : undefined,
p: j.options.priority ?? 0,
f: flags,
c: now,
@@ -445,6 +497,7 @@ export class NuQFdbQueue<JobData = any, JobReturnValue = any> {
p: entry.p,
o: entry.o,
g: gid,
+ k: entry.k,
f: flags,
to: timesOutAt,
dc: j.dataChunks.length,
@@ -463,11 +516,24 @@ export class NuQFdbQueue<JobData = any, JobReturnValue = any> {
const loc = appendCrawlPending(tn, ks, entry);
setStatusPending(tn, ks, j.id, loc);
placedStatus = "pending";
+ } else if (keyGated && keyFree <= 0) {
+ // holds the crawl slot (if any) while waiting in the key gate
+ if (crawlGated) {
+ crawlFree.set(gid!, crawlFree.get(gid!)! - 1);
+ crawlAcquired.set(gid!, (crawlAcquired.get(gid!) ?? 0) + 1);
+ }
+ const loc = appendKeyPending(tn, ks, entry);
+ setStatusPending(tn, ks, j.id, loc);
+ placedStatus = "pending";
} else {
if (crawlGated) {
crawlFree.set(gid!, crawlFree.get(gid!)! - 1);
crawlAcquired.set(gid!, (crawlAcquired.get(gid!) ?? 0) + 1);
}
+ if (keyGated) {
+ keyFree--;
+ keyGranted++;
+ }
if (free > 0) {
free--;
granted++;
@@ -501,6 +567,9 @@ export class NuQFdbQueue<JobData = any, JobReturnValue = any> {
if (granted > 0 && ownerId !== null) {
bumpTeamActive(tn, ks, ownerId, granted);
}
+ if (keyGranted > 0 && keyId !== null) {
+ tn.add(ks.keyActive(keyId), encodeI64(keyGranted));
+ }
for (const [gid, n] of crawlAcquired) {
tn.add(ks.groupCrawlActive(gid), encodeI64(n));
}
@@ -662,7 +731,7 @@ export class NuQFdbQueue<JobData = any, JobReturnValue = any> {
tn,
ks,
e,
- { team: true, crawl: true },
+ { team: true, key: true, crawl: true },
now,
txc,
);
@@ -837,6 +906,7 @@ export class NuQFdbQueue<JobData = any, JobReturnValue = any> {
i: id,
o: meta.o,
g: meta.g,
+ k: meta.k,
p: meta.p,
f: meta.f,
c: meta.c,
@@ -846,7 +916,7 @@ export class NuQFdbQueue<JobData = any, JobReturnValue = any> {
tn,
ks,
entry,
- { team: true, crawl: true },
+ { team: true, key: true, crawl: true },
now,
txc,
);
@@ -997,6 +1067,7 @@ export class NuQFdbQueue<JobData = any, JobReturnValue = any> {
i: id,
o: meta.o,
g: meta.g,
+ k: meta.k,
p: meta.p,
f: meta.f,
c: meta.c,
@@ -1006,14 +1077,24 @@ export class NuQFdbQueue<JobData = any, JobReturnValue = any> {
const accounted = !!(meta.f & F_GACC) && !!meta.g && !!this.groupOps;
if (st.s === "pending") {
- clearPendingPlacement(tn, ks, id, meta.o, meta.g, st.loc!, meta.to);
- // team-pending and delayed jobs hold a crawl slot
+ clearPendingPlacement(
+ tn,
+ ks,
+ id,
+ meta.o,
+ meta.g,
+ meta.k,
+ st.loc!,
+ meta.to,
+ );
+ // key-pending and delayed jobs hold a crawl slot; team-pending jobs
+ // hold a key slot on top
if (st.loc!.k !== "gq") {
await releaseSlotsAndPromote(
tn,
ks,
entry,
- { team: false, crawl: true },
+ { team: false, key: st.loc!.k === "tq", crawl: true },
now,
txc,
);
@@ -1037,7 +1118,7 @@ export class NuQFdbQueue<JobData = any, JobReturnValue = any> {
tn,
ks,
entry,
- { team: true, crawl: true },
+ { team: true, key: true, crawl: true },
now,
txc,
);
diff --git a/apps/api/src/services/worker/nuq-fdb/slots.ts b/apps/api/src/services/worker/nuq-fdb/slots.ts
index f85d30a5e..4e7f3c20e 100644
--- a/apps/api/src/services/worker/nuq-fdb/slots.ts
+++ b/apps/api/src/services/worker/nuq-fdb/slots.ts
@@ -106,7 +106,7 @@ export class NuqFdbExternalSlots {
tn,
this.ks,
entry,
- { team: true, crawl: false },
+ { team: true, key: false, crawl: false },
Date.now(),
newTxContext(),
);
diff --git a/apps/api/src/services/worker/nuq-fdb/sweeper.ts b/apps/api/src/services/worker/nuq-fdb/sweeper.ts
index 803064a9a..a49f5b751 100644
--- a/apps/api/src/services/worker/nuq-fdb/sweeper.ts
+++ b/apps/api/src/services/worker/nuq-fdb/sweeper.ts
@@ -9,6 +9,7 @@ import {
GroupMeta,
QueueEntry,
decodeI64,
+ encodeI64,
decodeJson,
encodeJson,
timeBucket,
@@ -17,6 +18,7 @@ import {
F_CRAWL_GATED,
F_COUNTABLE,
F_GACC,
+ F_KEY_GATED,
} from "./keyspace";
import {
ONE,
@@ -31,8 +33,10 @@ import {
clearPendingPlacement,
releaseSlotsAndPromote,
admitThroughTeamGate,
+ admitThroughGates,
deleteJobRecords,
popTeamPending,
+ popKeyPending,
setGroupJobIndex,
bumpGroupStatusCount,
bumpTeamActive,
@@ -62,6 +66,7 @@ function entryFromMeta(id: string, meta: JobMeta): QueueEntry {
i: id,
o: meta.o,
g: meta.g,
+ k: meta.k,
p: meta.p,
f: meta.f,
c: meta.c,
@@ -164,6 +169,7 @@ export class NuqFdbSweeper {
await this.sweepGroupFinishTasks(queue, now, logger);
await this.sweepGroupCancelTasks(queue, now, logger);
await this.sweepTeamRaiseTasks(queue, now, logger);
+ await this.sweepKeyRaiseTasks(queue, now, logger);
await this.sweepJobExpiry(queue, now, logger);
await this.sweepGroupExpiry(queue, now, logger);
}
@@ -275,7 +281,7 @@ export class NuqFdbSweeper {
tn,
ks,
entry,
- { team: true, crawl: true },
+ { team: true, key: true, crawl: true },
now,
txc,
);
@@ -325,13 +331,22 @@ export class NuqFdbSweeper {
if (!st || st.s !== "pending" || !st.loc) return;
const meta = decodeJson<JobMeta>(await tn.get(ks.jobMeta(id)));
if (!meta) return;
- clearPendingPlacement(tn, ks, id, meta.o, meta.g, st.loc, meta.to);
- if (st.loc.k !== "gq" && meta.f & F_CRAWL_GATED) {
+ clearPendingPlacement(
+ tn,
+ ks,
+ id,
+ meta.o,
+ meta.g,
+ meta.k,
+ st.loc,
+ meta.to,
+ );
+ if (st.loc.k !== "gq") {
await releaseSlotsAndPromote(
tn,
ks,
entryFromMeta(id, meta),
- { team: false, crawl: true },
+ { team: false, key: st.loc.k === "tq", crawl: true },
now,
txc,
);
@@ -378,8 +393,9 @@ export class NuqFdbSweeper {
);
tn.clear(key as Buffer);
if (!st || st.s !== "pending" || st.loc?.k !== "dl") return;
- // the job already holds its crawl slot; admit through the team gate
- await admitThroughTeamGate(tn, ks, e, txc);
+ // the job already holds its crawl slot; admit through the key gate
+ // and then the team gate
+ await admitThroughGates(tn, ks, e, txc);
});
stats.processedCount++;
}
@@ -459,12 +475,27 @@ export class NuqFdbSweeper {
}
const meta = decodeJson<JobMeta>(await tn.get(ks.jobMeta(id)));
if (!meta) continue;
- clearPendingPlacement(tn, ks, id, meta.o, meta.g, st.loc, meta.to);
- // team-pending/delayed members hold a crawl slot; the group is
- // cancelled so there is nothing to promote -- just release it
+ clearPendingPlacement(
+ tn,
+ ks,
+ id,
+ meta.o,
+ meta.g,
+ meta.k,
+ st.loc,
+ meta.to,
+ );
+ // team-/key-pending/delayed members hold a crawl slot; the group
+ // is cancelled so there is nothing to promote -- just release it
if (st.loc.k !== "gq" && meta.f & F_CRAWL_GATED) {
tn.add(ks.groupCrawlActive(gid), MINUS_ONE);
}
+ // team-pending members also hold a key slot; release it and let
+ // the raise task hand it to key-pending jobs outside this group
+ if (st.loc.k === "tq" && meta.k && meta.f & F_KEY_GATED) {
+ tn.add(ks.keyActive(meta.k), MINUS_ONE);
+ tn.set(ks.taskKeyRaise(meta.k), EMPTY);
+ }
tn.clear(mKey as Buffer);
tn.add(ks.groupRemaining(gid), MINUS_ONE);
if (meta.f & F_COUNTABLE)
@@ -530,6 +561,45 @@ export class NuqFdbSweeper {
}
}
+ // Key raises admit key-pending heads through the team gate: each promoted
+ // job acquires a key slot here and a team slot (or a team-pending place)
+ // in admitThroughTeamGate.
+ private async sweepKeyRaiseTasks(
+ queue: NuQFdbQueue,
+ now: number,
+ logger: Logger,
+ ): Promise<void> {
+ const ks = queue.ks;
+ const r = ks.taskKeyRaiseRange();
+ const tasks = await this.db.doTn(async tn =>
+ tn.snapshot().getRangeAll(r.begin, r.end, { limit: 50 }),
+ );
+ for (const [key] of tasks) {
+ const kid = ks.unpackId(key as Buffer);
+ const done = await this.db.doTn(async tn => {
+ const txc = newTxContext();
+ const limitBuf = await tn.get(ks.keyLimit(kid));
+ const limit = limitBuf ? decodeI64(limitBuf) : Infinity;
+ const active = decodeI64(await tn.get(ks.keyActive(kid)));
+ let free = Math.min(Math.max(0, limit - active), 32);
+ let promoted = 0;
+ while (free > 0) {
+ const e = await popKeyPending(tn, ks, kid);
+ if (!e) break;
+ await admitThroughTeamGate(tn, ks, e, txc);
+ promoted++;
+ free--;
+ }
+ if (promoted > 0) tn.add(ks.keyActive(kid), encodeI64(promoted));
+ // done when no free slots remain or the pending queue is drained
+ return free > 0 || limit - active <= 0;
+ });
+ if (done) {
+ await this.db.doTn(async tn => tn.clear(key as Buffer));
+ }
+ }
+ }
+
// === Record GC
private async sweepJobExpiry(
diff --git a/apps/api/src/services/worker/nuq-router.ts b/apps/api/src/services/worker/nuq-router.ts
index 8748faf14..d8e9d5f00 100644
--- a/apps/api/src/services/worker/nuq-router.ts
+++ b/apps/api/src/services/worker/nuq-router.ts
@@ -5,6 +5,7 @@ import { RateLimiterMode, ScrapeJobData } from "../../types";
import { getACUCTeam } from "../../controllers/auth";
import { redisEvictConnection } from "../../services/redis";
import { isSelfHosted } from "../../lib/deployment";
+import { getApiKeyConcurrencyLimit } from "../../lib/api-key-concurrency";
import {
getTeamQueueLimit,
getConcurrencyLimitActiveJobsCount,
@@ -257,6 +258,21 @@ export async function fdbEnqueueScrapeJobs(
const queueCap =
teamLimit === null ? Number.MAX_SAFE_INTEGER : getTeamQueueLimit(teamLimit);
+ // API-key-scoped concurrency: applies when every job in the batch was
+ // requested with the same key (batches always are; child jobs inherit the
+ // kickoff's apiKeyId) and that key has a limit configured.
+ let keyGate: { id: string; limit: number } | null = null;
+ if (teamLimit !== null) {
+ const keyIds = new Set(jobs.map(j => j.data.apiKeyId ?? null));
+ const apiKeyId = keyIds.size === 1 ? [...keyIds][0] : null;
+ if (apiKeyId !== null) {
+ const keyLimit = await getApiKeyConcurrencyLimit(apiKeyId);
+ if (keyLimit !== null) {
+ keyGate = { id: String(apiKeyId), limit: keyLimit };
+ }
+ }
+ }
+
const results = await optionalFdb(() =>
scrapeQueueFdb.addJobs(
jobs.map(j => ({
@@ -274,7 +290,7 @@ export async function fdbEnqueueScrapeJobs(
timesOutAt: new Date(Date.now() + j.backlogTimeoutMs),
},
})),
- { teamLimit, queueCap },
+ { teamLimit, queueCap, key: keyGate },
),
);