Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): send team concurrency with FirePDF async submits (#4051)
Abimael Martellcf5045315ded
diff --git a/apps/api/src/controllers/v2/parse.ts b/apps/api/src/controllers/v2/parse.tsindex e1e07d9db..f47a15917 100644--- a/apps/api/src/controllers/v2/parse.ts+++ b/apps/api/src/controllers/v2/parse.ts@@ -512,6 +512,7 @@ export async function parseController(bypassBilling: isDirectToBullMQ || !shouldBill,zeroDataRetention,teamFlags: req.acuc?.flags ?? null,+ teamConcurrency: req.acuc?.concurrency ?? null,uploadedFile: file,forceEngine,isParse: true,diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.tsindex e94d6d3d8..57b22730b 100644--- a/apps/api/src/controllers/v2/scrape.ts+++ b/apps/api/src/controllers/v2/scrape.ts@@ -328,6 +328,7 @@ export async function scrapeController(bypassBilling: isDirectToBullMQ || !shouldBill,zeroDataRetention,teamFlags: req.acuc?.flags ?? null,+ teamConcurrency: req.acuc?.concurrency ?? null,agentIndexOnly: (req as any).agentIndexOnly ?? false,threatProtection: threatProtection.policy ?? undefined,},diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.tsindex 90c2224b2..e3efdaaf4 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts@@ -7,6 +7,7 @@ vi.mock("../../../../../lib/gcs-pdf-cache", () => ({savePdfResultToCache: vi.fn(async () => null),}));+import {FirePdfAsyncFailure,scrapePDFWithFirePDFAsync,@@ -71,6 +72,7 @@ function makeMeta(overrides: Record<string, unknown> = {}) {internalOptions: {zeroDataRetention: false,teamId: "team-x",+ teamConcurrency: 12,crawlId: undefined,},options: {@@ -91,11 +93,18 @@ function makeFetchFromSequence(url: string;method: string;headers: Record<string, string> | undefined;+ body: unknown;}> = [];const cursor = { idx: 0 };const fetchImpl: any = async (url: string, init: any) => {const method = (init?.method ?? "GET").toUpperCase();- calls.push({ url, method, headers: init?.headers });+ let body: unknown;+ try {+ body = init?.body ? JSON.parse(init.body) : undefined;+ } catch {+ body = init?.body;+ }+ calls.push({ url, method, headers: init?.headers, body });const matcher = matchers[cursor.idx++];if (!matcher) {throw new Error(@@ -136,6 +145,7 @@ describe("scrapePDFWithFirePDFAsync", () => {internalOptions: {zeroDataRetention: true,teamId: "team-x",+ teamConcurrency: 12,crawlId: undefined,},});@@ -241,6 +251,49 @@ describe("scrapePDFWithFirePDFAsync", () => {expect(result.pagesProcessed).toBe(12);expect(fallback).not.toHaveBeenCalled();expect(calls).toHaveLength(4);+ // Account context rides the submit body (FirePDF ENG-5049).+ expect((calls[0].body as { team_concurrency?: number }).team_concurrency).toBe(12);+ });++ it("submits without team context when the snapshot is absent", async () => {+ const { fetchImpl, calls } = makeFetchFromSequence([+ {+ matchUrl: /\/jobs$/,+ matchMethod: "POST",+ response: {+ status: 200,+ body: { scrape_id: "scrape-id-test", status: "done", lane: "fast", retry_after_ms: 0 },+ },+ },+ {+ matchUrl: /\/jobs\/scrape-id-test\/result$/,+ matchMethod: "GET",+ response: {+ status: 200,+ body: {+ schema_version: 1,+ markdown: "# No context",+ pages_processed: 1,+ failed_pages: null,+ partial_pages: null,+ },+ },+ },+ ]);+ const fallback = vi.fn();++ const meta = makeMeta();+ meta.internalOptions.teamConcurrency = null;+ const result = await scrapePDFWithFirePDFAsync(meta, "BASE64", undefined, undefined, undefined, {+ fetchImpl,+ fallbackImpl: fallback,+ sleepImpl: noopSleep,+ });++ // Missing snapshot must never block the scrape β field simply absent.+ expect(result.markdown).toBe("# No context");+ expect((calls[0].body as { team_concurrency?: number }).team_concurrency).toBeUndefined();+ expect(fallback).not.toHaveBeenCalled();});it("cancels accepted work when polling is abandoned", async () => {@@ -282,7 +335,7 @@ describe("scrapePDFWithFirePDFAsync", () => {expect(error).toBeInstanceOf(FirePdfAsyncFailure);expect(error.reason).toBe("network_error");- expect(calls).toEqual([+ expect(calls.map(({ url, method, headers }) => ({ url, method, ...(headers !== undefined && { headers }) }))).toEqual([{url: "http://fire-pdf.test/jobs",method: "POST",@@ -401,7 +454,7 @@ describe("scrapePDFWithFirePDFAsync", () => {expect(err).toBeInstanceOf(FirePdfAsyncFailure);expect(err.reason).toBe("network_error");expect(fallback).not.toHaveBeenCalled();- expect(calls).toEqual([+ expect(calls.map(({ url, method }) => ({ url, method }))).toEqual([{ method: "POST", url: "http://fire-pdf.test/jobs" },{method: "DELETE",diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.tsindex cbe06c1e4..42cf54ffa 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts@@ -82,6 +82,16 @@ export async function scrapePDFWithFirePDFAsync(const deadlineAt = new Date(submitTime + deadlineFromNow).toISOString();const pollingDeadline = submitTime + deadlineFromNow + POLL_TIMEOUT_BUFFER_MS;+ // Account context for FirePDF's per-team admission observation,+ // snapshotted from the request ACUC into internalOptions at+ // acceptance (same pattern as teamFlags) β no re-fetch here. Absence+ // means FirePDF skips team observation for this submit.+ const rawConcurrency = meta.internalOptions.teamConcurrency;+ const teamConcurrency =+ typeof rawConcurrency === "number" && rawConcurrency > 0+ ? rawConcurrency+ : undefined;+// ββ Step 1: POST /jobs ββββββββββββββββββββββββββββββββββββββββββββββββlet submissionAccepted = false;let terminalReached = false;@@ -96,6 +106,7 @@ export async function scrapePDFWithFirePDFAsync(pagesProcessed,mode,deadlineAt,+ teamConcurrency,fetchImpl,});submissionAccepted = true;diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.tsindex e348889df..9d58ec993 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts@@ -20,6 +20,9 @@ type SubmitArgs = {pagesProcessed: number | undefined;mode: PDFMode | undefined;deadlineAt: string;+ /** Team's sold concurrency from the ACUC (ENG-5049 account context).+ * Optional: entitlement lookup must never block or fail a scrape. */+ teamConcurrency: number | undefined;fetchImpl: typeof undiciFetch;};@@ -56,6 +59,7 @@ export async function submitJob(args: SubmitArgs): Promise<SubmitOutcome> {pagesProcessed,mode,deadlineAt,+ teamConcurrency,fetchImpl,} = args;const scrapeId = meta.id;@@ -72,6 +76,12 @@ export async function submitJob(args: SubmitArgs): Promise<SubmitOutcome> {...(meta.internalOptions.crawlId && {crawl_id: meta.internalOptions.crawlId,}),+ // FirePDF per-team admission observation (its ENG-5049): sold+ // concurrency from the account context. Absence means FirePDF+ // skips team observation for this submit β never a rejection.+ ...(teamConcurrency !== undefined && {+ team_concurrency: teamConcurrency,+ }),options: {...(pagesProcessed !== undefined && { pages_estimate: pagesProcessed }),...(maxPages !== undefined && { max_pages: maxPages }),diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 5357e6167..904bd11cf 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -493,6 +493,10 @@ export type InternalOptions = {bypassBilling?: boolean;zeroDataRetention?: boolean;teamFlags?: TeamFlags;+ /** Team's sold concurrency, snapshotted from the request ACUC at+ * acceptance (same pattern as teamFlags). Rides the job payload so+ * downstream engines (FirePDF async account context) never re-fetch. */+ teamConcurrency?: number | null;/*** Effective threat protection policy for this scrape, resolved at the