Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): read the PDF service's region echo for US-only teams (#5553)
capy-ai[bot]4e7c68116dec
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.tsindex e7eb4fcb4..72315057e 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts@@ -482,6 +482,7 @@ describe("FirePDF table format", () => {includeBlocks: false,pageMarkers: false,trainingCapture,+ dataRegion: undefined,};expect(buildFirePdfJobOptions({ ...base, tableFormat: "dynamic" })).toEqual({diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFUsOnly.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFUsOnly.test.tsindex f11aab842..7edeb78e9 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFUsOnly.test.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFUsOnly.test.ts@@ -18,6 +18,7 @@ import { UsOnlyUnavailableError } from "../../../error";import { scrapePDFWithFirePDF } from "../firePDF";import { scrapePDFWithFirePDFAsync } from "../fire-pdf/async";import { lookupAdoptableFirePdfJob } from "../fire-pdf/lookup";+import { buildFirePdfJobOptions, FirePdfAsyncFailure } from "../fire-pdf/utils";import {makeFetchFromSequence,makeMeta as makeAsyncMeta,@@ -73,10 +74,10 @@ function failureStatus(status: number, body: unknown) {});}-const regionRefusal = {- code: "region_unavailable",- data_region: "us",- error: "no in-region capacity",+// The PDF service's sync answer when every in-region GPU acquire was refused.+const pinnedNoCapacity = {+ error: "No OCR pods available",+ data_region_refusals: { region_unroutable: 3 },};const ORIGINAL_BASE_URL = config.FIRE_PDF_BASE_URL;@@ -99,7 +100,7 @@ describe("FirePDF sync parse for US-only teams", () => {markdown: "Document",failed_pages: null,pages_processed: 1,- data_region: "us",+ data_region_applied: "us",} as any);await scrapePDFWithFirePDF(makeSyncMeta(), "BASE64", 1);@@ -116,7 +117,7 @@ describe("FirePDF sync parse for US-only teams", () => {markdown: "Document",failed_pages: null,pages_processed: 1,- data_region: "us",+ data_region_applied: "us",} as any);const result = await scrapePDFWithFirePDF(@@ -129,7 +130,7 @@ describe("FirePDF sync parse for US-only teams", () => {});it("refuses a US-only parse the PDF service could not place on US capacity", async () => {- mockedRobustFetch.mockRejectedValue(failureStatus(503, regionRefusal));+ mockedRobustFetch.mockRejectedValue(failureStatus(503, pinnedNoCapacity));const error = await scrapePDFWithFirePDF(makeSyncMeta(usOnlyFlags),@@ -142,17 +143,37 @@ describe("FirePDF sync parse for US-only teams", () => {expect(error.message).toContain("PDF parsing is not available");});+ it("keeps a US-only failure without region refusals on its own error", async () => {+ const failure = failureStatus(502, { error: "All 3 pages failed" });+ mockedRobustFetch.mockRejectedValue(failure);++ const error = await scrapePDFWithFirePDF(+ makeSyncMeta(usOnlyFlags),+ "BASE64",+ 1,+ ).catch(e => e);++ expect(error).toBe(failure);+ });+it.each([- ["missing", undefined],- ["null", null],+ ["missing", {}],+ ["null", { data_region_applied: null }],+ [+ "null with refusals",+ {+ data_region_applied: null,+ data_region_refusals: { region_unverified: 1 },+ },+ ],])("refuses a US-only result whose region echo is %s",- async (_name, dataRegion) => {+ async (_name, echo) => {mockedRobustFetch.mockResolvedValue({markdown: "Document",failed_pages: null,pages_processed: 1,- data_region: dataRegion,+ ...echo,} as any);const error = await scrapePDFWithFirePDF(@@ -167,7 +188,7 @@ describe("FirePDF sync parse for US-only teams", () => {);it("names image OCR when a US-only image is refused", async () => {- mockedRobustFetch.mockRejectedValue(failureStatus(503, regionRefusal));+ mockedRobustFetch.mockRejectedValue(failureStatus(503, pinnedNoCapacity));const error = await scrapePDFWithFirePDF(makeSyncMeta(usOnlyFlags),@@ -186,7 +207,7 @@ describe("FirePDF sync parse for US-only teams", () => {});it("keeps an unpinned failure and an unpinned result without the echo unchanged", async () => {- const failure = failureStatus(503, regionRefusal);+ const failure = failureStatus(503, pinnedNoCapacity);mockedRobustFetch.mockRejectedValueOnce(failure);expect(await scrapePDFWithFirePDF(makeSyncMeta(), "BASE64", 1).catch(e => e),@@ -248,18 +269,18 @@ function runAsync(meta: any, fetchImpl: any) {}describe("FirePDF async jobs for US-only teams", () => {- it("sends data_region on the submit only for a US-only team", async () => {+ it("sends data_region in the job options only for a US-only team", async () => {const unpinned = makeFetchFromSequence([submitAccepted,pollDone,- resultWith({}),+ resultWith({ data_region_applied: null }),]);await runAsync(makeUsOnlyAsyncMeta(), unpinned.fetchImpl);const pinned = makeFetchFromSequence([submitAccepted,pollDone,- resultWith({ data_region: "us" }),+ resultWith({ data_region_applied: "us" }),]);const result = await runAsync(makeUsOnlyAsyncMeta(usOnlyFlags),@@ -267,7 +288,9 @@ describe("FirePDF async jobs for US-only teams", () => {);expect(unpinned.calls[0].body).not.toHaveProperty("data_region");- expect(pinned.calls[0].body).toMatchObject({ data_region: "us" });+ expect(unpinned.calls[0].body).not.toHaveProperty("options.data_region");+ expect(pinned.calls[0].body).not.toHaveProperty("data_region");+ expect(pinned.calls[0].body).toHaveProperty("options.data_region", "us");expect(result.markdown).toBe("# Async");});@@ -281,10 +304,11 @@ describe("FirePDF async jobs for US-only teams", () => {status: 502,body: {scrape_id: "scrape-id-test",+ data_region_applied: "us",status: "failed",- error_class: "region_unavailable",- error_message: "no in-region capacity",- error_code: "region_unavailable",+ error_class: "ocr_unavailable",+ error_message:+ "No OCR pods available [data_region=us refused: region_unroutable=4]",},},},@@ -299,12 +323,43 @@ describe("FirePDF async jobs for US-only teams", () => {expect(calls.map(call => call.method)).toEqual(["POST", "GET"]);});- it("refuses a US-only submit the PDF service turned away for lack of US capacity, without retrying", async () => {- const { fetchImpl, calls } = makeFetchFromSequence([+ it("keeps a US-only job that finished in the region despite refused acquires", async () => {+ const { fetchImpl } = makeFetchFromSequence([+ submitAccepted,+ {+ ...pollDone,+ response: {+ status: 200,+ body: {+ ...pollDone.response.body,+ data_region_refusals: { region_cap: 1 },+ },+ },+ },+ resultWith({ data_region_applied: "us" }),+ ]);++ const result = await runAsync(makeUsOnlyAsyncMeta(usOnlyFlags), fetchImpl);++ expect(result.markdown).toBe("# Async");+ });++ it("keeps a US-only job failure without region refusals off the US-only refusal", async () => {+ const { fetchImpl } = makeFetchFromSequence([+ submitAccepted,{- matchUrl: /\/jobs$/,- matchMethod: "POST",- response: { status: 503, body: regionRefusal },+ matchUrl: /\/jobs\/scrape-id-test$/,+ matchMethod: "GET",+ response: {+ status: 502,+ body: {+ scrape_id: "scrape-id-test",+ data_region_applied: "us",+ status: "failed",+ error_class: "render_failed",+ error_message: "Batch render failed",+ },+ },},]);@@ -313,9 +368,8 @@ describe("FirePDF async jobs for US-only teams", () => {fetchImpl,).catch(e => e);- expect(error).toBeInstanceOf(UsOnlyUnavailableError);- expect(error.message).toContain("no US capacity could take this document");- expect(calls).toHaveLength(1);+ expect(error).toBeInstanceOf(FirePdfAsyncFailure);+ expect(error.reason).toBe("terminal_failed");});it("refuses a US-only result that lacks the region echo and caches nothing", async () => {@@ -334,7 +388,7 @@ describe("FirePDF async jobs for US-only teams", () => {expect(savePdfResultToCache).not.toHaveBeenCalled();});- it("sends data_region on the content lookup only for a US-only team", async () => {+ it("forwards data_region from the caller's job options to the content lookup", async () => {const lookupHit = {matchUrl: /\/jobs\/lookup$/,matchMethod: "POST" as const,@@ -347,23 +401,38 @@ describe("FirePDF async jobs for US-only teams", () => {const pinned = makeFetchFromSequence([lookupHit]);const sha = "ef".repeat(32);+ const jobOptions = (dataRegion?: string) =>+ buildFirePdfJobOptions({+ maxPages: undefined,+ pagesProcessed: undefined,+ mode: "auto",+ includePageMarkdown: false,+ includeBlocks: false,+ pageMarkers: false,+ tableFormat: undefined,+ trainingCapture: undefined,+ dataRegion,+ });+await lookupAdoptableFirePdfJob(makeUsOnlyAsyncMeta(),sha,- { mode: "auto" },+ jobOptions(),unpinned.fetchImpl,);await lookupAdoptableFirePdfJob(makeUsOnlyAsyncMeta(usOnlyFlags),sha,- { mode: "auto" },+ jobOptions("us"),pinned.fetchImpl,);expect(unpinned.calls[0].body).not.toHaveProperty("data_region");- expect(pinned.calls[0].body).toMatchObject({+ expect(unpinned.calls[0].body).toHaveProperty("options", { mode: "auto" });+ expect(pinned.calls[0].body).not.toHaveProperty("data_region");+ expect(pinned.calls[0].body).toHaveProperty("options", {+ mode: "auto",data_region: "us",- options: { mode: "auto" },});});});diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.tsindex 57f9667ac..0d7f51ef4 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts@@ -394,7 +394,7 @@ export async function scrapePDFWithFirePDFAsync(// echo, and its result may have come from GPU capacity outside the US.if (isUsOnly(meta.internalOptions.teamFlags ?? undefined) &&- fetched.data_region !== US_DATA_REGION+ fetched.data_region_applied !== US_DATA_REGION) {throw firePdfUsOnlyRefusal(meta, "region_unconfirmed");}diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/background-refresh.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/background-refresh.tsindex 88b0826e2..ad62f5210 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/background-refresh.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/background-refresh.ts@@ -144,6 +144,8 @@ async function submit(tableFormat: undefined,// Internal work under no team: never offered for training capture.trainingCapture: undefined,+ // US-only teams never reach here: their refreshes are skipped above.+ dataRegion: undefined,}),refresh: true,},diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference-flow.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference-flow.tsindex 8dda6556b..2e9d400fc 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference-flow.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference-flow.ts@@ -10,6 +10,7 @@ import {import { cacheKeyShape, tryGetCached } from "./cache";import { lookupAdoptableFirePdfJob } from "./lookup";import { buildFirePdfJobOptions, FirePdfAsyncFailure } from "./utils";+import { isUsOnly, US_DATA_REGION } from "../../../../../lib/us-only";import {buildFirePdfJobTrainingCapture,firePdfTableFormat,@@ -151,6 +152,9 @@ export async function runFirePdfByReferenceAttempt(pageMarkers,tableFormat: firePdfTableFormat(meta),trainingCapture: buildFirePdfJobTrainingCapture(meta),+ dataRegion: isUsOnly(meta.internalOptions.teamFlags ?? undefined)+ ? US_DATA_REGION+ : undefined,}),);meta.abort.throwIfAborted();diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/data-region.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/data-region.tsindex 14123aef9..784c72655 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/data-region.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/data-region.ts@@ -2,6 +2,39 @@ import type { Meta } from "../../..";import { UsOnlyUnavailableError } from "../../../error";import type { FirePdfSourceKind } from "./request-metadata";+/**+ * Whether a pinned PDF service failure says the GPU acquires it needed in the+ * region were refused: `data_region_refusals` on a sync error body, or the+ * refusal note a failed job appends to its `error_message`. Only call it on a+ * failure: a job that finished in the region can still report refused acquires.+ */+export function firePdfRegionRefused(body: unknown): boolean {+ let parsed = body;+ if (typeof body === "string") {+ try {+ parsed = JSON.parse(body);+ } catch {+ return false;+ }+ }+ if (typeof parsed !== "object" || parsed === null) return false;+ const { data_region_refusals, error_message } = parsed as Record<+ string,+ unknown+ >;+ if (+ typeof data_region_refusals === "object" &&+ data_region_refusals !== null &&+ !Array.isArray(data_region_refusals) &&+ Object.keys(data_region_refusals).length > 0+ )+ return true;+ return (+ typeof error_message === "string" &&+ /\[data_region=\w+ refused: /.test(error_message)+ );+}+/** The error a US-only team gets when the PDF service did not run, or did not confirm running, its document on US capacity. */export function firePdfUsOnlyRefusal(meta: Meta,diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/lookup.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/lookup.tsindex 9f215f9df..473edec00 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/lookup.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/lookup.ts@@ -8,7 +8,6 @@ import type { Meta } from "../../..";import { config } from "../../../../../config";import { fetch as undiciFetch } from "undici";import { firePdfHeaders } from "./utils";-import { isUsOnly, US_DATA_REGION } from "../../../../../lib/us-only";/** Handle for adopting an existing fire-pdf job instead of submitting a* new one. Produced by {@link lookupAdoptableFirePdfJob}: the job was@@ -72,9 +71,6 @@ export async function lookupAdoptableFirePdfJob(team_id: meta.internalOptions.teamId,}),options,- ...(isUsOnly(meta.internalOptions.teamFlags ?? undefined) && {- data_region: US_DATA_REGION,- }),}),signal,});diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/poll.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/poll.tsindex 6d440eff2..782851c50 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/poll.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/poll.ts@@ -1,7 +1,8 @@import type { Meta } from "../../..";import { fetch as undiciFetch } from "undici";import { AbortManagerThrownError } from "../../../lib/abortManager";-import { isRegionUnavailable, isUsOnly } from "../../../../../lib/us-only";+import { isUsOnly } from "../../../../../lib/us-only";+import { firePdfRegionRefused } from "./data-region";import {firePdfAsyncCompletedTotal,firePdfAsyncLongPollTotal,@@ -174,7 +175,8 @@ export async function pollUntilTerminal(deps: PollDeps): Promise<PollOk> {if (isUsOnly(meta.internalOptions.teamFlags ?? undefined) &&- isRegionUnavailable(pollBody)+ (pollBody as { status?: unknown }).status === "failed" &&+ firePdfRegionRefused(pollBody)) {if (waitMs > 0) firePdfAsyncLongPollTotal.labels("terminal").inc();firePdfAsyncPollCount.observe(pollCount);diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.tsindex fdf8d4820..37cb1feb3 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts@@ -301,7 +301,7 @@ export const resultResponseSchema = z.object({// Raw on purpose: parsed separately by parseProvenance so a stamp this// build does not understand never fails the scrape.provenance: z.unknown().optional(),- data_region: z.unknown().optional(),+ data_region_applied: z.unknown().optional(),});export type PollResponse = z.infer<typeof pollResponseSchema>;diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.tsindex 02459d586..2de06c019 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts@@ -2,11 +2,7 @@ import type { Meta } from "../../..";import type { PDFMode } from "../../../../../controllers/v2/types";import { fetch as undiciFetch } from "undici";import { AbortManagerThrownError } from "../../../lib/abortManager";-import {- isRegionUnavailable,- isUsOnly,- US_DATA_REGION,-} from "../../../../../lib/us-only";+import { isUsOnly, US_DATA_REGION } from "../../../../../lib/us-only";import {buildFirePdfJobTrainingCapture,buildFirePdfRequestMetadata,@@ -169,8 +165,8 @@ export async function submitJob(args: SubmitArgs): Promise<SubmitOutcome> {pageMarkers,tableFormat: firePdfTableFormat(meta),trainingCapture: buildFirePdfJobTrainingCapture(meta),+ dataRegion: usOnly ? US_DATA_REGION : undefined,}),- ...(usOnly && { data_region: US_DATA_REGION }),};// The retry does reach a different instance: Fastify's shutdown 503@@ -223,9 +219,6 @@ export async function submitJob(args: SubmitArgs): Promise<SubmitOutcome> {}failPossiblyAccepted(meta, "network_error", { error: String(error) });}- if (usOnly && isRegionUnavailable(json)) {- failAsync(meta, "region_unavailable", { status });- }const transient503 = attempt === 1 && classifyTransient503(status, json);if (transient503) {await retryAfterTransient(transient503, { body: json });diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.tsindex 37af82d48..ff953b0fc 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.ts@@ -161,6 +161,7 @@ export function buildFirePdfJobOptions(args: {pageMarkers: boolean;tableFormat: string | undefined;trainingCapture: FirePdfTrainingCapture | undefined;+ dataRegion: string | undefined;}): Record<string, unknown> {return {// fire-pdf accepts only a positive integer here. When the router has no@@ -184,6 +185,9 @@ export function buildFirePdfJobOptions(args: {...(args.pageMarkers && { pageMarkers: true }),...(args.tableFormat !== undefined && { tableFormat: args.tableFormat }),...(args.trainingCapture && { training_capture: args.trainingCapture }),+ // An idempotency key on the PDF service, so a pinned lookup only adopts+ // a pinned job.+ ...(args.dataRegion && { data_region: args.dataRegion }),};}diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.tsindex 726c3e3e7..fca9490aa 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts@@ -6,11 +6,7 @@ import type { PDFProcessorResult } from "./types";import type { PDFMode } from "../../../../controllers/v2/types";import { safeMarkdownToHtml } from "./markdownToHtml";import { createPdfCacheKey } from "../../../../lib/gcs-pdf-cache";-import {- isRegionUnavailable,- isUsOnly,- US_DATA_REGION,-} from "../../../../lib/us-only";+import { isUsOnly, US_DATA_REGION } from "../../../../lib/us-only";import {maybeSaveResult,provenanceFromResponse,@@ -23,7 +19,10 @@ import {firePdfTableFormat,type FirePdfSourceKind,} from "./fire-pdf/request-metadata";-import { firePdfUsOnlyRefusal } from "./fire-pdf/data-region";+import {+ firePdfRegionRefused,+ firePdfUsOnlyRefusal,+} from "./fire-pdf/data-region";/*** Reconcile an existing page count with what fire-pdf reported.@@ -185,14 +184,14 @@ export async function scrapePDFWithFirePDF(// the echo is the only proof the fire-pdf build understood the// option — older builds ignore unknown request fields and omit it.page_markers: z.literal(true).optional(),- data_region: z.unknown().optional(),+ data_region_applied: z.unknown().optional(),}),mock: meta.mock,abort: meta.abort.asSignal(),}).catch(error => {if (usOnly &&- isRegionUnavailable(+ firePdfRegionRefused((error as { cause?: { response?: { body?: unknown } } })?.cause?.response?.body,)@@ -203,7 +202,7 @@ export async function scrapePDFWithFirePDF(});// A build that ignores the pin answers without the echo; its result may// have come from GPU capacity outside the US.- if (usOnly && resp.data_region !== US_DATA_REGION) {+ if (usOnly && resp.data_region_applied !== US_DATA_REGION) {throw firePdfUsOnlyRefusal(meta, "region_unconfirmed", sourceKind);}