Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): keep US-only PDF parsing, image OCR and PII redaction on US capacity (#5552)
GergΕ MΓ³ricz09b8fd8e8fa3
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.ymlindex 5d2edfb3e..76f087043 100644--- a/.github/workflows/test-server.yml+++ b/.github/workflows/test-server.yml@@ -357,6 +357,21 @@ jobs:--reporter=defaultworking-directory: apps/api+ - name: Run US-only PDF, image OCR and PII redaction regression tests+ if: matrix.engine == 'fetch' && matrix.proxy == 'no-proxy' && matrix.nuq == 'postgres'+ run: >-+ pnpm exec vitest run+ src/scraper/scrapeURL/engines/pdf/__tests__/firePDFUsOnly.test.ts+ src/scraper/scrapeURL/engines/pdf/__tests__/usOnlyPdfRouting.test.ts+ src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts+ src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts+ src/scraper/scrapeURL/engines/pdf/__tests__/firePDFBackgroundRefresh.test.ts+ src/scraper/scrapeURL/engines/pdf/__tests__/parseFirePdfRouting.test.ts+ src/lib/fire-privacy-client.test.ts+ src/scraper/scrapeURL/transformers/redactPII.test.ts+ --reporter=default+ working-directory: apps/api+- name: Run country check route regression testsif: matrix.engine == 'fetch' && matrix.proxy == 'no-proxy' && matrix.nuq == 'postgres'run: pnpm exec vitest run src/__tests__/routes/country-check.routes.test.ts --reporter=defaultdiff --git a/apps/api/src/lib/fire-privacy-client.test.ts b/apps/api/src/lib/fire-privacy-client.test.tsindex 44227a577..3bd929bee 100644--- a/apps/api/src/lib/fire-privacy-client.test.ts+++ b/apps/api/src/lib/fire-privacy-client.test.ts@@ -2,6 +2,7 @@ import http from "http";import { AddressInfo } from "net";import { config } from "../config";import { redactText } from "./fire-privacy-client";+import { UsOnlyUnavailableError } from "../scraper/scrapeURL/error";type Handler = (req: http.IncomingMessage,@@ -718,3 +719,97 @@ describe("redactText", () => {expect(out.redactedMarkdown).toBe(" - email [email protected]");});});++describe("redactText for US-only teams", () => {+ function capturing(+ reply: unknown,+ status = 200,+ ): { handler: Handler; bodies: Record<string, unknown>[] } {+ const bodies: Record<string, unknown>[] = [];+ return {+ bodies,+ handler: async (req, res) => {+ const chunks: Buffer[] = [];+ for await (const chunk of req) chunks.push(chunk as Buffer);+ bodies.push(JSON.parse(Buffer.concat(chunks).toString()));+ res.statusCode = status;+ res.setHeader("content-type", "application/json");+ res.end(JSON.stringify(reply));+ },+ };+ }++ it("sends data_region only when usOnly is set", async () => {+ const reply = {+ redacted_text: "Hi <PERSON>",+ spans: [],+ model_status: "ok",+ data_region: "us",+ };+ const capture = capturing(reply);+ handler = capture.handler;++ const unpinned = await redactText({ text: "Hi Alice" });+ const pinned = await redactText({ text: "Hi Alice", usOnly: true });++ expect(Object.hasOwn(capture.bodies[0], "data_region")).toBe(false);+ expect(capture.bodies[1]).toMatchObject({ data_region: "us" });+ expect(unpinned.status).toBe("ok");+ expect(pinned.status).toBe("ok");+ expect(pinned.redactedMarkdown).toBe("Hi <PERSON>");+ });++ it("throws UsOnlyUnavailableError when the redaction service has no US capacity", async () => {+ handler = capturing(+ { code: "region_unavailable", data_region: "us" },+ 503,+ ).handler;++ await expect(+ redactText({ text: "Hi Alice", usOnly: true }),+ ).rejects.toBeInstanceOf(UsOnlyUnavailableError);+ });++ it("throws UsOnlyUnavailableError for a refusal on any status or region field", async () => {+ handler = capturing({ error_code: "region_unavailable" }, 403).handler;++ await expect(+ redactText({ text: "Hi Alice", usOnly: true }),+ ).rejects.toThrow("no US capacity could take this request");+ });++ it("throws UsOnlyUnavailableError when a pinned reply lacks the region echo", async () => {+ handler = capturing({+ redacted_text: "Hi <PERSON>",+ spans: [],+ model_status: "ok",+ }).handler;++ const error = await redactText({ text: "Hi Alice", usOnly: true }).catch(+ e => e,+ );+ expect(error).toBeInstanceOf(UsOnlyUnavailableError);+ expect(error.message).toContain("did not confirm US-only processing");+ });++ it("keeps an unpinned 503 as failed/service_unavailable", async () => {+ handler = capturing(+ { code: "region_unavailable", data_region: "us" },+ 503,+ ).handler;++ const out = await redactText({ text: "Hi Alice" });+ expect(out.status).toBe("failed");+ expect(out.reason).toBe("service_unavailable");+ expect(out.redactedMarkdown).toBeNull();+ });++ it("keeps an ordinary pinned 503 as failed with no markdown", async () => {+ handler = capturing({ error: "overloaded" }, 503).handler;++ const out = await redactText({ text: "Hi Alice", usOnly: true });+ expect(out.status).toBe("failed");+ expect(out.reason).toBe("service_unavailable");+ expect(out.redactedMarkdown).toBeNull();+ });+});diff --git a/apps/api/src/lib/fire-privacy-client.ts b/apps/api/src/lib/fire-privacy-client.tsindex 8649a1697..a52545d10 100644--- a/apps/api/src/lib/fire-privacy-client.ts+++ b/apps/api/src/lib/fire-privacy-client.ts@@ -5,11 +5,14 @@ import {type RedactPIIEntity,} from "../controllers/v2/types";import { chunkMarkdown, type Chunk } from "./fire-privacy-chunker";+import { UsOnlyUnavailableError } from "../scraper/scrapeURL/error";+import { isRegionUnavailable, US_DATA_REGION } from "./us-only";type FirePrivacyResponse = {redacted_text?: unknown;spans?: unknown;model_status?: unknown;+ data_region?: unknown;};type RedactOptions = {@@ -22,6 +25,9 @@ type RedactOptions = {// wouldn't trigger this code path at all (transformer skips when// meta.options.redactPII is falsy).options?: RedactPIIOptions;+ // Pins the model call to US capacity; a refusal or a reply without the+ // region echo throws UsOnlyUnavailableError.+ usOnly?: boolean;};type RedactionSource = "model" | "heuristics" | "unknown";@@ -43,7 +49,9 @@ type RedactionReason =| "upstream_skipped"| "service_unavailable"| "timeout"- | "error";+ | "error"+ | "region_unavailable"+ | "region_unconfirmed";type RedactionResult = {status: RedactionStatus;@@ -234,6 +242,7 @@ async function redactOnce(url: string | undefined,timeoutMs: number,logger: Logger | undefined,+ usOnly: boolean,): Promise<ChunkResult> {const controller = new AbortController();let timedOut = false;@@ -252,6 +261,7 @@ async function redactOnce(mode: MODE_MAP[options.mode],operator: REPLACE_MAP[options.replaceStyle],language: DEFAULTS.language,+ ...(usOnly && { data_region: US_DATA_REGION }),}),signal: controller.signal,});@@ -267,11 +277,17 @@ async function redactOnce(});return { ok: false, reason };}+ // Read under the request deadline so a stalled body cannot hang the chunk.+ const refusal: unknown =+ usOnly && !response.ok ? await response.json().catch(() => null) : null;clearTimeout(timer);if (!response.ok) {- const reason: RedactionReason =- response.status === 503 ? "service_unavailable" : "error";+ const reason: RedactionReason = isRegionUnavailable(refusal)+ ? "region_unavailable"+ : response.status === 503+ ? "service_unavailable"+ : "error";logger?.warn("fire-privacy returned non-2xx", {reason,httpStatus: response.status,@@ -302,6 +318,16 @@ async function redactOnce(return { ok: false, reason: "error" };}+ // A build that ignores the pin answers without the echo; its model call+ // may have run outside the US.+ if (usOnly && body.data_region !== US_DATA_REGION) {+ logger?.warn("fire-privacy did not confirm US-only processing", {+ url,+ chunkStart: chunk.start,+ });+ return { ok: false, reason: "region_unconfirmed" };+ }+// Lift spans into source coordinates.const spans = rawSpans.map(s => ({...s,@@ -325,6 +351,7 @@ async function runChunks(url: string | undefined,timeoutMs: number,logger: Logger | undefined,+ usOnly: boolean,): Promise<ChunkResult[]> {const results = new Array<ChunkResult>(chunks.length);let next = 0;@@ -333,7 +360,14 @@ async function runChunks(while (true) {const i = next++;if (i >= chunks.length) return;- results[i] = await redactOnce(chunks[i], options, url, timeoutMs, logger);+ results[i] = await redactOnce(+ chunks[i],+ options,+ url,+ timeoutMs,+ logger,+ usOnly,+ );}};@@ -405,7 +439,29 @@ export async function redactText(}const chunks = chunkMarkdown(text);- const results = await runChunks(chunks, options, opts.url, timeoutMs, logger);+ const results = await runChunks(+ chunks,+ options,+ opts.url,+ timeoutMs,+ logger,+ opts.usOnly === true,+ );++ const regionFailure = results.find(+ r =>+ !r.ok &&+ (r.reason === "region_unavailable" || r.reason === "region_unconfirmed"),+ );+ if (regionFailure && !regionFailure.ok) {+ const detail =+ regionFailure.reason === "region_unconfirmed"+ ? "the redaction service did not confirm US-only processing"+ : "no US capacity could take this request";+ throw new UsOnlyUnavailableError(+ `PII redaction is not available for teams restricted to US-only processing right now: ${detail}. Try again later.`,+ );+ }// All-or-nothing: any chunk failure poisons the whole response. Partial// redaction is worse than no redaction β callers can't tell whichdiff --git a/apps/api/src/lib/us-only.ts b/apps/api/src/lib/us-only.tsindex 564070d83..10f924639 100644--- a/apps/api/src/lib/us-only.ts+++ b/apps/api/src/lib/us-only.ts@@ -4,3 +4,26 @@ import type { TeamFlags } from "../controllers/v2/types";export function isUsOnly(flags: TeamFlags | undefined): boolean {return flags?.usOnly === true;}++/** The `data_region` value that pins a US-only team's GPU work to US capacity. */+export const US_DATA_REGION = "us";++/**+ * Whether a GPU service reply refuses a pinned request because no in-region+ * capacity could take it. Accepts a parsed body or raw JSON text.+ */+export function isRegionUnavailable(body: unknown): boolean {+ let parsed = body;+ if (typeof body === "string") {+ try {+ parsed = JSON.parse(body);+ } catch {+ return false;+ }+ }+ if (typeof parsed !== "object" || parsed === null) return false;+ const fields = parsed as Record<string, unknown>;+ return ["code", "error_code", "error_class", "error"].some(+ key => fields[key] === "region_unavailable",+ );+}diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFBackgroundRefresh.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFBackgroundRefresh.test.tsindex 5f31e4fb7..6f5d92113 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFBackgroundRefresh.test.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFBackgroundRefresh.test.ts@@ -296,4 +296,28 @@ describe("FirePDF cache background refresh", () => {expect(await refreshCount("skipped_zdr")).toBe(1);expect(await refreshCount("skipped_image")).toBe(1);});++ it("serves a US-only team the stale entry without offering or submitting a refresh", async () => {+ const meta = makeMeta();+ meta.internalOptions.teamFlags = { usOnly: true };+ answerWith(staleAnswer(true));+ expect(await lookupCachedResult(meta, base64, lookupArgs)).toEqual(+ staleResult,+ );+ await settle();+ expect(lookupFetch.mock.calls[0][0].body).not.toHaveProperty(+ "background_refresh",+ );+ expect(+ await submitBackgroundRefresh(meta, base64, {+ mode: undefined,+ includePageMarkdown: false,+ includeBlocks: false,+ pageMarkers: false,+ sourceKind: "pdf",+ }),+ ).toBe("skipped_us_only");+ expect(jobsFetch).not.toHaveBeenCalled();+ expect(await refreshCount("skipped_us_only")).toBe(1);+ });});diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFUsOnly.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFUsOnly.test.tsnew file mode 100644index 000000000..f11aab842--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFUsOnly.test.ts@@ -0,0 +1,369 @@+// Stub the GCS cache so the async client never reaches real cloud storage.+vi.mock("../../../../../lib/gcs-pdf-cache", () => ({+ pdfCacheConfigured: vi.fn(() => true),+ createPdfCacheKey: (s: string) => `sha-${s.length}`,+ resolvePdfCacheKey: (input: string | { key: string }) =>+ typeof input === "string" ? `sha-${input.length}` : input.key,+ getPdfResultFromCache: vi.fn(async () => null),+ savePdfResultToCache: vi.fn(async () => null),+}));+vi.mock("../../../lib/fetch", () => ({+ robustFetch: vi.fn(),+}));++import { robustFetch } from "../../../lib/fetch";+import { config } from "../../../../../config";+import { savePdfResultToCache } from "../../../../../lib/gcs-pdf-cache";+import { UsOnlyUnavailableError } from "../../../error";+import { scrapePDFWithFirePDF } from "../firePDF";+import { scrapePDFWithFirePDFAsync } from "../fire-pdf/async";+import { lookupAdoptableFirePdfJob } from "../fire-pdf/lookup";+import {+ makeFetchFromSequence,+ makeMeta as makeAsyncMeta,+ noopSleep,+} from "./firePDFAsyncFixtures";++const mockedRobustFetch = vi.mocked(robustFetch);++const usOnlyFlags = { usOnly: true };++function makeSyncMeta(teamFlags?: Record<string, unknown>) {+ const logger: any = {+ info: vi.fn(),+ warn: vi.fn(),+ error: vi.fn(),+ debug: vi.fn(),+ };+ return {+ id: "us-only-sync-test",+ url: "https://example.com/file.pdf",+ rewrittenUrl: undefined,+ options: {},+ logger,+ mock: null,+ abort: {+ throwIfAborted: vi.fn(),+ asSignal: vi.fn(() => new AbortController().signal),+ scrapeTimeout: vi.fn(() => 60_000),+ },+ internalOptions: {+ zeroDataRetention: false,+ teamId: "team-test",+ ...(teamFlags && { teamFlags }),+ },+ } as any;+}++function makeUsOnlyAsyncMeta(teamFlags?: Record<string, unknown>) {+ return makeAsyncMeta({+ internalOptions: {+ zeroDataRetention: false,+ teamId: "team-x",+ teamConcurrency: 12,+ crawlId: undefined,+ ...(teamFlags && { teamFlags }),+ },+ });+}++function failureStatus(status: number, body: unknown) {+ return new Error("Request sent failure status", {+ cause: { response: { status, body: JSON.stringify(body) } },+ });+}++const regionRefusal = {+ code: "region_unavailable",+ data_region: "us",+ error: "no in-region capacity",+};++const ORIGINAL_BASE_URL = config.FIRE_PDF_BASE_URL;++beforeAll(() => {+ config.FIRE_PDF_BASE_URL = "http://fire-pdf.test";+});++afterAll(() => {+ config.FIRE_PDF_BASE_URL = ORIGINAL_BASE_URL;+});++beforeEach(() => {+ vi.clearAllMocks();+});++describe("FirePDF sync parse for US-only teams", () => {+ it("sends data_region only for a US-only team", async () => {+ mockedRobustFetch.mockResolvedValue({+ markdown: "Document",+ failed_pages: null,+ pages_processed: 1,+ data_region: "us",+ } as any);++ await scrapePDFWithFirePDF(makeSyncMeta(), "BASE64", 1);+ await scrapePDFWithFirePDF(makeSyncMeta(usOnlyFlags), "BASE64", 1);++ const unpinned = mockedRobustFetch.mock.calls[0][0].body;+ const pinned = mockedRobustFetch.mock.calls[1][0].body;+ expect(Object.hasOwn(unpinned, "data_region")).toBe(false);+ expect(pinned.data_region).toBe("us");+ });++ it("returns a US-only result that carries the region echo", async () => {+ mockedRobustFetch.mockResolvedValue({+ markdown: "Document",+ failed_pages: null,+ pages_processed: 1,+ data_region: "us",+ } as any);++ const result = await scrapePDFWithFirePDF(+ makeSyncMeta(usOnlyFlags),+ "BASE64",+ 1,+ );++ expect(result.markdown).toBe("Document");+ });++ it("refuses a US-only parse the PDF service could not place on US capacity", async () => {+ mockedRobustFetch.mockRejectedValue(failureStatus(503, regionRefusal));++ const error = await scrapePDFWithFirePDF(+ makeSyncMeta(usOnlyFlags),+ "BASE64",+ 1,+ ).catch(e => e);++ expect(error).toBeInstanceOf(UsOnlyUnavailableError);+ expect(error.code).toBe("US_ONLY_UNAVAILABLE");+ expect(error.message).toContain("PDF parsing is not available");+ });++ it.each([+ ["missing", undefined],+ ["null", null],+ ])(+ "refuses a US-only result whose region echo is %s",+ async (_name, dataRegion) => {+ mockedRobustFetch.mockResolvedValue({+ markdown: "Document",+ failed_pages: null,+ pages_processed: 1,+ data_region: dataRegion,+ } as any);++ const error = await scrapePDFWithFirePDF(+ makeSyncMeta(usOnlyFlags),+ "BASE64",+ 1,+ ).catch(e => e);++ expect(error).toBeInstanceOf(UsOnlyUnavailableError);+ expect(error.message).toContain("did not confirm US-only processing");+ },+ );++ it("names image OCR when a US-only image is refused", async () => {+ mockedRobustFetch.mockRejectedValue(failureStatus(503, regionRefusal));++ const error = await scrapePDFWithFirePDF(+ makeSyncMeta(usOnlyFlags),+ "BASE64",+ 1,+ 1,+ "ocr",+ false,+ false,+ false,+ "image",+ ).catch(e => e);++ expect(error).toBeInstanceOf(UsOnlyUnavailableError);+ expect(error.message).toContain("Image OCR is not available");+ });++ it("keeps an unpinned failure and an unpinned result without the echo unchanged", async () => {+ const failure = failureStatus(503, regionRefusal);+ mockedRobustFetch.mockRejectedValueOnce(failure);+ expect(+ await scrapePDFWithFirePDF(makeSyncMeta(), "BASE64", 1).catch(e => e),+ ).toBe(failure);++ mockedRobustFetch.mockResolvedValueOnce({+ markdown: "Document",+ failed_pages: null,+ pages_processed: 1,+ } as any);+ const result = await scrapePDFWithFirePDF(makeSyncMeta(), "BASE64", 1);+ expect(result.markdown).toBe("Document");+ });+});++const submitAccepted = {+ matchUrl: /\/jobs$/,+ matchMethod: "POST" as const,+ response: {+ status: 202,+ body: { scrape_id: "scrape-id-test", status: "queued", lane: "fast" },+ },+};+const pollDone = {+ matchUrl: /\/jobs\/scrape-id-test$/,+ matchMethod: "GET" as const,+ response: {+ status: 200,+ body: { scrape_id: "scrape-id-test", status: "done", pages_processed: 3 },+ },+};+function resultWith(extra: Record<string, unknown>) {+ return {+ matchUrl: /\/jobs\/scrape-id-test\/result$/,+ matchMethod: "GET" as const,+ response: {+ status: 200,+ body: {+ schema_version: 1,+ markdown: "# Async",+ pages_processed: 3,+ failed_pages: null,+ partial_pages: null,+ ...extra,+ },+ },+ };+}++function runAsync(meta: any, fetchImpl: any) {+ return scrapePDFWithFirePDFAsync(+ meta,+ "BASE64",+ undefined,+ undefined,+ undefined,+ { fetchImpl, fallbackImpl: vi.fn(), sleepImpl: noopSleep },+ );+}++describe("FirePDF async jobs for US-only teams", () => {+ it("sends data_region on the submit only for a US-only team", async () => {+ const unpinned = makeFetchFromSequence([+ submitAccepted,+ pollDone,+ resultWith({}),+ ]);+ await runAsync(makeUsOnlyAsyncMeta(), unpinned.fetchImpl);++ const pinned = makeFetchFromSequence([+ submitAccepted,+ pollDone,+ resultWith({ data_region: "us" }),+ ]);+ const result = await runAsync(+ makeUsOnlyAsyncMeta(usOnlyFlags),+ pinned.fetchImpl,+ );++ expect(unpinned.calls[0].body).not.toHaveProperty("data_region");+ expect(pinned.calls[0].body).toMatchObject({ data_region: "us" });+ expect(result.markdown).toBe("# Async");+ });++ it("refuses a US-only job that failed for lack of US capacity, without cancelling it", async () => {+ const { fetchImpl, calls } = makeFetchFromSequence([+ submitAccepted,+ {+ matchUrl: /\/jobs\/scrape-id-test$/,+ matchMethod: "GET",+ response: {+ status: 502,+ body: {+ scrape_id: "scrape-id-test",+ status: "failed",+ error_class: "region_unavailable",+ error_message: "no in-region capacity",+ error_code: "region_unavailable",+ },+ },+ },+ ]);++ const error = await runAsync(+ makeUsOnlyAsyncMeta(usOnlyFlags),+ fetchImpl,+ ).catch(e => e);++ expect(error).toBeInstanceOf(UsOnlyUnavailableError);+ expect(calls.map(call => call.method)).toEqual(["POST", "GET"]);+ });++ it("refuses a US-only submit the PDF service turned away for lack of US capacity, without retrying", async () => {+ const { fetchImpl, calls } = makeFetchFromSequence([+ {+ matchUrl: /\/jobs$/,+ matchMethod: "POST",+ response: { status: 503, body: regionRefusal },+ },+ ]);++ const error = await runAsync(+ makeUsOnlyAsyncMeta(usOnlyFlags),+ fetchImpl,+ ).catch(e => e);++ expect(error).toBeInstanceOf(UsOnlyUnavailableError);+ expect(error.message).toContain("no US capacity could take this document");+ expect(calls).toHaveLength(1);+ });++ it("refuses a US-only result that lacks the region echo and caches nothing", async () => {+ const { fetchImpl } = makeFetchFromSequence([+ submitAccepted,+ pollDone,+ resultWith({}),+ ]);++ const error = await runAsync(+ makeUsOnlyAsyncMeta(usOnlyFlags),+ fetchImpl,+ ).catch(e => e);++ expect(error).toBeInstanceOf(UsOnlyUnavailableError);+ expect(savePdfResultToCache).not.toHaveBeenCalled();+ });++ it("sends data_region on the content lookup only for a US-only team", async () => {+ const lookupHit = {+ matchUrl: /\/jobs\/lookup$/,+ matchMethod: "POST" as const,+ response: {+ status: 200,+ body: { scrape_id: "found-1", status: "running" },+ },+ };+ const unpinned = makeFetchFromSequence([lookupHit]);+ const pinned = makeFetchFromSequence([lookupHit]);+ const sha = "ef".repeat(32);++ await lookupAdoptableFirePdfJob(+ makeUsOnlyAsyncMeta(),+ sha,+ { mode: "auto" },+ unpinned.fetchImpl,+ );+ await lookupAdoptableFirePdfJob(+ makeUsOnlyAsyncMeta(usOnlyFlags),+ sha,+ { mode: "auto" },+ pinned.fetchImpl,+ );++ expect(unpinned.calls[0].body).not.toHaveProperty("data_region");+ expect(pinned.calls[0].body).toMatchObject({+ data_region: "us",+ options: { mode: "auto" },+ });+ });+});diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/usOnlyPdfRouting.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/usOnlyPdfRouting.test.tsnew file mode 100644index 000000000..8d3d4c54f--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/usOnlyPdfRouting.test.ts@@ -0,0 +1,217 @@+import { mkdtemp, rm, writeFile } from "node:fs/promises";+import { tmpdir } from "node:os";+import path from "node:path";+import { processPdf } from "@mendable/firecrawl-rs";+import { config } from "../../../../../config";+import { UsOnlyUnavailableError } from "../../../error";+import { scrapePDFWithFirePDF } from "../firePDF";+import { scrapePDFWithParsePDF } from "../pdfParse";+import { scrapePDFWithRunPodMU } from "../runpodMU";+import { mineruDiverted } from "../fire-pdf/by-reference";+import { scrapePDF } from "../index";++vi.mock("@mendable/firecrawl-rs", () => ({+ processPdf: vi.fn(),+ detectPdf: vi.fn(),+}));++vi.mock("../firePDF", async importOriginal => ({+ ...(await importOriginal<typeof import("../firePDF")>()),+ scrapePDFWithFirePDF: vi.fn(),+}));++vi.mock("../pdfParse", () => ({+ scrapePDFWithParsePDF: vi.fn(),+}));++vi.mock("../runpodMU", () => ({+ scrapePDFWithRunPodMU: vi.fn(),+}));++vi.mock("../markdownToHtml", () => ({+ safeMarkdownToHtml: vi.fn(async (markdown: string) => `<p>${markdown}</p>`),+}));++const mockedProcessPdf = vi.mocked(processPdf);+const mockedFirePdf = vi.mocked(scrapePDFWithFirePDF);+const mockedPdfParse = vi.mocked(scrapePDFWithParsePDF);+const mockedRunPodMU = vi.mocked(scrapePDFWithRunPodMU);++const configKeys = [+ "PDF_RUST_EXTRACT_ENABLE",+ "FIRE_PDF_ENABLE",+ "FIRE_PDF_BASE_URL",+ "FIRE_PDF_PERCENT",+ "FIRE_PDF_ASYNC_PERCENT",+ "MINERU_PERCENT",+ "RUNPOD_MU_API_KEY",+ "RUNPOD_MU_POD_ID",+ "PARSE_PDF_ALWAYS_FIRE_PDF",+] as const;+const originalConfig = Object.fromEntries(+ configKeys.map(key => [key, config[key]]),+);++function setConfig(+ overrides: Partial<Record<(typeof configKeys)[number], unknown>>,+) {+ Object.assign(config as any, overrides);+}++const tempDirs: string[] = [];++async function makeMeta(usOnly: boolean) {+ const dir = await mkdtemp(path.join(tmpdir(), "us-only-pdf-"));+ tempDirs.push(dir);+ const filePath = path.join(dir, "doc.pdf");+ await writeFile(filePath, "%PDF-1.7\n%test\n");+ const logger: any = {+ info: vi.fn(),+ warn: vi.fn(),+ error: vi.fn(),+ debug: vi.fn(),+ };+ logger.child = vi.fn(() => logger);+ return {+ id: "01a00000-0000-7000-8000-000000000001",+ url: "https://example.com/doc.pdf",+ options: { parsers: ["pdf"] },+ logger,+ featureFlags: new Set(["pdf"]),+ pdfPrefetch: {+ filePath,+ url: "https://example.com/doc.pdf",+ status: 200,+ proxyUsed: "basic",+ },+ abort: {+ throwIfAborted: vi.fn(),+ asSignal: vi.fn(() => new AbortController().signal),+ scrapeTimeout: vi.fn(() => 60_000),+ },+ internalOptions: {+ teamId: "team-test",+ ...(usOnly && { teamFlags: { usOnly: true } }),+ },+ } as any;+}++function nativeResult(pdfType: string, markdown: string) {+ return {+ pdfType,+ confidence: 0.99,+ isComplex: true,+ pageCount: 1,+ markdown,+ title: null,+ pagesNeedingOcr: [],+ logs: [],+ } as any;+}++describe("PDF routing for US-only teams", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ setConfig({+ PDF_RUST_EXTRACT_ENABLE: true,+ FIRE_PDF_ENABLE: true,+ FIRE_PDF_BASE_URL: "http://fire-pdf.test",+ FIRE_PDF_PERCENT: 100,+ FIRE_PDF_ASYNC_PERCENT: 0,+ MINERU_PERCENT: 0,+ RUNPOD_MU_API_KEY: "test-key",+ RUNPOD_MU_POD_ID: "test-endpoint",+ PARSE_PDF_ALWAYS_FIRE_PDF: false,+ });+ mockedProcessPdf.mockResolvedValue(nativeResult("TextBased", "text layer"));+ mockedFirePdf.mockResolvedValue({+ markdown: "fire-pdf",+ html: "<p>fire-pdf</p>",+ pagesProcessed: 1,+ });+ mockedPdfParse.mockResolvedValue({+ markdown: "pdf-parse",+ html: "<p>pdf-parse</p>",+ });+ mockedRunPodMU.mockResolvedValue({+ markdown: "serverless",+ html: "<p>serverless</p>",+ });+ });++ afterEach(async () => {+ await Promise.all(+ tempDirs.splice(0).map(dir => rm(dir, { recursive: true, force: true })),+ );+ });++ afterAll(() => {+ Object.assign(config as any, originalConfig);+ });++ it("never diverts a US-only team to the alternate PDF engine", async () => {+ setConfig({ MINERU_PERCENT: 100 });++ expect(mineruDiverted(await makeMeta(false))).toBe(true);+ expect(mineruDiverted(await makeMeta(true))).toBe(false);++ const result = await scrapePDF(await makeMeta(true));++ expect(result.markdown).toBe("fire-pdf");+ expect(mockedFirePdf).toHaveBeenCalledTimes(1);+ expect(mockedRunPodMU).not.toHaveBeenCalled();+ });++ it("sends a US-only PDF to the PDF service regardless of FIRE_PDF_PERCENT", async () => {+ setConfig({ FIRE_PDF_PERCENT: 0, RUNPOD_MU_API_KEY: undefined });++ expect((await scrapePDF(await makeMeta(false))).markdown).toBe("pdf-parse");+ expect(mockedFirePdf).not.toHaveBeenCalled();++ expect((await scrapePDF(await makeMeta(true))).markdown).toBe("fire-pdf");+ expect(mockedFirePdf).toHaveBeenCalledTimes(1);+ });++ it("fails a US-only scrape the PDF service refuses, with no fallback", async () => {+ mockedFirePdf.mockRejectedValueOnce(+ new UsOnlyUnavailableError("no US capacity"),+ );++ const error = await scrapePDF(await makeMeta(true)).catch(e => e);++ expect(error).toBeInstanceOf(UsOnlyUnavailableError);+ expect(mockedRunPodMU).not.toHaveBeenCalled();+ expect(mockedPdfParse).not.toHaveBeenCalled();+ });++ it("skips the serverless fallback but keeps the text layer for a US-only team", async () => {+ mockedFirePdf.mockRejectedValue(new Error("fire-pdf 500"));++ expect((await scrapePDF(await makeMeta(false))).markdown).toBe(+ "serverless",+ );+ expect(mockedRunPodMU).toHaveBeenCalledTimes(1);++ mockedRunPodMU.mockClear();+ expect((await scrapePDF(await makeMeta(true))).markdown).toBe("pdf-parse");+ expect(mockedRunPodMU).not.toHaveBeenCalled();+ });++ it.each([+ ["a scanned PDF", "Scanned", "page header"],+ ["a PDF with no text layer", "Mixed", " "],+ ])(+ "refuses %s for a US-only team instead of returning it without OCR",+ async (_name, pdfType, text) => {+ mockedProcessPdf.mockResolvedValue(nativeResult(pdfType, ""));+ mockedFirePdf.mockRejectedValue(new Error("fire-pdf 500"));+ mockedPdfParse.mockResolvedValue({ markdown: text, html: text });++ const error = await scrapePDF(await makeMeta(true)).catch(e => e);++ expect(error).toBeInstanceOf(UsOnlyUnavailableError);+ expect(error.message).toContain("This PDF needs OCR");+ expect(mockedRunPodMU).not.toHaveBeenCalled();+ },+ );+});diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.tsindex e4b1d7b3c..57f9667ac 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts@@ -9,6 +9,8 @@ import { scrapePDFWithFirePDF } from "../firePDF";import { cancelJob } from "./cancel";import { maybeSaveResult, provenanceFromResponse, tryGetCached } from "./cache";import { resolvePdfCacheKey } from "../../../../../lib/gcs-pdf-cache";+import { isUsOnly, US_DATA_REGION } from "../../../../../lib/us-only";+import { firePdfUsOnlyRefusal } from "./data-region";import {firePdfAsyncAbandonedTotal,firePdfAsyncTotalDurationSeconds,@@ -327,7 +329,8 @@ export async function scrapePDFWithFirePDFAsync(error instanceof FirePdfAsyncFailure &&(error.reason === "terminal_failed" ||error.reason === "terminal_expired" ||- error.reason === "terminal_cancelled");+ error.reason === "terminal_cancelled" ||+ error.reason === "region_unavailable");if ((jobAlreadyTerminal || terminalReached) && meta.largePdfProcessing) {// The job is dead β a "processing continues" message would lie.meta.largePdfProcessing.current = undefined;@@ -371,6 +374,12 @@ export async function scrapePDFWithFirePDFAsync() {await cancelJob({ baseUrl, scrapeId: meta.id, meta, fetchImpl });}+ if (+ error instanceof FirePdfAsyncFailure &&+ error.reason === "region_unavailable"+ ) {+ throw firePdfUsOnlyRefusal(meta, "region_unavailable");+ }throw submitMayHaveBeenAccepted ? error.originalError : error;}@@ -381,6 +390,15 @@ export async function scrapePDFWithFirePDFAsync(meta.largePdfProcessing.current = undefined;}+ // Adopted jobs included: a job that ran before the pin existed carries no+ // echo, and its result may have come from GPU capacity outside the US.+ if (+ isUsOnly(meta.internalOptions.teamFlags ?? undefined) &&+ fetched.data_region !== US_DATA_REGION+ ) {+ throw firePdfUsOnlyRefusal(meta, "region_unconfirmed");+ }+// ββ Assemble + cache save βββββββββββββββββββββββββββββββββββββββββββββconst pages =fetched.pages_processed ?? polled.poll.pages_processed ?? pagesProcessed;diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/background-refresh.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/background-refresh.tsindex f8333b049..88b0826e2 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/background-refresh.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/background-refresh.ts@@ -3,6 +3,7 @@ import { fetch as undiciFetch } from "undici";import type { Meta } from "../../..";import type { PDFMode } from "../../../../../controllers/v2/types";import { config } from "../../../../../config";+import { isUsOnly } from "../../../../../lib/us-only";import type { PdfCacheKeyInput } from "../../../../../lib/gcs-pdf-cache";import {type BackgroundRefreshResult,@@ -107,6 +108,10 @@ async function submit(fetchImpl: typeof undiciFetch,): Promise<BackgroundRefreshResult> {if (meta.internalOptions.zeroDataRetention === true) return "skipped_zdr";+ // A refresh carries no team and nobody checks its region echo.+ if (isUsOnly(meta.internalOptions.teamFlags ?? undefined)) {+ return "skipped_us_only";+ }// A by-reference document's bytes are in fire-pdf's input bucket, not here.if (typeof input !== "string") return "skipped_by_reference";// Raster images are only ever parsed through the sync endpoint.diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference.tsindex 22b42e37c..a2aacae38 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference.ts@@ -12,6 +12,7 @@ import {} from "../../../../../controllers/v2/types";import { deterministicPercentage } from "./routing";import { storage } from "../../../../../lib/gcs-jobs";+import { isUsOnly } from "../../../../../lib/us-only";import { FIRE_PDF_BY_REFERENCE_MAX_FILE_SIZE } from "../types";import { parseAlwaysFirePdf } from "./request-metadata";@@ -201,11 +202,12 @@ export function largePdfLimitBytes(meta: Meta): number {* fire-engine handoff grant, the download admission, and the PDF engine's* routing β sees the SAME verdict; a per-call random draw would let them* drift. Keyed apart from the async-cohort hash. Parse requests that must- * reach FirePDF (parseAlwaysFirePdf) are never diverted. */+ * reach FirePDF (parseAlwaysFirePdf) and US-only teams are never diverted. */export function mineruDiverted(meta: Meta): boolean {return (config.MINERU_PERCENT > 0 &&!parseAlwaysFirePdf(meta) &&+ !isUsOnly(meta.internalOptions.teamFlags ?? undefined) &&deterministicPercentage(`mineru:${meta.id}`) < config.MINERU_PERCENT);}diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache-service.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache-service.tsindex b8579a190..2ab92c08a 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache-service.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache-service.ts@@ -8,6 +8,7 @@ import {} from "../../../../../lib/gcs-pdf-cache";import type { PDFMode } from "../../../../../controllers/v2/types";import { robustFetch } from "../../../lib/fetch";+import { isUsOnly } from "../../../../../lib/us-only";import {recordSpanException,type Span,@@ -193,12 +194,14 @@ async function lookupWithinSpan(meta.abort.throwIfAborted();// Background refresh is only offered when this client could act on a// grant: it needs the bytes (inline, not by reference) of a PDF (images- // are only parsed through the sync endpoint), outside zero data retention.+ // are only parsed through the sync endpoint), outside zero data retention+ // and outside US-only teams.const canRefreshInBackground =config.FIRE_PDF_CACHE_BACKGROUND_REFRESH === true &&typeof input === "string" &&sourceKind === "pdf" &&- meta.internalOptions.zeroDataRetention !== true;+ meta.internalOptions.zeroDataRetention !== true &&+ !isUsOnly(meta.internalOptions.teamFlags ?? undefined);const keys = await cacheLookupKeys(input);if (sourceKind === "pdf" &&diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/data-region.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/data-region.tsnew file mode 100644index 000000000..14123aef9--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/data-region.ts@@ -0,0 +1,24 @@+import type { Meta } from "../../..";+import { UsOnlyUnavailableError } from "../../../error";+import type { FirePdfSourceKind } from "./request-metadata";++/** The error a US-only team gets when the PDF service did not run, or did not confirm running, its document on US capacity. */+export function firePdfUsOnlyRefusal(+ meta: Meta,+ reason: "region_unavailable" | "region_unconfirmed",+ sourceKind: FirePdfSourceKind = "pdf",+): UsOnlyUnavailableError {+ meta.logger.warn("FirePDF could not keep a US-only request on US capacity", {+ scrapeId: meta.id,+ reason,+ sourceKind,+ });+ const feature = sourceKind === "image" ? "Image OCR" : "PDF parsing";+ const detail =+ reason === "region_unconfirmed"+ ? "the PDF service did not confirm US-only processing"+ : "no US capacity could take this document";+ return new UsOnlyUnavailableError(+ `${feature} is not available for teams restricted to US-only processing right now: ${detail}. Try again later.`,+ );+}diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/lookup.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/lookup.tsindex 473edec00..9f215f9df 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/lookup.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/lookup.ts@@ -8,6 +8,7 @@ import type { Meta } from "../../..";import { config } from "../../../../../config";import { fetch as undiciFetch } from "undici";import { firePdfHeaders } from "./utils";+import { isUsOnly, US_DATA_REGION } from "../../../../../lib/us-only";/** Handle for adopting an existing fire-pdf job instead of submitting a* new one. Produced by {@link lookupAdoptableFirePdfJob}: the job was@@ -71,6 +72,9 @@ export async function lookupAdoptableFirePdfJob(team_id: meta.internalOptions.teamId,}),options,+ ...(isUsOnly(meta.internalOptions.teamFlags ?? undefined) && {+ data_region: US_DATA_REGION,+ }),}),signal,});diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.tsindex 3befd414b..bb39bed16 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.ts@@ -14,7 +14,7 @@ export const firePdfCacheRefusedWritesTotal = new Counter({export const firePdfCacheBackgroundRefreshTotal = new Counter({name: "firecrawl_fire_pdf_cache_background_refresh_total",- help: "Background refreshes the fire-pdf cache service granted on a stale answer, by result: submitted, submit_failed, skipped_by_reference, skipped_zdr, skipped_image",+ help: "Background refreshes the fire-pdf cache service granted on a stale answer, by result: submitted, submit_failed, skipped_by_reference, skipped_zdr, skipped_image, skipped_us_only",labelNames: ["result"],});@@ -23,7 +23,8 @@ export type BackgroundRefreshResult =| "submit_failed"| "skipped_by_reference"| "skipped_zdr"- | "skipped_image";+ | "skipped_image"+ | "skipped_us_only";export type CacheRefusedReason =| "failed_pages"@@ -122,4 +123,5 @@ export type FallbackReason =| "terminal_expired"| "terminal_cancelled"| "polling_timeout"- | "result_503";+ | "result_503"+ | "region_unavailable";diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/poll.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/poll.tsindex 991ca2290..6d440eff2 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/poll.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/poll.ts@@ -1,6 +1,7 @@import type { Meta } from "../../..";import { fetch as undiciFetch } from "undici";import { AbortManagerThrownError } from "../../../lib/abortManager";+import { isRegionUnavailable, isUsOnly } from "../../../../../lib/us-only";import {firePdfAsyncCompletedTotal,firePdfAsyncLongPollTotal,@@ -171,6 +172,16 @@ export async function pollUntilTerminal(deps: PollDeps): Promise<PollOk> {const pollStatus = pollResp.status;const pollBody = await pollResp.json().catch(() => ({}));+ if (+ isUsOnly(meta.internalOptions.teamFlags ?? undefined) &&+ isRegionUnavailable(pollBody)+ ) {+ if (waitMs > 0) firePdfAsyncLongPollTotal.labels("terminal").inc();+ firePdfAsyncPollCount.observe(pollCount);+ firePdfAsyncCompletedTotal.labels("failed").inc();+ failAsync(meta, "region_unavailable", { pollCount, body: pollBody });+ }+if (pollStatus === 401) {firePdfAsyncPollCount.observe(pollCount);failAsync(meta, "http_401", { pollCount });diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.tsindex 66ac5ee08..fdf8d4820 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts@@ -301,6 +301,7 @@ export const resultResponseSchema = z.object({// Raw on purpose: parsed separately by parseProvenance so a stamp this// build does not understand never fails the scrape.provenance: z.unknown().optional(),+ data_region: z.unknown().optional(),});export type PollResponse = z.infer<typeof pollResponseSchema>;diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.tsindex 448d55973..02459d586 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts@@ -2,6 +2,11 @@ import type { Meta } from "../../..";import type { PDFMode } from "../../../../../controllers/v2/types";import { fetch as undiciFetch } from "undici";import { AbortManagerThrownError } from "../../../lib/abortManager";+import {+ isRegionUnavailable,+ isUsOnly,+ US_DATA_REGION,+} from "../../../../../lib/us-only";import {buildFirePdfJobTrainingCapture,buildFirePdfRequestMetadata,@@ -118,6 +123,7 @@ export async function submitJob(args: SubmitArgs): Promise<SubmitOutcome> {fetchImpl,} = args;const scrapeId = meta.id;+ const usOnly = isUsOnly(meta.internalOptions.teamFlags ?? undefined);if (input.kind === "byReference" &&@@ -164,6 +170,7 @@ export async function submitJob(args: SubmitArgs): Promise<SubmitOutcome> {tableFormat: firePdfTableFormat(meta),trainingCapture: buildFirePdfJobTrainingCapture(meta),}),+ ...(usOnly && { data_region: US_DATA_REGION }),};// The retry does reach a different instance: Fastify's shutdown 503@@ -216,6 +223,9 @@ export async function submitJob(args: SubmitArgs): Promise<SubmitOutcome> {}failPossiblyAccepted(meta, "network_error", { error: String(error) });}+ if (usOnly && isRegionUnavailable(json)) {+ failAsync(meta, "region_unavailable", { status });+ }const transient503 = attempt === 1 && classifyTransient503(status, json);if (transient503) {await retryAfterTransient(transient503, { body: json });diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.tsindex 19839f72e..726c3e3e7 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts@@ -6,6 +6,11 @@ import type { PDFProcessorResult } from "./types";import type { PDFMode } from "../../../../controllers/v2/types";import { safeMarkdownToHtml } from "./markdownToHtml";import { createPdfCacheKey } from "../../../../lib/gcs-pdf-cache";+import {+ isRegionUnavailable,+ isUsOnly,+ US_DATA_REGION,+} from "../../../../lib/us-only";import {maybeSaveResult,provenanceFromResponse,@@ -18,6 +23,7 @@ import {firePdfTableFormat,type FirePdfSourceKind,} from "./fire-pdf/request-metadata";+import { firePdfUsOnlyRefusal } from "./fire-pdf/data-region";/*** Reconcile an existing page count with what fire-pdf reported.@@ -107,6 +113,7 @@ export async function scrapePDFWithFirePDF(});const zdr = meta.internalOptions.zeroDataRetention === true;+ const usOnly = isUsOnly(meta.internalOptions.teamFlags ?? undefined);const pdfSha256 = createPdfCacheKey(base64Content);const tableFormat = firePdfTableFormat(meta);@@ -157,6 +164,7 @@ export async function scrapePDFWithFirePDF(zdr,training_capture: buildFirePdfTrainingCapture(meta),...deadlineFields,+ ...(usOnly && { data_region: US_DATA_REGION }),},logger,schema: z.object({@@ -177,10 +185,27 @@ export async function scrapePDFWithFirePDF(// the echo is the only proof the fire-pdf build understood the// option β older builds ignore unknown request fields and omit it.page_markers: z.literal(true).optional(),+ data_region: z.unknown().optional(),}),mock: meta.mock,abort: meta.abort.asSignal(),+ }).catch(error => {+ if (+ usOnly &&+ isRegionUnavailable(+ (error as { cause?: { response?: { body?: unknown } } })?.cause+ ?.response?.body,+ )+ ) {+ throw firePdfUsOnlyRefusal(meta, "region_unavailable", sourceKind);+ }+ throw error;});+ // A build that ignores the pin answers without the echo; its result may+ // have come from GPU capacity outside the US.+ if (usOnly && resp.data_region !== US_DATA_REGION) {+ throw firePdfUsOnlyRefusal(meta, "region_unconfirmed", sourceKind);+ }const durationMs = Date.now() - startedAt;if (includePageMarkdown && resp.pages === undefined) {diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex 0145efc85..a208996f9 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@ -16,6 +16,7 @@ import {RemoveFeatureError,EngineUnsuccessfulError,UnsupportedFileError,+ UsOnlyUnavailableError,} from "../../error";import { open, readFile, stat, unlink } from "node:fs/promises";import type { Response } from "undici";@@ -45,6 +46,7 @@ import {extractAndEmitNativeLogs,} from "../../../../lib/native-logging";import { withSpan, setSpanAttributes } from "../../../../lib/otel-tracer";+import { isUsOnly } from "../../../../lib/us-only";import { scrapePDFWithRunPodMU } from "./runpodMU";import { useFireEngine } from "../fire-engine/available";import { reconcilePageCountWithFirePdf, scrapePDFWithFirePDF } from "./firePDF";@@ -243,6 +245,11 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {// Unlike forceFirePDF this keeps the MinerU / pdf-parse fallbacks.const alwaysFirePdf = parseAlwaysFirePdf(meta);+ // US-only teams get GPU parsing only from the PDF service, pinned to US+ // capacity: no diversion to another engine, no serverless fallback, and no+ // sampling out of the PDF service.+ const usOnly = isUsOnly(meta.internalOptions.teamFlags ?? undefined);+// The MinerU diversion is deterministic on the scrape id (see// mineruDiverted) so this routing verdict is BY CONSTRUCTION the same// one inside byReferenceReachableForRequest() β and the same one@@ -324,6 +331,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {let shadowIsComplex: boolean | undefined;let shadowIneligibleReason: string | null | undefined;let shadowPagesNeedingOcr: number[] | undefined;+ let detectedPdfType: string | undefined;const rustEnabled = !!config.PDF_RUST_EXTRACT_ENABLE;const logger = meta.logger.child({ method: "scrapePDF/processPdf" });@@ -376,6 +384,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {});totalPageCount = detection.pageCount;+ detectedPdfType = detection.pdfType;effectivePageCount = maxPages? Math.min(detection.pageCount, maxPages): detection.pageCount;@@ -423,6 +432,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {});totalPageCount = pdfResult.pageCount;+ detectedPdfType = pdfResult.pdfType;effectivePageCount = maxPages? Math.min(pdfResult.pageCount, maxPages): pdfResult.pageCount;@@ -624,6 +634,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {fileSizeBytes <= FIRE_PDF_INLINE_HARD_MAX_FILE_SIZE));const runpodMuUsable =!forceFirePDF &&+ !usOnly &&fileSizeBytes < MAX_FILE_SIZE &&!!config.RUNPOD_MU_API_KEY &&!!config.RUNPOD_MU_POD_ID;@@ -670,7 +681,9 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {config.FIRE_PDF_ENABLE &&config.FIRE_PDF_BASE_URL &&fileSizeBytes < FIRE_PDF_MAX_FILE_SIZE &&- (alwaysFirePdf || Math.random() * 100 < config.FIRE_PDF_PERCENT)));+ (alwaysFirePdf ||+ usOnly ||+ Math.random() * 100 < config.FIRE_PDF_PERCENT)));if (useFirePDF) {// Async is a server-controlled cohort within traffic already selected@@ -735,7 +748,8 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {if ((!includePageMarkdown && !includeBlocks && !pageMarkers) ||error instanceof RemoveFeatureError ||- error instanceof AbortManagerThrownError+ error instanceof AbortManagerThrownError ||+ error instanceof UsOnlyUnavailableError) {throw error;}@@ -783,7 +797,8 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {} catch (error) {if (error instanceof RemoveFeatureError ||- error instanceof AbortManagerThrownError+ error instanceof AbortManagerThrownError ||+ error instanceof UsOnlyUnavailableError) {throw error;}@@ -820,6 +835,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {if (!result &&!forceFirePDF &&+ !usOnly &&fileSizeBytes >= MAX_FILE_SIZE &&config.RUNPOD_MU_API_KEY &&config.RUNPOD_MU_POD_ID@@ -838,6 +854,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {if (!result &&!forceFirePDF &&+ !usOnly &&fileSizeBytes < MAX_FILE_SIZE &&base64Content !== undefined &&config.RUNPOD_MU_API_KEY &&@@ -953,6 +970,23 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {if (totalPageCount === undefined && result.pageCount !== undefined) {totalPageCount = result.pageCount;}+ // Without GPU OCR a scanned document has no text layer to return.+ if (+ usOnly &&+ !skipOCR &&+ (detectedPdfType === "Scanned" ||+ detectedPdfType === "ImageBased" ||+ !result.markdown?.trim())+ ) {+ meta.logger.warn("US-only PDF needs OCR the PDF service did not run", {+ method: "scrapePDF",+ scrape_id: meta.id,+ pdf_type: detectedPdfType,+ });+ throw new UsOnlyUnavailableError(+ "This PDF needs OCR, which is not available for teams restricted to US-only processing right now. Try again later.",+ );+ }}return {diff --git a/apps/api/src/scraper/scrapeURL/transformers/redactPII.test.ts b/apps/api/src/scraper/scrapeURL/transformers/redactPII.test.tsindex 386b2e4a6..60628e9a6 100644--- a/apps/api/src/scraper/scrapeURL/transformers/redactPII.test.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/redactPII.test.ts@@ -1,6 +1,7 @@import type { MockedFunction } from "vitest";import { performRedactPII } from "./redactPII";import { redactText } from "../../../lib/fire-privacy-client";+import { UsOnlyUnavailableError } from "../error";vi.mock("../../../lib/fire-privacy-client", () => ({redactText: vi.fn(),@@ -101,4 +102,36 @@ describe("performRedactPII", () => {expect(mockedRedactText).not.toHaveBeenCalled();expect(document.markdown).toBe("");});++ it("pins redaction to US capacity only for a US-only team", async () => {+ mockedRedactText.mockResolvedValue({+ status: "ok",+ redactedMarkdown: "Hello <PERSON>",+ spans: [],+ counts: {},+ });++ const unpinned = baseMeta();+ unpinned.internalOptions = { teamFlags: {} };+ await performRedactPII(unpinned, baseDocument({ markdown: "Hello Alice" }));++ const pinned = baseMeta();+ pinned.internalOptions = { teamFlags: { usOnly: true } };+ await performRedactPII(pinned, baseDocument({ markdown: "Hello Alice" }));++ expect(mockedRedactText.mock.calls[0][0].usOnly).toBe(false);+ expect(mockedRedactText.mock.calls[1][0].usOnly).toBe(true);+ });++ it("fails the scrape instead of returning unredacted markdown when US capacity refuses", async () => {+ mockedRedactText.mockRejectedValue(+ new UsOnlyUnavailableError("no US capacity"),+ );+ const meta = baseMeta();+ meta.internalOptions = { teamFlags: { usOnly: true } };++ await expect(+ performRedactPII(meta, baseDocument({ markdown: "Hello Alice" })),+ ).rejects.toBeInstanceOf(UsOnlyUnavailableError);+ });});diff --git a/apps/api/src/scraper/scrapeURL/transformers/redactPII.ts b/apps/api/src/scraper/scrapeURL/transformers/redactPII.tsindex d0e62b3cb..f63e2d096 100644--- a/apps/api/src/scraper/scrapeURL/transformers/redactPII.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/redactPII.ts@@ -1,6 +1,7 @@import { Meta } from "..";import { Document } from "../../../controllers/v2/types";import { redactText } from "../../../lib/fire-privacy-client";+import { isUsOnly } from "../../../lib/us-only";export async function performRedactPII(meta: Meta,@@ -23,6 +24,7 @@ export async function performRedactPII(// truthy here means it's the options object; falsy was already// bailed out of above.options: meta.options.redactPII || undefined,+ usOnly: isUsOnly(meta.internalOptions?.teamFlags ?? undefined),});// Swap raw markdown for the redacted version. Caller asked for PII