Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

feat(api): keep US-only PDF parsing, image OCR and PII redaction on US capacity (#5552)

GergΕ‘ MΓ³ricz09b8fd8e8fa3
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.yml
index 5d2edfb3e..76f087043 100644
--- a/.github/workflows/test-server.yml
+++ b/.github/workflows/test-server.yml
@@ -357,6 +357,21 @@ jobs:
--reporter=default
working-directory: apps/api
+ - name: Run US-only PDF, image OCR and PII redaction regression tests
+ if: matrix.engine == 'fetch' && matrix.proxy == 'no-proxy' && matrix.nuq == 'postgres'
+ run: >-
+ pnpm exec vitest run
+ src/scraper/scrapeURL/engines/pdf/__tests__/firePDFUsOnly.test.ts
+ src/scraper/scrapeURL/engines/pdf/__tests__/usOnlyPdfRouting.test.ts
+ src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts
+ src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts
+ src/scraper/scrapeURL/engines/pdf/__tests__/firePDFBackgroundRefresh.test.ts
+ src/scraper/scrapeURL/engines/pdf/__tests__/parseFirePdfRouting.test.ts
+ src/lib/fire-privacy-client.test.ts
+ src/scraper/scrapeURL/transformers/redactPII.test.ts
+ --reporter=default
+ working-directory: apps/api
+
- name: Run country check route regression tests
if: matrix.engine == 'fetch' && matrix.proxy == 'no-proxy' && matrix.nuq == 'postgres'
run: pnpm exec vitest run src/__tests__/routes/country-check.routes.test.ts --reporter=default
diff --git a/apps/api/src/lib/fire-privacy-client.test.ts b/apps/api/src/lib/fire-privacy-client.test.ts
index 44227a577..3bd929bee 100644
--- a/apps/api/src/lib/fire-privacy-client.test.ts
+++ b/apps/api/src/lib/fire-privacy-client.test.ts
@@ -2,6 +2,7 @@ import http from "http";
import { AddressInfo } from "net";
import { config } from "../config";
import { redactText } from "./fire-privacy-client";
+import { UsOnlyUnavailableError } from "../scraper/scrapeURL/error";
type Handler = (
req: http.IncomingMessage,
@@ -718,3 +719,97 @@ describe("redactText", () => {
expect(out.redactedMarkdown).toBe(" - email [email protected]");
});
});
+
+describe("redactText for US-only teams", () => {
+ function capturing(
+ reply: unknown,
+ status = 200,
+ ): { handler: Handler; bodies: Record<string, unknown>[] } {
+ const bodies: Record<string, unknown>[] = [];
+ return {
+ bodies,
+ handler: async (req, res) => {
+ const chunks: Buffer[] = [];
+ for await (const chunk of req) chunks.push(chunk as Buffer);
+ bodies.push(JSON.parse(Buffer.concat(chunks).toString()));
+ res.statusCode = status;
+ res.setHeader("content-type", "application/json");
+ res.end(JSON.stringify(reply));
+ },
+ };
+ }
+
+ it("sends data_region only when usOnly is set", async () => {
+ const reply = {
+ redacted_text: "Hi <PERSON>",
+ spans: [],
+ model_status: "ok",
+ data_region: "us",
+ };
+ const capture = capturing(reply);
+ handler = capture.handler;
+
+ const unpinned = await redactText({ text: "Hi Alice" });
+ const pinned = await redactText({ text: "Hi Alice", usOnly: true });
+
+ expect(Object.hasOwn(capture.bodies[0], "data_region")).toBe(false);
+ expect(capture.bodies[1]).toMatchObject({ data_region: "us" });
+ expect(unpinned.status).toBe("ok");
+ expect(pinned.status).toBe("ok");
+ expect(pinned.redactedMarkdown).toBe("Hi <PERSON>");
+ });
+
+ it("throws UsOnlyUnavailableError when the redaction service has no US capacity", async () => {
+ handler = capturing(
+ { code: "region_unavailable", data_region: "us" },
+ 503,
+ ).handler;
+
+ await expect(
+ redactText({ text: "Hi Alice", usOnly: true }),
+ ).rejects.toBeInstanceOf(UsOnlyUnavailableError);
+ });
+
+ it("throws UsOnlyUnavailableError for a refusal on any status or region field", async () => {
+ handler = capturing({ error_code: "region_unavailable" }, 403).handler;
+
+ await expect(
+ redactText({ text: "Hi Alice", usOnly: true }),
+ ).rejects.toThrow("no US capacity could take this request");
+ });
+
+ it("throws UsOnlyUnavailableError when a pinned reply lacks the region echo", async () => {
+ handler = capturing({
+ redacted_text: "Hi <PERSON>",
+ spans: [],
+ model_status: "ok",
+ }).handler;
+
+ const error = await redactText({ text: "Hi Alice", usOnly: true }).catch(
+ e => e,
+ );
+ expect(error).toBeInstanceOf(UsOnlyUnavailableError);
+ expect(error.message).toContain("did not confirm US-only processing");
+ });
+
+ it("keeps an unpinned 503 as failed/service_unavailable", async () => {
+ handler = capturing(
+ { code: "region_unavailable", data_region: "us" },
+ 503,
+ ).handler;
+
+ const out = await redactText({ text: "Hi Alice" });
+ expect(out.status).toBe("failed");
+ expect(out.reason).toBe("service_unavailable");
+ expect(out.redactedMarkdown).toBeNull();
+ });
+
+ it("keeps an ordinary pinned 503 as failed with no markdown", async () => {
+ handler = capturing({ error: "overloaded" }, 503).handler;
+
+ const out = await redactText({ text: "Hi Alice", usOnly: true });
+ expect(out.status).toBe("failed");
+ expect(out.reason).toBe("service_unavailable");
+ expect(out.redactedMarkdown).toBeNull();
+ });
+});
diff --git a/apps/api/src/lib/fire-privacy-client.ts b/apps/api/src/lib/fire-privacy-client.ts
index 8649a1697..a52545d10 100644
--- a/apps/api/src/lib/fire-privacy-client.ts
+++ b/apps/api/src/lib/fire-privacy-client.ts
@@ -5,11 +5,14 @@ import {
type RedactPIIEntity,
} from "../controllers/v2/types";
import { chunkMarkdown, type Chunk } from "./fire-privacy-chunker";
+import { UsOnlyUnavailableError } from "../scraper/scrapeURL/error";
+import { isRegionUnavailable, US_DATA_REGION } from "./us-only";
type FirePrivacyResponse = {
redacted_text?: unknown;
spans?: unknown;
model_status?: unknown;
+ data_region?: unknown;
};
type RedactOptions = {
@@ -22,6 +25,9 @@ type RedactOptions = {
// wouldn't trigger this code path at all (transformer skips when
// meta.options.redactPII is falsy).
options?: RedactPIIOptions;
+ // Pins the model call to US capacity; a refusal or a reply without the
+ // region echo throws UsOnlyUnavailableError.
+ usOnly?: boolean;
};
type RedactionSource = "model" | "heuristics" | "unknown";
@@ -43,7 +49,9 @@ type RedactionReason =
| "upstream_skipped"
| "service_unavailable"
| "timeout"
- | "error";
+ | "error"
+ | "region_unavailable"
+ | "region_unconfirmed";
type RedactionResult = {
status: RedactionStatus;
@@ -234,6 +242,7 @@ async function redactOnce(
url: string | undefined,
timeoutMs: number,
logger: Logger | undefined,
+ usOnly: boolean,
): Promise<ChunkResult> {
const controller = new AbortController();
let timedOut = false;
@@ -252,6 +261,7 @@ async function redactOnce(
mode: MODE_MAP[options.mode],
operator: REPLACE_MAP[options.replaceStyle],
language: DEFAULTS.language,
+ ...(usOnly && { data_region: US_DATA_REGION }),
}),
signal: controller.signal,
});
@@ -267,11 +277,17 @@ async function redactOnce(
});
return { ok: false, reason };
}
+ // Read under the request deadline so a stalled body cannot hang the chunk.
+ const refusal: unknown =
+ usOnly && !response.ok ? await response.json().catch(() => null) : null;
clearTimeout(timer);
if (!response.ok) {
- const reason: RedactionReason =
- response.status === 503 ? "service_unavailable" : "error";
+ const reason: RedactionReason = isRegionUnavailable(refusal)
+ ? "region_unavailable"
+ : response.status === 503
+ ? "service_unavailable"
+ : "error";
logger?.warn("fire-privacy returned non-2xx", {
reason,
httpStatus: response.status,
@@ -302,6 +318,16 @@ async function redactOnce(
return { ok: false, reason: "error" };
}
+ // A build that ignores the pin answers without the echo; its model call
+ // may have run outside the US.
+ if (usOnly && body.data_region !== US_DATA_REGION) {
+ logger?.warn("fire-privacy did not confirm US-only processing", {
+ url,
+ chunkStart: chunk.start,
+ });
+ return { ok: false, reason: "region_unconfirmed" };
+ }
+
// Lift spans into source coordinates.
const spans = rawSpans.map(s => ({
...s,
@@ -325,6 +351,7 @@ async function runChunks(
url: string | undefined,
timeoutMs: number,
logger: Logger | undefined,
+ usOnly: boolean,
): Promise<ChunkResult[]> {
const results = new Array<ChunkResult>(chunks.length);
let next = 0;
@@ -333,7 +360,14 @@ async function runChunks(
while (true) {
const i = next++;
if (i >= chunks.length) return;
- results[i] = await redactOnce(chunks[i], options, url, timeoutMs, logger);
+ results[i] = await redactOnce(
+ chunks[i],
+ options,
+ url,
+ timeoutMs,
+ logger,
+ usOnly,
+ );
}
};
@@ -405,7 +439,29 @@ export async function redactText(
}
const chunks = chunkMarkdown(text);
- const results = await runChunks(chunks, options, opts.url, timeoutMs, logger);
+ const results = await runChunks(
+ chunks,
+ options,
+ opts.url,
+ timeoutMs,
+ logger,
+ opts.usOnly === true,
+ );
+
+ const regionFailure = results.find(
+ r =>
+ !r.ok &&
+ (r.reason === "region_unavailable" || r.reason === "region_unconfirmed"),
+ );
+ if (regionFailure && !regionFailure.ok) {
+ const detail =
+ regionFailure.reason === "region_unconfirmed"
+ ? "the redaction service did not confirm US-only processing"
+ : "no US capacity could take this request";
+ throw new UsOnlyUnavailableError(
+ `PII redaction is not available for teams restricted to US-only processing right now: ${detail}. Try again later.`,
+ );
+ }
// All-or-nothing: any chunk failure poisons the whole response. Partial
// redaction is worse than no redaction β€” callers can't tell which
diff --git a/apps/api/src/lib/us-only.ts b/apps/api/src/lib/us-only.ts
index 564070d83..10f924639 100644
--- a/apps/api/src/lib/us-only.ts
+++ b/apps/api/src/lib/us-only.ts
@@ -4,3 +4,26 @@ import type { TeamFlags } from "../controllers/v2/types";
export function isUsOnly(flags: TeamFlags | undefined): boolean {
return flags?.usOnly === true;
}
+
+/** The `data_region` value that pins a US-only team's GPU work to US capacity. */
+export const US_DATA_REGION = "us";
+
+/**
+ * Whether a GPU service reply refuses a pinned request because no in-region
+ * capacity could take it. Accepts a parsed body or raw JSON text.
+ */
+export function isRegionUnavailable(body: unknown): boolean {
+ let parsed = body;
+ if (typeof body === "string") {
+ try {
+ parsed = JSON.parse(body);
+ } catch {
+ return false;
+ }
+ }
+ if (typeof parsed !== "object" || parsed === null) return false;
+ const fields = parsed as Record<string, unknown>;
+ return ["code", "error_code", "error_class", "error"].some(
+ key => fields[key] === "region_unavailable",
+ );
+}
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFBackgroundRefresh.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFBackgroundRefresh.test.ts
index 5f31e4fb7..6f5d92113 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFBackgroundRefresh.test.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFBackgroundRefresh.test.ts
@@ -296,4 +296,28 @@ describe("FirePDF cache background refresh", () => {
expect(await refreshCount("skipped_zdr")).toBe(1);
expect(await refreshCount("skipped_image")).toBe(1);
});
+
+ it("serves a US-only team the stale entry without offering or submitting a refresh", async () => {
+ const meta = makeMeta();
+ meta.internalOptions.teamFlags = { usOnly: true };
+ answerWith(staleAnswer(true));
+ expect(await lookupCachedResult(meta, base64, lookupArgs)).toEqual(
+ staleResult,
+ );
+ await settle();
+ expect(lookupFetch.mock.calls[0][0].body).not.toHaveProperty(
+ "background_refresh",
+ );
+ expect(
+ await submitBackgroundRefresh(meta, base64, {
+ mode: undefined,
+ includePageMarkdown: false,
+ includeBlocks: false,
+ pageMarkers: false,
+ sourceKind: "pdf",
+ }),
+ ).toBe("skipped_us_only");
+ expect(jobsFetch).not.toHaveBeenCalled();
+ expect(await refreshCount("skipped_us_only")).toBe(1);
+ });
});
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFUsOnly.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFUsOnly.test.ts
new file mode 100644
index 000000000..f11aab842
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFUsOnly.test.ts
@@ -0,0 +1,369 @@
+// Stub the GCS cache so the async client never reaches real cloud storage.
+vi.mock("../../../../../lib/gcs-pdf-cache", () => ({
+ pdfCacheConfigured: vi.fn(() => true),
+ createPdfCacheKey: (s: string) => `sha-${s.length}`,
+ resolvePdfCacheKey: (input: string | { key: string }) =>
+ typeof input === "string" ? `sha-${input.length}` : input.key,
+ getPdfResultFromCache: vi.fn(async () => null),
+ savePdfResultToCache: vi.fn(async () => null),
+}));
+vi.mock("../../../lib/fetch", () => ({
+ robustFetch: vi.fn(),
+}));
+
+import { robustFetch } from "../../../lib/fetch";
+import { config } from "../../../../../config";
+import { savePdfResultToCache } from "../../../../../lib/gcs-pdf-cache";
+import { UsOnlyUnavailableError } from "../../../error";
+import { scrapePDFWithFirePDF } from "../firePDF";
+import { scrapePDFWithFirePDFAsync } from "../fire-pdf/async";
+import { lookupAdoptableFirePdfJob } from "../fire-pdf/lookup";
+import {
+ makeFetchFromSequence,
+ makeMeta as makeAsyncMeta,
+ noopSleep,
+} from "./firePDFAsyncFixtures";
+
+const mockedRobustFetch = vi.mocked(robustFetch);
+
+const usOnlyFlags = { usOnly: true };
+
+function makeSyncMeta(teamFlags?: Record<string, unknown>) {
+ const logger: any = {
+ info: vi.fn(),
+ warn: vi.fn(),
+ error: vi.fn(),
+ debug: vi.fn(),
+ };
+ return {
+ id: "us-only-sync-test",
+ url: "https://example.com/file.pdf",
+ rewrittenUrl: undefined,
+ options: {},
+ logger,
+ mock: null,
+ abort: {
+ throwIfAborted: vi.fn(),
+ asSignal: vi.fn(() => new AbortController().signal),
+ scrapeTimeout: vi.fn(() => 60_000),
+ },
+ internalOptions: {
+ zeroDataRetention: false,
+ teamId: "team-test",
+ ...(teamFlags && { teamFlags }),
+ },
+ } as any;
+}
+
+function makeUsOnlyAsyncMeta(teamFlags?: Record<string, unknown>) {
+ return makeAsyncMeta({
+ internalOptions: {
+ zeroDataRetention: false,
+ teamId: "team-x",
+ teamConcurrency: 12,
+ crawlId: undefined,
+ ...(teamFlags && { teamFlags }),
+ },
+ });
+}
+
+function failureStatus(status: number, body: unknown) {
+ return new Error("Request sent failure status", {
+ cause: { response: { status, body: JSON.stringify(body) } },
+ });
+}
+
+const regionRefusal = {
+ code: "region_unavailable",
+ data_region: "us",
+ error: "no in-region capacity",
+};
+
+const ORIGINAL_BASE_URL = config.FIRE_PDF_BASE_URL;
+
+beforeAll(() => {
+ config.FIRE_PDF_BASE_URL = "http://fire-pdf.test";
+});
+
+afterAll(() => {
+ config.FIRE_PDF_BASE_URL = ORIGINAL_BASE_URL;
+});
+
+beforeEach(() => {
+ vi.clearAllMocks();
+});
+
+describe("FirePDF sync parse for US-only teams", () => {
+ it("sends data_region only for a US-only team", async () => {
+ mockedRobustFetch.mockResolvedValue({
+ markdown: "Document",
+ failed_pages: null,
+ pages_processed: 1,
+ data_region: "us",
+ } as any);
+
+ await scrapePDFWithFirePDF(makeSyncMeta(), "BASE64", 1);
+ await scrapePDFWithFirePDF(makeSyncMeta(usOnlyFlags), "BASE64", 1);
+
+ const unpinned = mockedRobustFetch.mock.calls[0][0].body;
+ const pinned = mockedRobustFetch.mock.calls[1][0].body;
+ expect(Object.hasOwn(unpinned, "data_region")).toBe(false);
+ expect(pinned.data_region).toBe("us");
+ });
+
+ it("returns a US-only result that carries the region echo", async () => {
+ mockedRobustFetch.mockResolvedValue({
+ markdown: "Document",
+ failed_pages: null,
+ pages_processed: 1,
+ data_region: "us",
+ } as any);
+
+ const result = await scrapePDFWithFirePDF(
+ makeSyncMeta(usOnlyFlags),
+ "BASE64",
+ 1,
+ );
+
+ expect(result.markdown).toBe("Document");
+ });
+
+ it("refuses a US-only parse the PDF service could not place on US capacity", async () => {
+ mockedRobustFetch.mockRejectedValue(failureStatus(503, regionRefusal));
+
+ const error = await scrapePDFWithFirePDF(
+ makeSyncMeta(usOnlyFlags),
+ "BASE64",
+ 1,
+ ).catch(e => e);
+
+ expect(error).toBeInstanceOf(UsOnlyUnavailableError);
+ expect(error.code).toBe("US_ONLY_UNAVAILABLE");
+ expect(error.message).toContain("PDF parsing is not available");
+ });
+
+ it.each([
+ ["missing", undefined],
+ ["null", null],
+ ])(
+ "refuses a US-only result whose region echo is %s",
+ async (_name, dataRegion) => {
+ mockedRobustFetch.mockResolvedValue({
+ markdown: "Document",
+ failed_pages: null,
+ pages_processed: 1,
+ data_region: dataRegion,
+ } as any);
+
+ const error = await scrapePDFWithFirePDF(
+ makeSyncMeta(usOnlyFlags),
+ "BASE64",
+ 1,
+ ).catch(e => e);
+
+ expect(error).toBeInstanceOf(UsOnlyUnavailableError);
+ expect(error.message).toContain("did not confirm US-only processing");
+ },
+ );
+
+ it("names image OCR when a US-only image is refused", async () => {
+ mockedRobustFetch.mockRejectedValue(failureStatus(503, regionRefusal));
+
+ const error = await scrapePDFWithFirePDF(
+ makeSyncMeta(usOnlyFlags),
+ "BASE64",
+ 1,
+ 1,
+ "ocr",
+ false,
+ false,
+ false,
+ "image",
+ ).catch(e => e);
+
+ expect(error).toBeInstanceOf(UsOnlyUnavailableError);
+ expect(error.message).toContain("Image OCR is not available");
+ });
+
+ it("keeps an unpinned failure and an unpinned result without the echo unchanged", async () => {
+ const failure = failureStatus(503, regionRefusal);
+ mockedRobustFetch.mockRejectedValueOnce(failure);
+ expect(
+ await scrapePDFWithFirePDF(makeSyncMeta(), "BASE64", 1).catch(e => e),
+ ).toBe(failure);
+
+ mockedRobustFetch.mockResolvedValueOnce({
+ markdown: "Document",
+ failed_pages: null,
+ pages_processed: 1,
+ } as any);
+ const result = await scrapePDFWithFirePDF(makeSyncMeta(), "BASE64", 1);
+ expect(result.markdown).toBe("Document");
+ });
+});
+
+const submitAccepted = {
+ matchUrl: /\/jobs$/,
+ matchMethod: "POST" as const,
+ response: {
+ status: 202,
+ body: { scrape_id: "scrape-id-test", status: "queued", lane: "fast" },
+ },
+};
+const pollDone = {
+ matchUrl: /\/jobs\/scrape-id-test$/,
+ matchMethod: "GET" as const,
+ response: {
+ status: 200,
+ body: { scrape_id: "scrape-id-test", status: "done", pages_processed: 3 },
+ },
+};
+function resultWith(extra: Record<string, unknown>) {
+ return {
+ matchUrl: /\/jobs\/scrape-id-test\/result$/,
+ matchMethod: "GET" as const,
+ response: {
+ status: 200,
+ body: {
+ schema_version: 1,
+ markdown: "# Async",
+ pages_processed: 3,
+ failed_pages: null,
+ partial_pages: null,
+ ...extra,
+ },
+ },
+ };
+}
+
+function runAsync(meta: any, fetchImpl: any) {
+ return scrapePDFWithFirePDFAsync(
+ meta,
+ "BASE64",
+ undefined,
+ undefined,
+ undefined,
+ { fetchImpl, fallbackImpl: vi.fn(), sleepImpl: noopSleep },
+ );
+}
+
+describe("FirePDF async jobs for US-only teams", () => {
+ it("sends data_region on the submit only for a US-only team", async () => {
+ const unpinned = makeFetchFromSequence([
+ submitAccepted,
+ pollDone,
+ resultWith({}),
+ ]);
+ await runAsync(makeUsOnlyAsyncMeta(), unpinned.fetchImpl);
+
+ const pinned = makeFetchFromSequence([
+ submitAccepted,
+ pollDone,
+ resultWith({ data_region: "us" }),
+ ]);
+ const result = await runAsync(
+ makeUsOnlyAsyncMeta(usOnlyFlags),
+ pinned.fetchImpl,
+ );
+
+ expect(unpinned.calls[0].body).not.toHaveProperty("data_region");
+ expect(pinned.calls[0].body).toMatchObject({ data_region: "us" });
+ expect(result.markdown).toBe("# Async");
+ });
+
+ it("refuses a US-only job that failed for lack of US capacity, without cancelling it", async () => {
+ const { fetchImpl, calls } = makeFetchFromSequence([
+ submitAccepted,
+ {
+ matchUrl: /\/jobs\/scrape-id-test$/,
+ matchMethod: "GET",
+ response: {
+ status: 502,
+ body: {
+ scrape_id: "scrape-id-test",
+ status: "failed",
+ error_class: "region_unavailable",
+ error_message: "no in-region capacity",
+ error_code: "region_unavailable",
+ },
+ },
+ },
+ ]);
+
+ const error = await runAsync(
+ makeUsOnlyAsyncMeta(usOnlyFlags),
+ fetchImpl,
+ ).catch(e => e);
+
+ expect(error).toBeInstanceOf(UsOnlyUnavailableError);
+ expect(calls.map(call => call.method)).toEqual(["POST", "GET"]);
+ });
+
+ it("refuses a US-only submit the PDF service turned away for lack of US capacity, without retrying", async () => {
+ const { fetchImpl, calls } = makeFetchFromSequence([
+ {
+ matchUrl: /\/jobs$/,
+ matchMethod: "POST",
+ response: { status: 503, body: regionRefusal },
+ },
+ ]);
+
+ const error = await runAsync(
+ makeUsOnlyAsyncMeta(usOnlyFlags),
+ fetchImpl,
+ ).catch(e => e);
+
+ expect(error).toBeInstanceOf(UsOnlyUnavailableError);
+ expect(error.message).toContain("no US capacity could take this document");
+ expect(calls).toHaveLength(1);
+ });
+
+ it("refuses a US-only result that lacks the region echo and caches nothing", async () => {
+ const { fetchImpl } = makeFetchFromSequence([
+ submitAccepted,
+ pollDone,
+ resultWith({}),
+ ]);
+
+ const error = await runAsync(
+ makeUsOnlyAsyncMeta(usOnlyFlags),
+ fetchImpl,
+ ).catch(e => e);
+
+ expect(error).toBeInstanceOf(UsOnlyUnavailableError);
+ expect(savePdfResultToCache).not.toHaveBeenCalled();
+ });
+
+ it("sends data_region on the content lookup only for a US-only team", async () => {
+ const lookupHit = {
+ matchUrl: /\/jobs\/lookup$/,
+ matchMethod: "POST" as const,
+ response: {
+ status: 200,
+ body: { scrape_id: "found-1", status: "running" },
+ },
+ };
+ const unpinned = makeFetchFromSequence([lookupHit]);
+ const pinned = makeFetchFromSequence([lookupHit]);
+ const sha = "ef".repeat(32);
+
+ await lookupAdoptableFirePdfJob(
+ makeUsOnlyAsyncMeta(),
+ sha,
+ { mode: "auto" },
+ unpinned.fetchImpl,
+ );
+ await lookupAdoptableFirePdfJob(
+ makeUsOnlyAsyncMeta(usOnlyFlags),
+ sha,
+ { mode: "auto" },
+ pinned.fetchImpl,
+ );
+
+ expect(unpinned.calls[0].body).not.toHaveProperty("data_region");
+ expect(pinned.calls[0].body).toMatchObject({
+ data_region: "us",
+ options: { mode: "auto" },
+ });
+ });
+});
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/usOnlyPdfRouting.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/usOnlyPdfRouting.test.ts
new file mode 100644
index 000000000..8d3d4c54f
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/usOnlyPdfRouting.test.ts
@@ -0,0 +1,217 @@
+import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { processPdf } from "@mendable/firecrawl-rs";
+import { config } from "../../../../../config";
+import { UsOnlyUnavailableError } from "../../../error";
+import { scrapePDFWithFirePDF } from "../firePDF";
+import { scrapePDFWithParsePDF } from "../pdfParse";
+import { scrapePDFWithRunPodMU } from "../runpodMU";
+import { mineruDiverted } from "../fire-pdf/by-reference";
+import { scrapePDF } from "../index";
+
+vi.mock("@mendable/firecrawl-rs", () => ({
+ processPdf: vi.fn(),
+ detectPdf: vi.fn(),
+}));
+
+vi.mock("../firePDF", async importOriginal => ({
+ ...(await importOriginal<typeof import("../firePDF")>()),
+ scrapePDFWithFirePDF: vi.fn(),
+}));
+
+vi.mock("../pdfParse", () => ({
+ scrapePDFWithParsePDF: vi.fn(),
+}));
+
+vi.mock("../runpodMU", () => ({
+ scrapePDFWithRunPodMU: vi.fn(),
+}));
+
+vi.mock("../markdownToHtml", () => ({
+ safeMarkdownToHtml: vi.fn(async (markdown: string) => `<p>${markdown}</p>`),
+}));
+
+const mockedProcessPdf = vi.mocked(processPdf);
+const mockedFirePdf = vi.mocked(scrapePDFWithFirePDF);
+const mockedPdfParse = vi.mocked(scrapePDFWithParsePDF);
+const mockedRunPodMU = vi.mocked(scrapePDFWithRunPodMU);
+
+const configKeys = [
+ "PDF_RUST_EXTRACT_ENABLE",
+ "FIRE_PDF_ENABLE",
+ "FIRE_PDF_BASE_URL",
+ "FIRE_PDF_PERCENT",
+ "FIRE_PDF_ASYNC_PERCENT",
+ "MINERU_PERCENT",
+ "RUNPOD_MU_API_KEY",
+ "RUNPOD_MU_POD_ID",
+ "PARSE_PDF_ALWAYS_FIRE_PDF",
+] as const;
+const originalConfig = Object.fromEntries(
+ configKeys.map(key => [key, config[key]]),
+);
+
+function setConfig(
+ overrides: Partial<Record<(typeof configKeys)[number], unknown>>,
+) {
+ Object.assign(config as any, overrides);
+}
+
+const tempDirs: string[] = [];
+
+async function makeMeta(usOnly: boolean) {
+ const dir = await mkdtemp(path.join(tmpdir(), "us-only-pdf-"));
+ tempDirs.push(dir);
+ const filePath = path.join(dir, "doc.pdf");
+ await writeFile(filePath, "%PDF-1.7\n%test\n");
+ const logger: any = {
+ info: vi.fn(),
+ warn: vi.fn(),
+ error: vi.fn(),
+ debug: vi.fn(),
+ };
+ logger.child = vi.fn(() => logger);
+ return {
+ id: "01a00000-0000-7000-8000-000000000001",
+ url: "https://example.com/doc.pdf",
+ options: { parsers: ["pdf"] },
+ logger,
+ featureFlags: new Set(["pdf"]),
+ pdfPrefetch: {
+ filePath,
+ url: "https://example.com/doc.pdf",
+ status: 200,
+ proxyUsed: "basic",
+ },
+ abort: {
+ throwIfAborted: vi.fn(),
+ asSignal: vi.fn(() => new AbortController().signal),
+ scrapeTimeout: vi.fn(() => 60_000),
+ },
+ internalOptions: {
+ teamId: "team-test",
+ ...(usOnly && { teamFlags: { usOnly: true } }),
+ },
+ } as any;
+}
+
+function nativeResult(pdfType: string, markdown: string) {
+ return {
+ pdfType,
+ confidence: 0.99,
+ isComplex: true,
+ pageCount: 1,
+ markdown,
+ title: null,
+ pagesNeedingOcr: [],
+ logs: [],
+ } as any;
+}
+
+describe("PDF routing for US-only teams", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ setConfig({
+ PDF_RUST_EXTRACT_ENABLE: true,
+ FIRE_PDF_ENABLE: true,
+ FIRE_PDF_BASE_URL: "http://fire-pdf.test",
+ FIRE_PDF_PERCENT: 100,
+ FIRE_PDF_ASYNC_PERCENT: 0,
+ MINERU_PERCENT: 0,
+ RUNPOD_MU_API_KEY: "test-key",
+ RUNPOD_MU_POD_ID: "test-endpoint",
+ PARSE_PDF_ALWAYS_FIRE_PDF: false,
+ });
+ mockedProcessPdf.mockResolvedValue(nativeResult("TextBased", "text layer"));
+ mockedFirePdf.mockResolvedValue({
+ markdown: "fire-pdf",
+ html: "<p>fire-pdf</p>",
+ pagesProcessed: 1,
+ });
+ mockedPdfParse.mockResolvedValue({
+ markdown: "pdf-parse",
+ html: "<p>pdf-parse</p>",
+ });
+ mockedRunPodMU.mockResolvedValue({
+ markdown: "serverless",
+ html: "<p>serverless</p>",
+ });
+ });
+
+ afterEach(async () => {
+ await Promise.all(
+ tempDirs.splice(0).map(dir => rm(dir, { recursive: true, force: true })),
+ );
+ });
+
+ afterAll(() => {
+ Object.assign(config as any, originalConfig);
+ });
+
+ it("never diverts a US-only team to the alternate PDF engine", async () => {
+ setConfig({ MINERU_PERCENT: 100 });
+
+ expect(mineruDiverted(await makeMeta(false))).toBe(true);
+ expect(mineruDiverted(await makeMeta(true))).toBe(false);
+
+ const result = await scrapePDF(await makeMeta(true));
+
+ expect(result.markdown).toBe("fire-pdf");
+ expect(mockedFirePdf).toHaveBeenCalledTimes(1);
+ expect(mockedRunPodMU).not.toHaveBeenCalled();
+ });
+
+ it("sends a US-only PDF to the PDF service regardless of FIRE_PDF_PERCENT", async () => {
+ setConfig({ FIRE_PDF_PERCENT: 0, RUNPOD_MU_API_KEY: undefined });
+
+ expect((await scrapePDF(await makeMeta(false))).markdown).toBe("pdf-parse");
+ expect(mockedFirePdf).not.toHaveBeenCalled();
+
+ expect((await scrapePDF(await makeMeta(true))).markdown).toBe("fire-pdf");
+ expect(mockedFirePdf).toHaveBeenCalledTimes(1);
+ });
+
+ it("fails a US-only scrape the PDF service refuses, with no fallback", async () => {
+ mockedFirePdf.mockRejectedValueOnce(
+ new UsOnlyUnavailableError("no US capacity"),
+ );
+
+ const error = await scrapePDF(await makeMeta(true)).catch(e => e);
+
+ expect(error).toBeInstanceOf(UsOnlyUnavailableError);
+ expect(mockedRunPodMU).not.toHaveBeenCalled();
+ expect(mockedPdfParse).not.toHaveBeenCalled();
+ });
+
+ it("skips the serverless fallback but keeps the text layer for a US-only team", async () => {
+ mockedFirePdf.mockRejectedValue(new Error("fire-pdf 500"));
+
+ expect((await scrapePDF(await makeMeta(false))).markdown).toBe(
+ "serverless",
+ );
+ expect(mockedRunPodMU).toHaveBeenCalledTimes(1);
+
+ mockedRunPodMU.mockClear();
+ expect((await scrapePDF(await makeMeta(true))).markdown).toBe("pdf-parse");
+ expect(mockedRunPodMU).not.toHaveBeenCalled();
+ });
+
+ it.each([
+ ["a scanned PDF", "Scanned", "page header"],
+ ["a PDF with no text layer", "Mixed", " "],
+ ])(
+ "refuses %s for a US-only team instead of returning it without OCR",
+ async (_name, pdfType, text) => {
+ mockedProcessPdf.mockResolvedValue(nativeResult(pdfType, ""));
+ mockedFirePdf.mockRejectedValue(new Error("fire-pdf 500"));
+ mockedPdfParse.mockResolvedValue({ markdown: text, html: text });
+
+ const error = await scrapePDF(await makeMeta(true)).catch(e => e);
+
+ expect(error).toBeInstanceOf(UsOnlyUnavailableError);
+ expect(error.message).toContain("This PDF needs OCR");
+ expect(mockedRunPodMU).not.toHaveBeenCalled();
+ },
+ );
+});
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts
index e4b1d7b3c..57f9667ac 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts
@@ -9,6 +9,8 @@ import { scrapePDFWithFirePDF } from "../firePDF";
import { cancelJob } from "./cancel";
import { maybeSaveResult, provenanceFromResponse, tryGetCached } from "./cache";
import { resolvePdfCacheKey } from "../../../../../lib/gcs-pdf-cache";
+import { isUsOnly, US_DATA_REGION } from "../../../../../lib/us-only";
+import { firePdfUsOnlyRefusal } from "./data-region";
import {
firePdfAsyncAbandonedTotal,
firePdfAsyncTotalDurationSeconds,
@@ -327,7 +329,8 @@ export async function scrapePDFWithFirePDFAsync(
error instanceof FirePdfAsyncFailure &&
(error.reason === "terminal_failed" ||
error.reason === "terminal_expired" ||
- error.reason === "terminal_cancelled");
+ error.reason === "terminal_cancelled" ||
+ error.reason === "region_unavailable");
if ((jobAlreadyTerminal || terminalReached) && meta.largePdfProcessing) {
// The job is dead β€” a "processing continues" message would lie.
meta.largePdfProcessing.current = undefined;
@@ -371,6 +374,12 @@ export async function scrapePDFWithFirePDFAsync(
) {
await cancelJob({ baseUrl, scrapeId: meta.id, meta, fetchImpl });
}
+ if (
+ error instanceof FirePdfAsyncFailure &&
+ error.reason === "region_unavailable"
+ ) {
+ throw firePdfUsOnlyRefusal(meta, "region_unavailable");
+ }
throw submitMayHaveBeenAccepted ? error.originalError : error;
}
@@ -381,6 +390,15 @@ export async function scrapePDFWithFirePDFAsync(
meta.largePdfProcessing.current = undefined;
}
+ // Adopted jobs included: a job that ran before the pin existed carries no
+ // echo, and its result may have come from GPU capacity outside the US.
+ if (
+ isUsOnly(meta.internalOptions.teamFlags ?? undefined) &&
+ fetched.data_region !== US_DATA_REGION
+ ) {
+ throw firePdfUsOnlyRefusal(meta, "region_unconfirmed");
+ }
+
// ── Assemble + cache save ─────────────────────────────────────────────
const pages =
fetched.pages_processed ?? polled.poll.pages_processed ?? pagesProcessed;
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/background-refresh.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/background-refresh.ts
index f8333b049..88b0826e2 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/background-refresh.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/background-refresh.ts
@@ -3,6 +3,7 @@ import { fetch as undiciFetch } from "undici";
import type { Meta } from "../../..";
import type { PDFMode } from "../../../../../controllers/v2/types";
import { config } from "../../../../../config";
+import { isUsOnly } from "../../../../../lib/us-only";
import type { PdfCacheKeyInput } from "../../../../../lib/gcs-pdf-cache";
import {
type BackgroundRefreshResult,
@@ -107,6 +108,10 @@ async function submit(
fetchImpl: typeof undiciFetch,
): Promise<BackgroundRefreshResult> {
if (meta.internalOptions.zeroDataRetention === true) return "skipped_zdr";
+ // A refresh carries no team and nobody checks its region echo.
+ if (isUsOnly(meta.internalOptions.teamFlags ?? undefined)) {
+ return "skipped_us_only";
+ }
// A by-reference document's bytes are in fire-pdf's input bucket, not here.
if (typeof input !== "string") return "skipped_by_reference";
// Raster images are only ever parsed through the sync endpoint.
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference.ts
index 22b42e37c..a2aacae38 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference.ts
@@ -12,6 +12,7 @@ import {
} from "../../../../../controllers/v2/types";
import { deterministicPercentage } from "./routing";
import { storage } from "../../../../../lib/gcs-jobs";
+import { isUsOnly } from "../../../../../lib/us-only";
import { FIRE_PDF_BY_REFERENCE_MAX_FILE_SIZE } from "../types";
import { parseAlwaysFirePdf } from "./request-metadata";
@@ -201,11 +202,12 @@ export function largePdfLimitBytes(meta: Meta): number {
* fire-engine handoff grant, the download admission, and the PDF engine's
* routing β€” sees the SAME verdict; a per-call random draw would let them
* drift. Keyed apart from the async-cohort hash. Parse requests that must
- * reach FirePDF (parseAlwaysFirePdf) are never diverted. */
+ * reach FirePDF (parseAlwaysFirePdf) and US-only teams are never diverted. */
export function mineruDiverted(meta: Meta): boolean {
return (
config.MINERU_PERCENT > 0 &&
!parseAlwaysFirePdf(meta) &&
+ !isUsOnly(meta.internalOptions.teamFlags ?? undefined) &&
deterministicPercentage(`mineru:${meta.id}`) < config.MINERU_PERCENT
);
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache-service.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache-service.ts
index b8579a190..2ab92c08a 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache-service.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache-service.ts
@@ -8,6 +8,7 @@ import {
} from "../../../../../lib/gcs-pdf-cache";
import type { PDFMode } from "../../../../../controllers/v2/types";
import { robustFetch } from "../../../lib/fetch";
+import { isUsOnly } from "../../../../../lib/us-only";
import {
recordSpanException,
type Span,
@@ -193,12 +194,14 @@ async function lookupWithinSpan(
meta.abort.throwIfAborted();
// Background refresh is only offered when this client could act on a
// grant: it needs the bytes (inline, not by reference) of a PDF (images
- // are only parsed through the sync endpoint), outside zero data retention.
+ // are only parsed through the sync endpoint), outside zero data retention
+ // and outside US-only teams.
const canRefreshInBackground =
config.FIRE_PDF_CACHE_BACKGROUND_REFRESH === true &&
typeof input === "string" &&
sourceKind === "pdf" &&
- meta.internalOptions.zeroDataRetention !== true;
+ meta.internalOptions.zeroDataRetention !== true &&
+ !isUsOnly(meta.internalOptions.teamFlags ?? undefined);
const keys = await cacheLookupKeys(input);
if (
sourceKind === "pdf" &&
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/data-region.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/data-region.ts
new file mode 100644
index 000000000..14123aef9
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/data-region.ts
@@ -0,0 +1,24 @@
+import type { Meta } from "../../..";
+import { UsOnlyUnavailableError } from "../../../error";
+import type { FirePdfSourceKind } from "./request-metadata";
+
+/** The error a US-only team gets when the PDF service did not run, or did not confirm running, its document on US capacity. */
+export function firePdfUsOnlyRefusal(
+ meta: Meta,
+ reason: "region_unavailable" | "region_unconfirmed",
+ sourceKind: FirePdfSourceKind = "pdf",
+): UsOnlyUnavailableError {
+ meta.logger.warn("FirePDF could not keep a US-only request on US capacity", {
+ scrapeId: meta.id,
+ reason,
+ sourceKind,
+ });
+ const feature = sourceKind === "image" ? "Image OCR" : "PDF parsing";
+ const detail =
+ reason === "region_unconfirmed"
+ ? "the PDF service did not confirm US-only processing"
+ : "no US capacity could take this document";
+ return new UsOnlyUnavailableError(
+ `${feature} is not available for teams restricted to US-only processing right now: ${detail}. Try again later.`,
+ );
+}
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/lookup.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/lookup.ts
index 473edec00..9f215f9df 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/lookup.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/lookup.ts
@@ -8,6 +8,7 @@ import type { Meta } from "../../..";
import { config } from "../../../../../config";
import { fetch as undiciFetch } from "undici";
import { firePdfHeaders } from "./utils";
+import { isUsOnly, US_DATA_REGION } from "../../../../../lib/us-only";
/** Handle for adopting an existing fire-pdf job instead of submitting a
* new one. Produced by {@link lookupAdoptableFirePdfJob}: the job was
@@ -71,6 +72,9 @@ export async function lookupAdoptableFirePdfJob(
team_id: meta.internalOptions.teamId,
}),
options,
+ ...(isUsOnly(meta.internalOptions.teamFlags ?? undefined) && {
+ data_region: US_DATA_REGION,
+ }),
}),
signal,
});
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.ts
index 3befd414b..bb39bed16 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.ts
@@ -14,7 +14,7 @@ export const firePdfCacheRefusedWritesTotal = new Counter({
export const firePdfCacheBackgroundRefreshTotal = new Counter({
name: "firecrawl_fire_pdf_cache_background_refresh_total",
- help: "Background refreshes the fire-pdf cache service granted on a stale answer, by result: submitted, submit_failed, skipped_by_reference, skipped_zdr, skipped_image",
+ help: "Background refreshes the fire-pdf cache service granted on a stale answer, by result: submitted, submit_failed, skipped_by_reference, skipped_zdr, skipped_image, skipped_us_only",
labelNames: ["result"],
});
@@ -23,7 +23,8 @@ export type BackgroundRefreshResult =
| "submit_failed"
| "skipped_by_reference"
| "skipped_zdr"
- | "skipped_image";
+ | "skipped_image"
+ | "skipped_us_only";
export type CacheRefusedReason =
| "failed_pages"
@@ -122,4 +123,5 @@ export type FallbackReason =
| "terminal_expired"
| "terminal_cancelled"
| "polling_timeout"
- | "result_503";
+ | "result_503"
+ | "region_unavailable";
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/poll.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/poll.ts
index 991ca2290..6d440eff2 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/poll.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/poll.ts
@@ -1,6 +1,7 @@
import type { Meta } from "../../..";
import { fetch as undiciFetch } from "undici";
import { AbortManagerThrownError } from "../../../lib/abortManager";
+import { isRegionUnavailable, isUsOnly } from "../../../../../lib/us-only";
import {
firePdfAsyncCompletedTotal,
firePdfAsyncLongPollTotal,
@@ -171,6 +172,16 @@ export async function pollUntilTerminal(deps: PollDeps): Promise<PollOk> {
const pollStatus = pollResp.status;
const pollBody = await pollResp.json().catch(() => ({}));
+ if (
+ isUsOnly(meta.internalOptions.teamFlags ?? undefined) &&
+ isRegionUnavailable(pollBody)
+ ) {
+ if (waitMs > 0) firePdfAsyncLongPollTotal.labels("terminal").inc();
+ firePdfAsyncPollCount.observe(pollCount);
+ firePdfAsyncCompletedTotal.labels("failed").inc();
+ failAsync(meta, "region_unavailable", { pollCount, body: pollBody });
+ }
+
if (pollStatus === 401) {
firePdfAsyncPollCount.observe(pollCount);
failAsync(meta, "http_401", { pollCount });
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts
index 66ac5ee08..fdf8d4820 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts
@@ -301,6 +301,7 @@ export const resultResponseSchema = z.object({
// Raw on purpose: parsed separately by parseProvenance so a stamp this
// build does not understand never fails the scrape.
provenance: z.unknown().optional(),
+ data_region: z.unknown().optional(),
});
export type PollResponse = z.infer<typeof pollResponseSchema>;
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts
index 448d55973..02459d586 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts
@@ -2,6 +2,11 @@ import type { Meta } from "../../..";
import type { PDFMode } from "../../../../../controllers/v2/types";
import { fetch as undiciFetch } from "undici";
import { AbortManagerThrownError } from "../../../lib/abortManager";
+import {
+ isRegionUnavailable,
+ isUsOnly,
+ US_DATA_REGION,
+} from "../../../../../lib/us-only";
import {
buildFirePdfJobTrainingCapture,
buildFirePdfRequestMetadata,
@@ -118,6 +123,7 @@ export async function submitJob(args: SubmitArgs): Promise<SubmitOutcome> {
fetchImpl,
} = args;
const scrapeId = meta.id;
+ const usOnly = isUsOnly(meta.internalOptions.teamFlags ?? undefined);
if (
input.kind === "byReference" &&
@@ -164,6 +170,7 @@ export async function submitJob(args: SubmitArgs): Promise<SubmitOutcome> {
tableFormat: firePdfTableFormat(meta),
trainingCapture: buildFirePdfJobTrainingCapture(meta),
}),
+ ...(usOnly && { data_region: US_DATA_REGION }),
};
// The retry does reach a different instance: Fastify's shutdown 503
@@ -216,6 +223,9 @@ export async function submitJob(args: SubmitArgs): Promise<SubmitOutcome> {
}
failPossiblyAccepted(meta, "network_error", { error: String(error) });
}
+ if (usOnly && isRegionUnavailable(json)) {
+ failAsync(meta, "region_unavailable", { status });
+ }
const transient503 = attempt === 1 && classifyTransient503(status, json);
if (transient503) {
await retryAfterTransient(transient503, { body: json });
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts
index 19839f72e..726c3e3e7 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts
@@ -6,6 +6,11 @@ import type { PDFProcessorResult } from "./types";
import type { PDFMode } from "../../../../controllers/v2/types";
import { safeMarkdownToHtml } from "./markdownToHtml";
import { createPdfCacheKey } from "../../../../lib/gcs-pdf-cache";
+import {
+ isRegionUnavailable,
+ isUsOnly,
+ US_DATA_REGION,
+} from "../../../../lib/us-only";
import {
maybeSaveResult,
provenanceFromResponse,
@@ -18,6 +23,7 @@ import {
firePdfTableFormat,
type FirePdfSourceKind,
} from "./fire-pdf/request-metadata";
+import { firePdfUsOnlyRefusal } from "./fire-pdf/data-region";
/**
* Reconcile an existing page count with what fire-pdf reported.
@@ -107,6 +113,7 @@ export async function scrapePDFWithFirePDF(
});
const zdr = meta.internalOptions.zeroDataRetention === true;
+ const usOnly = isUsOnly(meta.internalOptions.teamFlags ?? undefined);
const pdfSha256 = createPdfCacheKey(base64Content);
const tableFormat = firePdfTableFormat(meta);
@@ -157,6 +164,7 @@ export async function scrapePDFWithFirePDF(
zdr,
training_capture: buildFirePdfTrainingCapture(meta),
...deadlineFields,
+ ...(usOnly && { data_region: US_DATA_REGION }),
},
logger,
schema: z.object({
@@ -177,10 +185,27 @@ export async function scrapePDFWithFirePDF(
// the echo is the only proof the fire-pdf build understood the
// option β€” older builds ignore unknown request fields and omit it.
page_markers: z.literal(true).optional(),
+ data_region: z.unknown().optional(),
}),
mock: meta.mock,
abort: meta.abort.asSignal(),
+ }).catch(error => {
+ if (
+ usOnly &&
+ isRegionUnavailable(
+ (error as { cause?: { response?: { body?: unknown } } })?.cause
+ ?.response?.body,
+ )
+ ) {
+ throw firePdfUsOnlyRefusal(meta, "region_unavailable", sourceKind);
+ }
+ throw error;
});
+ // A build that ignores the pin answers without the echo; its result may
+ // have come from GPU capacity outside the US.
+ if (usOnly && resp.data_region !== US_DATA_REGION) {
+ throw firePdfUsOnlyRefusal(meta, "region_unconfirmed", sourceKind);
+ }
const durationMs = Date.now() - startedAt;
if (includePageMarkdown && resp.pages === undefined) {
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
index 0145efc85..a208996f9 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
@@ -16,6 +16,7 @@ import {
RemoveFeatureError,
EngineUnsuccessfulError,
UnsupportedFileError,
+ UsOnlyUnavailableError,
} from "../../error";
import { open, readFile, stat, unlink } from "node:fs/promises";
import type { Response } from "undici";
@@ -45,6 +46,7 @@ import {
extractAndEmitNativeLogs,
} from "../../../../lib/native-logging";
import { withSpan, setSpanAttributes } from "../../../../lib/otel-tracer";
+import { isUsOnly } from "../../../../lib/us-only";
import { scrapePDFWithRunPodMU } from "./runpodMU";
import { useFireEngine } from "../fire-engine/available";
import { reconcilePageCountWithFirePdf, scrapePDFWithFirePDF } from "./firePDF";
@@ -243,6 +245,11 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
// Unlike forceFirePDF this keeps the MinerU / pdf-parse fallbacks.
const alwaysFirePdf = parseAlwaysFirePdf(meta);
+ // US-only teams get GPU parsing only from the PDF service, pinned to US
+ // capacity: no diversion to another engine, no serverless fallback, and no
+ // sampling out of the PDF service.
+ const usOnly = isUsOnly(meta.internalOptions.teamFlags ?? undefined);
+
// The MinerU diversion is deterministic on the scrape id (see
// mineruDiverted) so this routing verdict is BY CONSTRUCTION the same
// one inside byReferenceReachableForRequest() β€” and the same one
@@ -324,6 +331,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
let shadowIsComplex: boolean | undefined;
let shadowIneligibleReason: string | null | undefined;
let shadowPagesNeedingOcr: number[] | undefined;
+ let detectedPdfType: string | undefined;
const rustEnabled = !!config.PDF_RUST_EXTRACT_ENABLE;
const logger = meta.logger.child({ method: "scrapePDF/processPdf" });
@@ -376,6 +384,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
});
totalPageCount = detection.pageCount;
+ detectedPdfType = detection.pdfType;
effectivePageCount = maxPages
? Math.min(detection.pageCount, maxPages)
: detection.pageCount;
@@ -423,6 +432,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
});
totalPageCount = pdfResult.pageCount;
+ detectedPdfType = pdfResult.pdfType;
effectivePageCount = maxPages
? Math.min(pdfResult.pageCount, maxPages)
: pdfResult.pageCount;
@@ -624,6 +634,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
fileSizeBytes <= FIRE_PDF_INLINE_HARD_MAX_FILE_SIZE));
const runpodMuUsable =
!forceFirePDF &&
+ !usOnly &&
fileSizeBytes < MAX_FILE_SIZE &&
!!config.RUNPOD_MU_API_KEY &&
!!config.RUNPOD_MU_POD_ID;
@@ -670,7 +681,9 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
config.FIRE_PDF_ENABLE &&
config.FIRE_PDF_BASE_URL &&
fileSizeBytes < FIRE_PDF_MAX_FILE_SIZE &&
- (alwaysFirePdf || Math.random() * 100 < config.FIRE_PDF_PERCENT)));
+ (alwaysFirePdf ||
+ usOnly ||
+ Math.random() * 100 < config.FIRE_PDF_PERCENT)));
if (useFirePDF) {
// Async is a server-controlled cohort within traffic already selected
@@ -735,7 +748,8 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
if (
(!includePageMarkdown && !includeBlocks && !pageMarkers) ||
error instanceof RemoveFeatureError ||
- error instanceof AbortManagerThrownError
+ error instanceof AbortManagerThrownError ||
+ error instanceof UsOnlyUnavailableError
) {
throw error;
}
@@ -783,7 +797,8 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
} catch (error) {
if (
error instanceof RemoveFeatureError ||
- error instanceof AbortManagerThrownError
+ error instanceof AbortManagerThrownError ||
+ error instanceof UsOnlyUnavailableError
) {
throw error;
}
@@ -820,6 +835,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
if (
!result &&
!forceFirePDF &&
+ !usOnly &&
fileSizeBytes >= MAX_FILE_SIZE &&
config.RUNPOD_MU_API_KEY &&
config.RUNPOD_MU_POD_ID
@@ -838,6 +854,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
if (
!result &&
!forceFirePDF &&
+ !usOnly &&
fileSizeBytes < MAX_FILE_SIZE &&
base64Content !== undefined &&
config.RUNPOD_MU_API_KEY &&
@@ -953,6 +970,23 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
if (totalPageCount === undefined && result.pageCount !== undefined) {
totalPageCount = result.pageCount;
}
+ // Without GPU OCR a scanned document has no text layer to return.
+ if (
+ usOnly &&
+ !skipOCR &&
+ (detectedPdfType === "Scanned" ||
+ detectedPdfType === "ImageBased" ||
+ !result.markdown?.trim())
+ ) {
+ meta.logger.warn("US-only PDF needs OCR the PDF service did not run", {
+ method: "scrapePDF",
+ scrape_id: meta.id,
+ pdf_type: detectedPdfType,
+ });
+ throw new UsOnlyUnavailableError(
+ "This PDF needs OCR, which is not available for teams restricted to US-only processing right now. Try again later.",
+ );
+ }
}
return {
diff --git a/apps/api/src/scraper/scrapeURL/transformers/redactPII.test.ts b/apps/api/src/scraper/scrapeURL/transformers/redactPII.test.ts
index 386b2e4a6..60628e9a6 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/redactPII.test.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/redactPII.test.ts
@@ -1,6 +1,7 @@
import type { MockedFunction } from "vitest";
import { performRedactPII } from "./redactPII";
import { redactText } from "../../../lib/fire-privacy-client";
+import { UsOnlyUnavailableError } from "../error";
vi.mock("../../../lib/fire-privacy-client", () => ({
redactText: vi.fn(),
@@ -101,4 +102,36 @@ describe("performRedactPII", () => {
expect(mockedRedactText).not.toHaveBeenCalled();
expect(document.markdown).toBe("");
});
+
+ it("pins redaction to US capacity only for a US-only team", async () => {
+ mockedRedactText.mockResolvedValue({
+ status: "ok",
+ redactedMarkdown: "Hello <PERSON>",
+ spans: [],
+ counts: {},
+ });
+
+ const unpinned = baseMeta();
+ unpinned.internalOptions = { teamFlags: {} };
+ await performRedactPII(unpinned, baseDocument({ markdown: "Hello Alice" }));
+
+ const pinned = baseMeta();
+ pinned.internalOptions = { teamFlags: { usOnly: true } };
+ await performRedactPII(pinned, baseDocument({ markdown: "Hello Alice" }));
+
+ expect(mockedRedactText.mock.calls[0][0].usOnly).toBe(false);
+ expect(mockedRedactText.mock.calls[1][0].usOnly).toBe(true);
+ });
+
+ it("fails the scrape instead of returning unredacted markdown when US capacity refuses", async () => {
+ mockedRedactText.mockRejectedValue(
+ new UsOnlyUnavailableError("no US capacity"),
+ );
+ const meta = baseMeta();
+ meta.internalOptions = { teamFlags: { usOnly: true } };
+
+ await expect(
+ performRedactPII(meta, baseDocument({ markdown: "Hello Alice" })),
+ ).rejects.toBeInstanceOf(UsOnlyUnavailableError);
+ });
});
diff --git a/apps/api/src/scraper/scrapeURL/transformers/redactPII.ts b/apps/api/src/scraper/scrapeURL/transformers/redactPII.ts
index d0e62b3cb..f63e2d096 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/redactPII.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/redactPII.ts
@@ -1,6 +1,7 @@
import { Meta } from "..";
import { Document } from "../../../controllers/v2/types";
import { redactText } from "../../../lib/fire-privacy-client";
+import { isUsOnly } from "../../../lib/us-only";
export async function performRedactPII(
meta: Meta,
@@ -23,6 +24,7 @@ export async function performRedactPII(
// truthy here means it's the options object; falsy was already
// bailed out of above.
options: meta.options.redactPII || undefined,
+ usOnly: isUsOnly(meta.internalOptions?.teamFlags ?? undefined),
});
// Swap raw markdown for the redacted version. Caller asked for PII