Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

feat(api): reservedCapacity team flag for PDF parsing (#5707)

Abimael Martell1183fc786775
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.yml
index 657f4a16c..ead32ad7e 100644
--- a/.github/workflows/test-server.yml
+++ b/.github/workflows/test-server.yml
@@ -394,6 +394,17 @@ jobs:
--reporter=default
working-directory: apps/api
+ - name: Run reserved capacity PDF regression tests
+ if: matrix.engine == 'fetch' && matrix.proxy == 'no-proxy' && matrix.nuq == 'postgres'
+ run: >-
+ pnpm exec vitest run
+ src/scraper/scrapeURL/engines/pdf/__tests__/firePDFReservedCapacity.test.ts
+ src/scraper/scrapeURL/engines/pdf/__tests__/reservedCapacityPdfRouting.test.ts
+ src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts
+ src/scraper/scrapeURL/engines/pdf/__tests__/indexPdfSupersede.test.ts
+ --reporter=default
+ working-directory: apps/api
+
- name: Run US-only page fetch regression tests
if: matrix.engine == 'fetch' && matrix.proxy == 'no-proxy' && matrix.nuq == 'postgres'
run: >-
diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.ts
index 70489f63d..1ab28e811 100644
--- a/apps/api/src/controllers/v1/scrape.ts
+++ b/apps/api/src/controllers/v1/scrape.ts
@@ -372,7 +372,8 @@ async function scrapeControllerInner(
if (
e.code === "unsafe_domain_blocked" ||
- e.code === "US_ONLY_UNAVAILABLE"
+ e.code === "US_ONLY_UNAVAILABLE" ||
+ e.code === "RESERVED_CAPACITY_UNAVAILABLE"
) {
return res.status(403).json({
success: false,
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index 3ad00e9cd..a196161f6 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -1417,6 +1417,9 @@ export type TeamFlags = {
// Every request is processed by US-based vendors only. A route with no US
// vendor is refused or skipped with a warning.
usOnly?: boolean;
+ // PDF and image parsing runs only on reserved processing capacity. A parse
+ // that cannot be kept there is refused.
+ reservedCapacity?: boolean;
// Per-team extensions past a deprecation's sunset (lib/deprecations.ts):
// surface or entry key to the YYYY-MM-DD or UTC timestamp it runs until.
deprecationExtensions?: Record<string, string>;
diff --git a/apps/api/src/controllers/v2/parse.ts b/apps/api/src/controllers/v2/parse.ts
index 839933c80..2140ccfb5 100644
--- a/apps/api/src/controllers/v2/parse.ts
+++ b/apps/api/src/controllers/v2/parse.ts
@@ -652,7 +652,10 @@ export async function parseController(
});
}
- if (e.code === "US_ONLY_UNAVAILABLE") {
+ if (
+ e.code === "US_ONLY_UNAVAILABLE" ||
+ e.code === "RESERVED_CAPACITY_UNAVAILABLE"
+ ) {
setSpanAttributes(span, {
"parse.status_code": 403,
});
diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.ts
index 801987624..31ff53935 100644
--- a/apps/api/src/controllers/v2/scrape.ts
+++ b/apps/api/src/controllers/v2/scrape.ts
@@ -544,7 +544,8 @@ export async function scrapeController(
if (
e.code === "unsafe_domain_blocked" ||
- e.code === "US_ONLY_UNAVAILABLE"
+ e.code === "US_ONLY_UNAVAILABLE" ||
+ e.code === "RESERVED_CAPACITY_UNAVAILABLE"
) {
setSpanAttributes(span, {
"scrape.status_code": 403,
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 7680dc445..bc15c3b0b 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -2192,6 +2192,9 @@ export type TeamFlags = {
// Every request is processed by US-based vendors only. A route with no US
// vendor is refused or skipped with a warning.
usOnly?: boolean;
+ // PDF and image parsing runs only on reserved processing capacity. A parse
+ // that cannot be kept there is refused.
+ reservedCapacity?: boolean;
// Per-team extensions past a deprecation's sunset (lib/deprecations.ts):
// surface or entry key to the YYYY-MM-DD or UTC timestamp it runs until.
deprecationExtensions?: Record<string, string>;
diff --git a/apps/api/src/lib/error-serde.ts b/apps/api/src/lib/error-serde.ts
index 31fb73971..b9a2c3263 100644
--- a/apps/api/src/lib/error-serde.ts
+++ b/apps/api/src/lib/error-serde.ts
@@ -43,6 +43,7 @@ import {
JsonExtractionContentTooLargeError,
XTwitterConfigurationError,
UsOnlyUnavailableError,
+ ReservedCapacityUnavailableError,
ExchangeRefusedError,
} from "../scraper/scrapeURL/error";
import { UnsafeDomainBlockedError } from "./threat-protection/error";
@@ -98,6 +99,7 @@ const errorMap: Record<ErrorCodes, any> = {
SCRAPE_JSON_CONTENT_TOO_LARGE: JsonExtractionContentTooLargeError,
SCRAPE_X_TWITTER_CONFIGURATION_ERROR: XTwitterConfigurationError,
US_ONLY_UNAVAILABLE: UsOnlyUnavailableError,
+ RESERVED_CAPACITY_UNAVAILABLE: ReservedCapacityUnavailableError,
MAP_FAILED: MapFailedError,
CONCURRENCY_QUEUE_TIMEOUT: ConcurrencyQueueTimeoutError,
unsafe_domain_blocked: UnsafeDomainBlockedError,
diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.ts
index f3952e70b..c6fde4732 100644
--- a/apps/api/src/lib/error.ts
+++ b/apps/api/src/lib/error.ts
@@ -44,6 +44,7 @@ export type ErrorCodes =
| "SCRAPE_JSON_CONTENT_TOO_LARGE"
| "SCRAPE_X_TWITTER_CONFIGURATION_ERROR"
| "US_ONLY_UNAVAILABLE"
+ | "RESERVED_CAPACITY_UNAVAILABLE"
| "PARSE_UNSUPPORTED_OPTIONS"
| "CRAWL_DENIAL"
| "UNSUPPORTED_SITE"
diff --git a/apps/api/src/lib/reserved-capacity.ts b/apps/api/src/lib/reserved-capacity.ts
new file mode 100644
index 000000000..bae9880a8
--- /dev/null
+++ b/apps/api/src/lib/reserved-capacity.ts
@@ -0,0 +1,17 @@
+import type { TeamFlags } from "../controllers/v2/types";
+import { isUsOnly } from "./us-only";
+
+/** Whether the team's PDF and image parsing must run on reserved processing capacity. Any value but `true` reads as unrestricted. */
+export function isReservedCapacity(flags: TeamFlags | undefined): boolean {
+ return flags?.reservedCapacity === true;
+}
+
+/**
+ * Whether the team's PDF and image parsing carries a restriction the PDF
+ * service must honor and confirm (US-only or reserved capacity). Such work
+ * runs only on the PDF service: no other parsing engine, and no background
+ * work the service would run without the restriction.
+ */
+export function isProcessingRestricted(flags: TeamFlags | undefined): boolean {
+ return isUsOnly(flags) || isReservedCapacity(flags);
+}
diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
index 053a76eae..eb211f27a 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
@@ -52,6 +52,7 @@ import {
} from "../../../../controllers/v2/types";
import { hasFormatOfType } from "../../../../lib/format-utils";
import { hasCustomRequestContext } from "../../lib/request-context";
+import { isReservedCapacity } from "../../../../lib/reserved-capacity";
export async function sendDocumentToIndex(meta: Meta, document: Document) {
// Skip caching if screenshot format has custom viewport or quality settings
@@ -588,6 +589,19 @@ export async function scrapeURLWithIndex(
throw new IndexMissError();
}
+ // A parsed PDF or OCR'd image may have been produced on other capacity, so
+ // a team restricted to reserved capacity parses it afresh. Raw PDF
+ // bytes were never parsed and are still served.
+ if (
+ isReservedCapacity(meta.internalOptions.teamFlags ?? undefined) &&
+ !isCachedPdfBase64 &&
+ (doc.contentType === "application/pdf" ||
+ doc.contentType?.startsWith("image/"))
+ ) {
+ logLookup("debug", "hit", { reservedCapacityMismatch: true });
+ throw new IndexMissError();
+ }
+
// fire-pdf may have re-parsed this PDF since it was indexed (a refresh or a
// campaign). Only once a fresh entry exists is the index copy skipped, so
// the request lands on that entry without a parse and is re-indexed.
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFBackgroundRefresh.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFBackgroundRefresh.test.ts
index 6f5d92113..5f6801657 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFBackgroundRefresh.test.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFBackgroundRefresh.test.ts
@@ -320,4 +320,28 @@ describe("FirePDF cache background refresh", () => {
expect(jobsFetch).not.toHaveBeenCalled();
expect(await refreshCount("skipped_us_only")).toBe(1);
});
+
+ it("serves a team restricted to reserved capacity the stale entry without offering or submitting a refresh", async () => {
+ const meta = makeMeta();
+ meta.internalOptions.teamFlags = { reservedCapacity: true };
+ answerWith(staleAnswer(true));
+ expect(await lookupCachedResult(meta, base64, lookupArgs)).toEqual(
+ staleResult,
+ );
+ await settle();
+ expect(lookupFetch.mock.calls[0][0].body).not.toHaveProperty(
+ "background_refresh",
+ );
+ expect(
+ await submitBackgroundRefresh(meta, base64, {
+ mode: undefined,
+ includePageMarkdown: false,
+ includeBlocks: false,
+ pageMarkers: false,
+ sourceKind: "pdf",
+ }),
+ ).toBe("skipped_reserved_capacity");
+ expect(jobsFetch).not.toHaveBeenCalled();
+ expect(await refreshCount("skipped_reserved_capacity")).toBe(1);
+ });
});
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFReservedCapacity.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFReservedCapacity.test.ts
new file mode 100644
index 000000000..8e1928b3a
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFReservedCapacity.test.ts
@@ -0,0 +1,628 @@
+// Stub the GCS cache so the async client never reaches real cloud storage.
+vi.mock("../../../../../lib/gcs-pdf-cache", () => ({
+ pdfCacheConfigured: vi.fn(() => true),
+ createPdfCacheKey: (s: string) => `sha-${s.length}`,
+ resolvePdfCacheKey: (input: string | { key: string }) =>
+ typeof input === "string" ? `sha-${input.length}` : input.key,
+ getPdfResultFromCache: vi.fn(async () => null),
+ savePdfResultToCache: vi.fn(async () => null),
+}));
+vi.mock("../../../lib/fetch", () => ({
+ robustFetch: vi.fn(),
+}));
+
+import { robustFetch } from "../../../lib/fetch";
+import { config } from "../../../../../config";
+import {
+ getPdfResultFromCache,
+ savePdfResultToCache,
+} from "../../../../../lib/gcs-pdf-cache";
+import {
+ ReservedCapacityUnavailableError,
+ UsOnlyUnavailableError,
+} from "../../../error";
+import { scrapePDFWithFirePDF } from "../firePDF";
+import { tryGetCached } from "../fire-pdf/cache";
+import { scrapePDFWithFirePDFAsync } from "../fire-pdf/async";
+import { lookupAdoptableFirePdfJob } from "../fire-pdf/lookup";
+import { buildFirePdfJobOptions, FirePdfAsyncFailure } from "../fire-pdf/utils";
+import {
+ makeFetchFromSequence,
+ makeMeta as makeAsyncMeta,
+ noopSleep,
+} from "./firePDFAsyncFixtures";
+
+const mockedRobustFetch = vi.mocked(robustFetch);
+const mockedGetCached = vi.mocked(getPdfResultFromCache);
+
+const reservedFlags = { reservedCapacity: true };
+const bothFlags = { usOnly: true, reservedCapacity: true };
+
+function makeSyncMeta(teamFlags?: Record<string, unknown>) {
+ const logger: any = {
+ info: vi.fn(),
+ warn: vi.fn(),
+ error: vi.fn(),
+ debug: vi.fn(),
+ };
+ return {
+ id: "reserved-capacity-sync-test",
+ url: "https://example.com/file.pdf",
+ rewrittenUrl: undefined,
+ options: {},
+ logger,
+ mock: null,
+ abort: {
+ throwIfAborted: vi.fn(),
+ asSignal: vi.fn(() => new AbortController().signal),
+ scrapeTimeout: vi.fn(() => 60_000),
+ },
+ internalOptions: {
+ zeroDataRetention: false,
+ teamId: "team-test",
+ ...(teamFlags && { teamFlags }),
+ },
+ } as any;
+}
+
+function makeReservedAsyncMeta(teamFlags?: Record<string, unknown>) {
+ return makeAsyncMeta({
+ internalOptions: {
+ zeroDataRetention: false,
+ teamId: "team-x",
+ teamConcurrency: 12,
+ crawlId: undefined,
+ ...(teamFlags && { teamFlags }),
+ },
+ });
+}
+
+function failureStatus(status: number, body: unknown) {
+ return new Error("Request sent failure status", {
+ cause: { response: { status, body: JSON.stringify(body) } },
+ });
+}
+
+const syncResult = (extra: Record<string, unknown>) =>
+ ({
+ markdown: "Document",
+ failed_pages: null,
+ pages_processed: 1,
+ ...extra,
+ }) as any;
+
+// The PDF service's sync answer when the reserved capacity it needed was refused.
+const reservedNoCapacity = {
+ error: "No OCR pods available",
+ reserved_refusals: { reserved_unroutable: 3 },
+};
+
+const ORIGINAL_BASE_URL = config.FIRE_PDF_BASE_URL;
+
+beforeAll(() => {
+ config.FIRE_PDF_BASE_URL = "http://fire-pdf.test";
+});
+
+afterAll(() => {
+ config.FIRE_PDF_BASE_URL = ORIGINAL_BASE_URL;
+});
+
+beforeEach(() => {
+ vi.clearAllMocks();
+});
+
+describe("FirePDF sync parse for teams restricted to reserved capacity", () => {
+ it("sends reserved only for a restricted team", async () => {
+ mockedRobustFetch.mockResolvedValue(syncResult({ reserved_applied: true }));
+
+ await scrapePDFWithFirePDF(makeSyncMeta(), "BASE64", 1);
+ await scrapePDFWithFirePDF(makeSyncMeta(reservedFlags), "BASE64", 1);
+
+ const unrestricted = mockedRobustFetch.mock.calls[0][0].body;
+ const restricted = mockedRobustFetch.mock.calls[1][0].body;
+ expect(Object.hasOwn(unrestricted, "reserved")).toBe(false);
+ expect(restricted.reserved).toBe(true);
+ expect(Object.hasOwn(restricted, "data_region")).toBe(false);
+ });
+
+ it("returns a restricted result that carries the reserved echo", async () => {
+ mockedRobustFetch.mockResolvedValue(syncResult({ reserved_applied: true }));
+
+ const result = await scrapePDFWithFirePDF(
+ makeSyncMeta(reservedFlags),
+ "BASE64",
+ 1,
+ );
+
+ expect(result.markdown).toBe("Document");
+ });
+
+ it("refuses a restricted parse the PDF service could not place", async () => {
+ mockedRobustFetch.mockRejectedValue(failureStatus(503, reservedNoCapacity));
+
+ const error = await scrapePDFWithFirePDF(
+ makeSyncMeta(reservedFlags),
+ "BASE64",
+ 1,
+ ).catch(e => e);
+
+ expect(error).toBeInstanceOf(ReservedCapacityUnavailableError);
+ expect(error.code).toBe("RESERVED_CAPACITY_UNAVAILABLE");
+ expect(error.message).toContain(
+ "PDF processing is unavailable for this team's configuration. Please retry later.",
+ );
+ });
+
+ it("keeps a restricted failure without reserved refusals on its own error", async () => {
+ const failure = failureStatus(502, { error: "All 3 pages failed" });
+ mockedRobustFetch.mockRejectedValue(failure);
+
+ const error = await scrapePDFWithFirePDF(
+ makeSyncMeta(reservedFlags),
+ "BASE64",
+ 1,
+ ).catch(e => e);
+
+ expect(error).toBe(failure);
+ });
+
+ it.each([
+ ["missing", {}],
+ ["null", { reserved_applied: null }],
+ ["false", { reserved_applied: false }],
+ ])(
+ "refuses a restricted result whose reserved echo is %s",
+ async (_name, echo) => {
+ mockedRobustFetch.mockResolvedValue(syncResult(echo));
+
+ const error = await scrapePDFWithFirePDF(
+ makeSyncMeta(reservedFlags),
+ "BASE64",
+ 1,
+ ).catch(e => e);
+
+ expect(error).toBeInstanceOf(ReservedCapacityUnavailableError);
+ expect(error.message).toContain(
+ "PDF processing is unavailable for this team's configuration. Please retry later.",
+ );
+ },
+ );
+
+ it("uses the same error for a restricted image", async () => {
+ mockedRobustFetch.mockRejectedValue(failureStatus(503, reservedNoCapacity));
+
+ const error = await scrapePDFWithFirePDF(
+ makeSyncMeta(reservedFlags),
+ "BASE64",
+ 1,
+ 1,
+ "ocr",
+ false,
+ false,
+ false,
+ "image",
+ ).catch(e => e);
+
+ expect(error).toBeInstanceOf(ReservedCapacityUnavailableError);
+ expect(error.message).toContain(
+ "PDF processing is unavailable for this team's configuration. Please retry later.",
+ );
+ });
+
+ it("keeps an unrestricted failure and an unrestricted result without the echo unchanged", async () => {
+ const failure = failureStatus(503, reservedNoCapacity);
+ mockedRobustFetch.mockRejectedValueOnce(failure);
+ expect(
+ await scrapePDFWithFirePDF(makeSyncMeta(), "BASE64", 1).catch(e => e),
+ ).toBe(failure);
+
+ mockedRobustFetch.mockResolvedValueOnce(syncResult({}));
+ const result = await scrapePDFWithFirePDF(makeSyncMeta(), "BASE64", 1);
+ expect(result.markdown).toBe("Document");
+ });
+
+ describe("with US-only as well", () => {
+ it("sends both fields and returns a result carrying both echoes", async () => {
+ mockedRobustFetch.mockResolvedValue(
+ syncResult({ data_region_applied: "us", reserved_applied: true }),
+ );
+
+ const result = await scrapePDFWithFirePDF(
+ makeSyncMeta(bothFlags),
+ "BASE64",
+ 1,
+ );
+
+ const body = mockedRobustFetch.mock.calls[0][0].body;
+ expect(body.data_region).toBe("us");
+ expect(body.reserved).toBe(true);
+ expect(result.markdown).toBe("Document");
+ });
+
+ it.each([
+ ["the region echo", { reserved_applied: true }, UsOnlyUnavailableError],
+ [
+ "the reserved echo",
+ { data_region_applied: "us" },
+ ReservedCapacityUnavailableError,
+ ],
+ ])("refuses a result missing %s", async (_name, echo, errorClass) => {
+ mockedRobustFetch.mockResolvedValue(syncResult(echo));
+
+ const error = await scrapePDFWithFirePDF(
+ makeSyncMeta(bothFlags),
+ "BASE64",
+ 1,
+ ).catch(e => e);
+
+ expect(error).toBeInstanceOf(errorClass);
+ });
+
+ it("keeps the US-only error when both restrictions were refused", async () => {
+ mockedRobustFetch.mockRejectedValue(
+ failureStatus(503, {
+ ...reservedNoCapacity,
+ data_region_refusals: { region_unroutable: 1 },
+ }),
+ );
+
+ const error = await scrapePDFWithFirePDF(
+ makeSyncMeta(bothFlags),
+ "BASE64",
+ 1,
+ ).catch(e => e);
+
+ expect(error).toBeInstanceOf(UsOnlyUnavailableError);
+ });
+
+ it("maps a reserved-only refusal to the reserved capacity error", async () => {
+ mockedRobustFetch.mockRejectedValue(
+ failureStatus(503, reservedNoCapacity),
+ );
+
+ const error = await scrapePDFWithFirePDF(
+ makeSyncMeta(bothFlags),
+ "BASE64",
+ 1,
+ ).catch(e => e);
+
+ expect(error).toBeInstanceOf(ReservedCapacityUnavailableError);
+ });
+ });
+});
+
+const submitAccepted = {
+ matchUrl: /\/jobs$/,
+ matchMethod: "POST" as const,
+ response: {
+ status: 202,
+ body: { scrape_id: "scrape-id-test", status: "queued", lane: "fast" },
+ },
+};
+const pollDone = {
+ matchUrl: /\/jobs\/scrape-id-test$/,
+ matchMethod: "GET" as const,
+ response: {
+ status: 200,
+ body: { scrape_id: "scrape-id-test", status: "done", pages_processed: 3 },
+ },
+};
+function resultWith(extra: Record<string, unknown>) {
+ return {
+ matchUrl: /\/jobs\/scrape-id-test\/result$/,
+ matchMethod: "GET" as const,
+ response: {
+ status: 200,
+ body: {
+ schema_version: 1,
+ markdown: "# Async",
+ pages_processed: 3,
+ failed_pages: null,
+ partial_pages: null,
+ ...extra,
+ },
+ },
+ };
+}
+function pollFailed(errorMessage: string) {
+ return {
+ matchUrl: /\/jobs\/scrape-id-test$/,
+ matchMethod: "GET" as const,
+ response: {
+ status: 502,
+ body: {
+ scrape_id: "scrape-id-test",
+ reserved_applied: true,
+ status: "failed",
+ error_class: "ocr_unavailable",
+ error_message: errorMessage,
+ },
+ },
+ };
+}
+
+function runAsync(meta: any, fetchImpl: any) {
+ return scrapePDFWithFirePDFAsync(
+ meta,
+ "BASE64",
+ undefined,
+ undefined,
+ undefined,
+ { fetchImpl, fallbackImpl: vi.fn(), sleepImpl: noopSleep },
+ );
+}
+
+describe("FirePDF async jobs for teams restricted to reserved capacity", () => {
+ it("sends reserved in the job options only for a restricted team", async () => {
+ const unrestricted = makeFetchFromSequence([
+ submitAccepted,
+ pollDone,
+ resultWith({ reserved_applied: null }),
+ ]);
+ await runAsync(makeReservedAsyncMeta(), unrestricted.fetchImpl);
+
+ const restricted = makeFetchFromSequence([
+ submitAccepted,
+ pollDone,
+ resultWith({ reserved_applied: true }),
+ ]);
+ const result = await runAsync(
+ makeReservedAsyncMeta(reservedFlags),
+ restricted.fetchImpl,
+ );
+
+ expect(unrestricted.calls[0].body).not.toHaveProperty("reserved");
+ expect(unrestricted.calls[0].body).not.toHaveProperty("options.reserved");
+ expect(restricted.calls[0].body).not.toHaveProperty("reserved");
+ expect(restricted.calls[0].body).toHaveProperty("options.reserved", true);
+ expect(restricted.calls[0].body).not.toHaveProperty("options.data_region");
+ expect(result.markdown).toBe("# Async");
+ });
+
+ it("refuses a restricted job that failed for lack of reserved capacity, without cancelling it", async () => {
+ const { fetchImpl, calls } = makeFetchFromSequence([
+ submitAccepted,
+ pollFailed(
+ "No OCR pods available [reserved refused: reserved_unroutable=4]",
+ ),
+ ]);
+
+ const error = await runAsync(
+ makeReservedAsyncMeta(reservedFlags),
+ fetchImpl,
+ ).catch(e => e);
+
+ expect(error).toBeInstanceOf(ReservedCapacityUnavailableError);
+ expect(calls.map(call => call.method)).toEqual(["POST", "GET"]);
+ });
+
+ it("keeps a restricted job failure without reserved refusals off the restriction error", async () => {
+ const { fetchImpl } = makeFetchFromSequence([
+ submitAccepted,
+ pollFailed("Batch render failed"),
+ ]);
+
+ const error = await runAsync(
+ makeReservedAsyncMeta(reservedFlags),
+ fetchImpl,
+ ).catch(e => e);
+
+ expect(error).toBeInstanceOf(FirePdfAsyncFailure);
+ expect(error.reason).toBe("terminal_failed");
+ });
+
+ it.each([
+ ["missing", {}],
+ ["null", { reserved_applied: null }],
+ ["false", { reserved_applied: false }],
+ ])(
+ "refuses a restricted result whose reserved echo is %s and caches nothing",
+ async (_name, echo) => {
+ const { fetchImpl } = makeFetchFromSequence([
+ submitAccepted,
+ pollDone,
+ resultWith(echo),
+ ]);
+
+ const error = await runAsync(
+ makeReservedAsyncMeta(reservedFlags),
+ fetchImpl,
+ ).catch(e => e);
+
+ expect(error).toBeInstanceOf(ReservedCapacityUnavailableError);
+ expect(savePdfResultToCache).not.toHaveBeenCalled();
+ },
+ );
+
+ describe("by reference", () => {
+ const byReference = {
+ gcsUri: "gs://bucket/inputs/scrape-id-test.pdf",
+ sha256: "ab".repeat(32),
+ sizeBytes: 200 * 1024 * 1024,
+ };
+ const runByReference = (meta: any, fetchImpl: any) =>
+ scrapePDFWithFirePDFAsync(meta, byReference, undefined, 3, undefined, {
+ fetchImpl,
+ fallbackImpl: vi.fn(),
+ sleepImpl: noopSleep,
+ });
+
+ it("sends reserved in the job options and accepts the reserved echo", async () => {
+ const { fetchImpl, calls } = makeFetchFromSequence([
+ submitAccepted,
+ pollDone,
+ resultWith({ reserved_applied: true }),
+ ]);
+
+ const result = await runByReference(
+ makeReservedAsyncMeta(reservedFlags),
+ fetchImpl,
+ );
+
+ expect(calls[0].body).toHaveProperty("input_gcs_uri", byReference.gcsUri);
+ expect(calls[0].body).toHaveProperty("options.reserved", true);
+ expect(result.markdown).toBe("# Async");
+ });
+
+ it("refuses a result without the reserved echo and caches nothing", async () => {
+ const { fetchImpl } = makeFetchFromSequence([
+ submitAccepted,
+ pollDone,
+ resultWith({}),
+ ]);
+
+ const error = await runByReference(
+ makeReservedAsyncMeta(reservedFlags),
+ fetchImpl,
+ ).catch(e => e);
+
+ expect(error).toBeInstanceOf(ReservedCapacityUnavailableError);
+ expect(savePdfResultToCache).not.toHaveBeenCalled();
+ });
+ });
+
+ it("sends both restrictions and requires both echoes for a team with both flags", async () => {
+ const both = makeFetchFromSequence([
+ submitAccepted,
+ pollDone,
+ resultWith({ data_region_applied: "us", reserved_applied: true }),
+ ]);
+ const result = await runAsync(
+ makeReservedAsyncMeta(bothFlags),
+ both.fetchImpl,
+ );
+ expect(both.calls[0].body).toHaveProperty("options.data_region", "us");
+ expect(both.calls[0].body).toHaveProperty("options.reserved", true);
+ expect(result.markdown).toBe("# Async");
+
+ const regionOnly = makeFetchFromSequence([
+ submitAccepted,
+ pollDone,
+ resultWith({ data_region_applied: "us" }),
+ ]);
+ const error = await runAsync(
+ makeReservedAsyncMeta(bothFlags),
+ regionOnly.fetchImpl,
+ ).catch(e => e);
+ expect(error).toBeInstanceOf(ReservedCapacityUnavailableError);
+ });
+
+ it("forwards reserved from the caller's job options to the content lookup", async () => {
+ const lookupHit = {
+ matchUrl: /\/jobs\/lookup$/,
+ matchMethod: "POST" as const,
+ response: {
+ status: 200,
+ body: { scrape_id: "found-1", status: "running" },
+ },
+ };
+ const restricted = makeFetchFromSequence([lookupHit]);
+
+ await lookupAdoptableFirePdfJob(
+ makeReservedAsyncMeta(reservedFlags),
+ "ef".repeat(32),
+ buildFirePdfJobOptions({
+ maxPages: undefined,
+ pagesProcessed: undefined,
+ mode: "auto",
+ includePageMarkdown: false,
+ includeBlocks: false,
+ pageMarkers: false,
+ tableFormat: undefined,
+ trainingCapture: undefined,
+ dataRegion: undefined,
+ reserved: true,
+ }),
+ restricted.fetchImpl,
+ );
+
+ expect(restricted.calls[0].body).not.toHaveProperty("reserved");
+ expect(restricted.calls[0].body).toHaveProperty("options", {
+ mode: "auto",
+ reserved: true,
+ });
+ });
+});
+
+describe("FirePDF content cache for teams restricted to reserved capacity", () => {
+ const cachedEntry = { markdown: "cached", html: "<p>cached</p>" } as any;
+
+ afterEach(() => {
+ config.FIRE_PDF_CACHE_BASE_URL = undefined;
+ });
+
+ it("never reads the cache for a restricted team but still writes its result", async () => {
+ mockedGetCached.mockResolvedValue(cachedEntry);
+ mockedRobustFetch.mockResolvedValue(syncResult({ reserved_applied: true }));
+
+ const result = await scrapePDFWithFirePDF(
+ makeSyncMeta(reservedFlags),
+ "BASE64",
+ );
+
+ expect(result.markdown).toBe("Document");
+ expect(mockedGetCached).not.toHaveBeenCalled();
+ expect(mockedRobustFetch).toHaveBeenCalledTimes(1);
+ expect(savePdfResultToCache).toHaveBeenCalled();
+ });
+
+ it.each([
+ ["an unrestricted team", undefined],
+ ["a US-only team", { usOnly: true }],
+ ])("still serves %s from the cache", async (_name, flags) => {
+ mockedGetCached.mockResolvedValue(cachedEntry);
+
+ const result = await scrapePDFWithFirePDF(makeSyncMeta(flags), "BASE64");
+
+ expect(result.markdown).toBe("cached");
+ expect(mockedRobustFetch).not.toHaveBeenCalled();
+ });
+
+ it("skips the cache service lookup for a restricted team", async () => {
+ config.FIRE_PDF_CACHE_BASE_URL = "http://cache.test";
+ mockedRobustFetch.mockResolvedValue(syncResult({ reserved_applied: true }));
+
+ await scrapePDFWithFirePDF(makeSyncMeta(reservedFlags), "BASE64");
+
+ const urls = mockedRobustFetch.mock.calls.map(call => call[0].url);
+ expect(urls).toEqual(["http://fire-pdf.test/ocr"]);
+ });
+
+ it("still records the raw cache key for the URL index without a lookup", async () => {
+ config.FIRE_PDF_CACHE_BASE_URL = "http://cache.test";
+ const meta = { ...makeSyncMeta(reservedFlags), firePdfCacheKey: {} };
+ const rawKey = `raw-${"cd".repeat(32)}`;
+
+ const cached = await tryGetCached(
+ meta,
+ { key: rawKey },
+ "auto",
+ undefined,
+ 3,
+ false,
+ false,
+ );
+
+ expect(cached).toBeNull();
+ expect(meta.firePdfCacheKey).toEqual({ current: rawKey });
+ expect(mockedRobustFetch).not.toHaveBeenCalled();
+ });
+
+ it("skips the content cache read on async jobs for a restricted team", async () => {
+ mockedGetCached.mockResolvedValue(cachedEntry);
+ const { fetchImpl, calls } = makeFetchFromSequence([
+ submitAccepted,
+ pollDone,
+ resultWith({ reserved_applied: true }),
+ ]);
+
+ const result = await runAsync(
+ makeReservedAsyncMeta(reservedFlags),
+ fetchImpl,
+ );
+
+ expect(result.markdown).toBe("# Async");
+ expect(mockedGetCached).not.toHaveBeenCalled();
+ expect(calls[0].method).toBe("POST");
+ });
+});
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPdfSupersede.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPdfSupersede.test.ts
index c102ec137..422b8793d 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPdfSupersede.test.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPdfSupersede.test.ts
@@ -40,7 +40,7 @@ const mutable = config as unknown as MutableConfig;
const createdAt = "2026-10-01T00:00:00.000Z";
const rawKey = `raw-${"b".repeat(64)}`;
-function meta() {
+function meta(teamFlags?: Record<string, unknown>) {
return {
id: "index-supersede",
url: "https://example.com/doc.pdf",
@@ -51,7 +51,7 @@ function meta() {
error: vi.fn(),
child: () => ({ info: vi.fn(), warn: vi.fn(), error: vi.fn() }),
},
- internalOptions: { teamId: "team-1" },
+ internalOptions: { teamId: "team-1", ...(teamFlags && { teamFlags }) },
featureFlags: new Set(),
options: {
maxAge: 7 * 24 * 60 * 60 * 1000,
@@ -189,3 +189,59 @@ describe("URL index PDF supersede check", () => {
expect(checkFirePdfSupersedes).not.toHaveBeenCalled();
});
});
+
+describe("URL index for teams restricted to reserved capacity", () => {
+ const reserved = { reservedCapacity: true };
+
+ beforeEach(() => {
+ vi.mocked(indexGetRecent5).mockResolvedValue([
+ {
+ id: "row-1",
+ created_at: createdAt,
+ status: 200,
+ has_screenshot: false,
+ has_screenshot_fullscreen: false,
+ wait_time_ms: null,
+ },
+ ] as any);
+ vi.mocked(getCachedIndexEntries).mockResolvedValue({ status: "miss" });
+ });
+
+ it("misses a parsed PDF entry for a restricted team", async () => {
+ vi.mocked(getIndexFromGCS).mockResolvedValue(indexedPdf());
+ await expect(scrapeURLWithIndex(meta(reserved))).rejects.toBeInstanceOf(
+ IndexMissError,
+ );
+ });
+
+ it("misses an OCR'd image entry for a restricted team", async () => {
+ vi.mocked(getIndexFromGCS).mockResolvedValue(
+ indexedPdf({ contentType: "image/png", pdfMetadata: undefined }),
+ );
+ const restricted = { ...meta(reserved), imageOcrEnabled: async () => true };
+ await expect(scrapeURLWithIndex(restricted)).rejects.toBeInstanceOf(
+ IndexMissError,
+ );
+ });
+
+ it.each([
+ ["an unrestricted team", undefined],
+ ["a US-only team", { usOnly: true }],
+ ])("still serves a parsed PDF entry to %s", async (_name, flags) => {
+ vi.mocked(getIndexFromGCS).mockResolvedValue(indexedPdf());
+ const served = await scrapeURLWithIndex(meta(flags));
+ expect(served.html).toBe("<p>old parse</p>");
+ });
+
+ it("still serves a non-PDF entry to a restricted team", async () => {
+ vi.mocked(getIndexFromGCS).mockResolvedValue(
+ indexedPdf({
+ url: "https://example.com/page",
+ contentType: "text/html",
+ pdfMetadata: undefined,
+ }),
+ );
+ const served = await scrapeURLWithIndex(meta(reserved));
+ expect(served.html).toBe("<p>old parse</p>");
+ });
+});
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/reservedCapacityPdfRouting.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/reservedCapacityPdfRouting.test.ts
new file mode 100644
index 000000000..1923311c1
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/reservedCapacityPdfRouting.test.ts
@@ -0,0 +1,239 @@
+import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { processPdf } from "@mendable/firecrawl-rs";
+import { config } from "../../../../../config";
+import {
+ ReservedCapacityUnavailableError,
+ UsOnlyUnavailableError,
+} from "../../../error";
+import { scrapePDFWithFirePDF } from "../firePDF";
+import { scrapePDFWithParsePDF } from "../pdfParse";
+import { scrapePDFWithRunPodMU } from "../runpodMU";
+import { mineruDiverted } from "../fire-pdf/by-reference";
+import { scrapePDF } from "../index";
+
+vi.mock("@mendable/firecrawl-rs", () => ({
+ processPdf: vi.fn(),
+ detectPdf: vi.fn(),
+}));
+
+vi.mock("../firePDF", async importOriginal => ({
+ ...(await importOriginal<typeof import("../firePDF")>()),
+ scrapePDFWithFirePDF: vi.fn(),
+}));
+
+vi.mock("../pdfParse", () => ({
+ scrapePDFWithParsePDF: vi.fn(),
+}));
+
+vi.mock("../runpodMU", () => ({
+ scrapePDFWithRunPodMU: vi.fn(),
+}));
+
+vi.mock("../markdownToHtml", () => ({
+ safeMarkdownToHtml: vi.fn(async (markdown: string) => `<p>${markdown}</p>`),
+}));
+
+const mockedProcessPdf = vi.mocked(processPdf);
+const mockedFirePdf = vi.mocked(scrapePDFWithFirePDF);
+const mockedPdfParse = vi.mocked(scrapePDFWithParsePDF);
+const mockedRunPodMU = vi.mocked(scrapePDFWithRunPodMU);
+
+const configKeys = [
+ "PDF_RUST_EXTRACT_ENABLE",
+ "FIRE_PDF_ENABLE",
+ "FIRE_PDF_BASE_URL",
+ "FIRE_PDF_PERCENT",
+ "FIRE_PDF_ASYNC_PERCENT",
+ "MINERU_PERCENT",
+ "RUNPOD_MU_API_KEY",
+ "RUNPOD_MU_POD_ID",
+ "PARSE_PDF_ALWAYS_FIRE_PDF",
+] as const;
+const originalConfig = Object.fromEntries(
+ configKeys.map(key => [key, config[key]]),
+);
+
+function setConfig(
+ overrides: Partial<Record<(typeof configKeys)[number], unknown>>,
+) {
+ Object.assign(config as any, overrides);
+}
+
+const tempDirs: string[] = [];
+
+async function makeMeta(teamFlags?: Record<string, unknown>) {
+ const dir = await mkdtemp(path.join(tmpdir(), "reserved-capacity-pdf-"));
+ tempDirs.push(dir);
+ const filePath = path.join(dir, "doc.pdf");
+ await writeFile(filePath, "%PDF-1.7\n%test\n");
+ const logger: any = {
+ info: vi.fn(),
+ warn: vi.fn(),
+ error: vi.fn(),
+ debug: vi.fn(),
+ };
+ logger.child = vi.fn(() => logger);
+ return {
+ id: "01a00000-0000-7000-8000-000000000001",
+ url: "https://example.com/doc.pdf",
+ options: { parsers: ["pdf"] },
+ logger,
+ featureFlags: new Set(["pdf"]),
+ pdfPrefetch: {
+ filePath,
+ url: "https://example.com/doc.pdf",
+ status: 200,
+ proxyUsed: "basic",
+ },
+ abort: {
+ throwIfAborted: vi.fn(),
+ asSignal: vi.fn(() => new AbortController().signal),
+ scrapeTimeout: vi.fn(() => 60_000),
+ },
+ internalOptions: {
+ teamId: "team-test",
+ ...(teamFlags && { teamFlags }),
+ },
+ } as any;
+}
+
+function nativeResult(pdfType: string, markdown: string) {
+ return {
+ pdfType,
+ confidence: 0.99,
+ isComplex: true,
+ pageCount: 1,
+ markdown,
+ title: null,
+ pagesNeedingOcr: [],
+ logs: [],
+ } as any;
+}
+
+const reserved = { reservedCapacity: true };
+
+describe("PDF routing for teams restricted to reserved capacity", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ setConfig({
+ PDF_RUST_EXTRACT_ENABLE: true,
+ FIRE_PDF_ENABLE: true,
+ FIRE_PDF_BASE_URL: "http://fire-pdf.test",
+ FIRE_PDF_PERCENT: 100,
+ FIRE_PDF_ASYNC_PERCENT: 0,
+ MINERU_PERCENT: 0,
+ RUNPOD_MU_API_KEY: "test-key",
+ RUNPOD_MU_POD_ID: "test-endpoint",
+ PARSE_PDF_ALWAYS_FIRE_PDF: false,
+ });
+ mockedProcessPdf.mockResolvedValue(nativeResult("TextBased", "text layer"));
+ mockedFirePdf.mockResolvedValue({
+ markdown: "fire-pdf",
+ html: "<p>fire-pdf</p>",
+ pagesProcessed: 1,
+ });
+ mockedPdfParse.mockResolvedValue({
+ markdown: "pdf-parse",
+ html: "<p>pdf-parse</p>",
+ });
+ mockedRunPodMU.mockResolvedValue({
+ markdown: "alternate",
+ html: "<p>alternate</p>",
+ });
+ });
+
+ afterEach(async () => {
+ await Promise.all(
+ tempDirs.splice(0).map(dir => rm(dir, { recursive: true, force: true })),
+ );
+ });
+
+ afterAll(() => {
+ Object.assign(config as any, originalConfig);
+ });
+
+ it("never diverts a restricted team to the alternate PDF engine", async () => {
+ setConfig({ MINERU_PERCENT: 100 });
+
+ expect(mineruDiverted(await makeMeta())).toBe(true);
+ expect(mineruDiverted(await makeMeta(reserved))).toBe(false);
+
+ const result = await scrapePDF(await makeMeta(reserved));
+
+ expect(result.markdown).toBe("fire-pdf");
+ expect(mockedFirePdf).toHaveBeenCalledTimes(1);
+ expect(mockedRunPodMU).not.toHaveBeenCalled();
+ });
+
+ it("sends a restricted PDF to the PDF service regardless of FIRE_PDF_PERCENT", async () => {
+ setConfig({ FIRE_PDF_PERCENT: 0, RUNPOD_MU_API_KEY: undefined });
+
+ expect((await scrapePDF(await makeMeta())).markdown).toBe("pdf-parse");
+ expect(mockedFirePdf).not.toHaveBeenCalled();
+
+ expect((await scrapePDF(await makeMeta(reserved))).markdown).toBe(
+ "fire-pdf",
+ );
+ expect(mockedFirePdf).toHaveBeenCalledTimes(1);
+ });
+
+ it("fails a restricted scrape the PDF service refuses, with no fallback", async () => {
+ mockedFirePdf.mockRejectedValueOnce(
+ new ReservedCapacityUnavailableError("no reserved capacity"),
+ );
+
+ const error = await scrapePDF(await makeMeta(reserved)).catch(e => e);
+
+ expect(error).toBeInstanceOf(ReservedCapacityUnavailableError);
+ expect(mockedRunPodMU).not.toHaveBeenCalled();
+ expect(mockedPdfParse).not.toHaveBeenCalled();
+ });
+
+ it("skips the alternate engine fallback but keeps the local pdf-parse fallback for a restricted team", async () => {
+ mockedFirePdf.mockRejectedValue(new Error("fire-pdf 500"));
+
+ expect((await scrapePDF(await makeMeta())).markdown).toBe("alternate");
+ expect(mockedRunPodMU).toHaveBeenCalledTimes(1);
+
+ mockedRunPodMU.mockClear();
+ expect((await scrapePDF(await makeMeta(reserved))).markdown).toBe(
+ "pdf-parse",
+ );
+ expect(mockedRunPodMU).not.toHaveBeenCalled();
+ });
+
+ it.each([
+ ["a scanned PDF", "Scanned", "page header"],
+ ["a PDF with no text layer", "Mixed", " "],
+ ])(
+ "refuses %s for a restricted team instead of returning it without OCR",
+ async (_name, pdfType, text) => {
+ mockedProcessPdf.mockResolvedValue(nativeResult(pdfType, ""));
+ mockedFirePdf.mockRejectedValue(new Error("fire-pdf 500"));
+ mockedPdfParse.mockResolvedValue({ markdown: text, html: text });
+
+ const error = await scrapePDF(await makeMeta(reserved)).catch(e => e);
+
+ expect(error).toBeInstanceOf(ReservedCapacityUnavailableError);
+ expect(error.message).toContain(
+ "PDF processing is unavailable for this team's configuration. Please retry later.",
+ );
+ expect(mockedRunPodMU).not.toHaveBeenCalled();
+ },
+ );
+
+ it("keeps the US-only OCR error for a team with both restrictions", async () => {
+ mockedProcessPdf.mockResolvedValue(nativeResult("Scanned", ""));
+ mockedFirePdf.mockRejectedValue(new Error("fire-pdf 500"));
+ mockedPdfParse.mockResolvedValue({ markdown: "", html: "" });
+
+ const error = await scrapePDF(
+ await makeMeta({ usOnly: true, reservedCapacity: true }),
+ ).catch(e => e);
+
+ expect(error).toBeInstanceOf(UsOnlyUnavailableError);
+ expect(mockedRunPodMU).not.toHaveBeenCalled();
+ });
+});
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts
index 0d7f51ef4..b6423c0a8 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts
@@ -10,7 +10,11 @@ import { cancelJob } from "./cancel";
import { maybeSaveResult, provenanceFromResponse, tryGetCached } from "./cache";
import { resolvePdfCacheKey } from "../../../../../lib/gcs-pdf-cache";
import { isUsOnly, US_DATA_REGION } from "../../../../../lib/us-only";
-import { firePdfUsOnlyRefusal } from "./data-region";
+import { isReservedCapacity } from "../../../../../lib/reserved-capacity";
+import {
+ firePdfReservedCapacityRefusal,
+ firePdfUsOnlyRefusal,
+} from "./data-region";
import {
firePdfAsyncAbandonedTotal,
firePdfAsyncTotalDurationSeconds,
@@ -330,7 +334,8 @@ export async function scrapePDFWithFirePDFAsync(
(error.reason === "terminal_failed" ||
error.reason === "terminal_expired" ||
error.reason === "terminal_cancelled" ||
- error.reason === "region_unavailable");
+ error.reason === "region_unavailable" ||
+ error.reason === "reserved_unavailable");
if ((jobAlreadyTerminal || terminalReached) && meta.largePdfProcessing) {
// The job is dead β€” a "processing continues" message would lie.
meta.largePdfProcessing.current = undefined;
@@ -380,6 +385,12 @@ export async function scrapePDFWithFirePDFAsync(
) {
throw firePdfUsOnlyRefusal(meta, "region_unavailable");
}
+ if (
+ error instanceof FirePdfAsyncFailure &&
+ error.reason === "reserved_unavailable"
+ ) {
+ throw firePdfReservedCapacityRefusal(meta, "reserved_unavailable");
+ }
throw submitMayHaveBeenAccepted ? error.originalError : error;
}
@@ -398,6 +409,12 @@ export async function scrapePDFWithFirePDFAsync(
) {
throw firePdfUsOnlyRefusal(meta, "region_unconfirmed");
}
+ if (
+ isReservedCapacity(meta.internalOptions.teamFlags ?? undefined) &&
+ fetched.reserved_applied !== true
+ ) {
+ throw firePdfReservedCapacityRefusal(meta, "reserved_unconfirmed");
+ }
// ── Assemble + cache save ─────────────────────────────────────────────
const pages =
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/background-refresh.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/background-refresh.ts
index ad62f5210..61a6c30a2 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/background-refresh.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/background-refresh.ts
@@ -4,6 +4,7 @@ import type { Meta } from "../../..";
import type { PDFMode } from "../../../../../controllers/v2/types";
import { config } from "../../../../../config";
import { isUsOnly } from "../../../../../lib/us-only";
+import { isReservedCapacity } from "../../../../../lib/reserved-capacity";
import type { PdfCacheKeyInput } from "../../../../../lib/gcs-pdf-cache";
import {
type BackgroundRefreshResult,
@@ -112,6 +113,10 @@ async function submit(
if (isUsOnly(meta.internalOptions.teamFlags ?? undefined)) {
return "skipped_us_only";
}
+ // Nor its reserved capacity echo.
+ if (isReservedCapacity(meta.internalOptions.teamFlags ?? undefined)) {
+ return "skipped_reserved_capacity";
+ }
// A by-reference document's bytes are in fire-pdf's input bucket, not here.
if (typeof input !== "string") return "skipped_by_reference";
// Raster images are only ever parsed through the sync endpoint.
@@ -144,8 +149,9 @@ async function submit(
tableFormat: undefined,
// Internal work under no team: never offered for training capture.
trainingCapture: undefined,
- // US-only teams never reach here: their refreshes are skipped above.
+ // Restricted teams never reach here: their refreshes are skipped above.
dataRegion: undefined,
+ reserved: undefined,
}),
refresh: true,
},
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference-flow.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference-flow.ts
index 2e9d400fc..3c3c1ff45 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference-flow.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference-flow.ts
@@ -11,6 +11,7 @@ import { cacheKeyShape, tryGetCached } from "./cache";
import { lookupAdoptableFirePdfJob } from "./lookup";
import { buildFirePdfJobOptions, FirePdfAsyncFailure } from "./utils";
import { isUsOnly, US_DATA_REGION } from "../../../../../lib/us-only";
+import { isReservedCapacity } from "../../../../../lib/reserved-capacity";
import {
buildFirePdfJobTrainingCapture,
firePdfTableFormat,
@@ -155,6 +156,9 @@ export async function runFirePdfByReferenceAttempt(
dataRegion: isUsOnly(meta.internalOptions.teamFlags ?? undefined)
? US_DATA_REGION
: undefined,
+ reserved: isReservedCapacity(
+ meta.internalOptions.teamFlags ?? undefined,
+ ),
}),
);
meta.abort.throwIfAborted();
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference.ts
index a2aacae38..6b4603fc1 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference.ts
@@ -12,7 +12,7 @@ import {
} from "../../../../../controllers/v2/types";
import { deterministicPercentage } from "./routing";
import { storage } from "../../../../../lib/gcs-jobs";
-import { isUsOnly } from "../../../../../lib/us-only";
+import { isProcessingRestricted } from "../../../../../lib/reserved-capacity";
import { FIRE_PDF_BY_REFERENCE_MAX_FILE_SIZE } from "../types";
import { parseAlwaysFirePdf } from "./request-metadata";
@@ -202,12 +202,13 @@ export function largePdfLimitBytes(meta: Meta): number {
* fire-engine handoff grant, the download admission, and the PDF engine's
* routing β€” sees the SAME verdict; a per-call random draw would let them
* drift. Keyed apart from the async-cohort hash. Parse requests that must
- * reach FirePDF (parseAlwaysFirePdf) and US-only teams are never diverted. */
+ * reach FirePDF (parseAlwaysFirePdf) and teams whose processing is restricted
+ * (isProcessingRestricted) are never diverted. */
export function mineruDiverted(meta: Meta): boolean {
return (
config.MINERU_PERCENT > 0 &&
!parseAlwaysFirePdf(meta) &&
- !isUsOnly(meta.internalOptions.teamFlags ?? undefined) &&
+ !isProcessingRestricted(meta.internalOptions.teamFlags ?? undefined) &&
deterministicPercentage(`mineru:${meta.id}`) < config.MINERU_PERCENT
);
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache-service.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache-service.ts
index 2ab92c08a..5440c8eeb 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache-service.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache-service.ts
@@ -8,7 +8,7 @@ import {
} from "../../../../../lib/gcs-pdf-cache";
import type { PDFMode } from "../../../../../controllers/v2/types";
import { robustFetch } from "../../../lib/fetch";
-import { isUsOnly } from "../../../../../lib/us-only";
+import { isProcessingRestricted } from "../../../../../lib/reserved-capacity";
import {
recordSpanException,
type Span,
@@ -45,6 +45,27 @@ async function cacheLookupKeys(input: PdfCacheKeyInput): Promise<string[]> {
];
}
+/**
+ * The keys a cache-service lookup asks with. A PDF's raw-byte key is also
+ * recorded on the scrape, so the URL index can later tell whether the cache
+ * holds a newer parse of this document.
+ */
+export async function recordFirePdfCacheKey(
+ meta: Meta,
+ input: PdfCacheKeyInput,
+ sourceKind: FirePdfSourceKind,
+): Promise<string[]> {
+ const keys = await cacheLookupKeys(input);
+ if (
+ sourceKind === "pdf" &&
+ keys[0].startsWith("raw-") &&
+ meta.firePdfCacheKey !== undefined
+ ) {
+ meta.firePdfCacheKey.current = keys[0];
+ }
+ return keys;
+}
+
// The service's own read bound, so a lookup never waits longer than the
// service would; past it the lookup is a miss and the document is parsed. One
// retry, within the same bound, covers a connection that dropped before the
@@ -195,21 +216,14 @@ async function lookupWithinSpan(
// Background refresh is only offered when this client could act on a
// grant: it needs the bytes (inline, not by reference) of a PDF (images
// are only parsed through the sync endpoint), outside zero data retention
- // and outside US-only teams.
+ // and outside teams whose processing is restricted.
const canRefreshInBackground =
config.FIRE_PDF_CACHE_BACKGROUND_REFRESH === true &&
typeof input === "string" &&
sourceKind === "pdf" &&
meta.internalOptions.zeroDataRetention !== true &&
- !isUsOnly(meta.internalOptions.teamFlags ?? undefined);
- const keys = await cacheLookupKeys(input);
- if (
- sourceKind === "pdf" &&
- keys[0].startsWith("raw-") &&
- meta.firePdfCacheKey !== undefined
- ) {
- meta.firePdfCacheKey.current = keys[0];
- }
+ !isProcessingRestricted(meta.internalOptions.teamFlags ?? undefined);
+ const keys = await recordFirePdfCacheKey(meta, input, sourceKind);
let answer: z.infer<typeof lookupOutcomeSchema>;
try {
answer = await robustFetch({
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts
index af8976c9d..a812489c5 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts
@@ -19,7 +19,12 @@ import {
firePdfCacheEventsTotal,
firePdfCacheRefusedWritesTotal,
} from "./metrics";
-import { cacheServiceConfigured, lookupCachedResult } from "./cache-service";
+import {
+ cacheServiceConfigured,
+ lookupCachedResult,
+ recordFirePdfCacheKey,
+} from "./cache-service";
+import { isReservedCapacity } from "../../../../../lib/reserved-capacity";
import { firePdfTableFormat } from "./request-metadata";
import { consumeRefresh, refreshDecisionFor } from "./refresh-budget";
import {
@@ -206,6 +211,15 @@ export async function tryGetCached(
pageMarkers,
);
if (!cacheable) return null;
+ // A stored result may have been produced on other capacity, so a team
+ // restricted to reserved capacity is always parsed afresh. Its own
+ // results are still written, and the key is still recorded for the index.
+ if (isReservedCapacity(meta.internalOptions.teamFlags ?? undefined)) {
+ if (viaService && !isRasterImagePayload(base64Content)) {
+ await recordFirePdfCacheKey(meta, base64Content, "pdf");
+ }
+ return null;
+ }
if (viaService) {
// The service probes the variants, applies the read-time rules and
// budgets `refresh`; a failed lookup is a miss.
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/data-region.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/data-region.ts
index 784c72655..65e00b0ac 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/data-region.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/data-region.ts
@@ -1,5 +1,8 @@
import type { Meta } from "../../..";
-import { UsOnlyUnavailableError } from "../../../error";
+import {
+ ReservedCapacityUnavailableError,
+ UsOnlyUnavailableError,
+} from "../../../error";
import type { FirePdfSourceKind } from "./request-metadata";
/**
@@ -9,6 +12,18 @@ import type { FirePdfSourceKind } from "./request-metadata";
* failure: a job that finished in the region can still report refused acquires.
*/
export function firePdfRegionRefused(body: unknown): boolean {
+ return firePdfRestrictionRefused(
+ body,
+ "data_region_refusals",
+ /\[data_region=\w+ refused: /,
+ );
+}
+
+function firePdfRestrictionRefused(
+ body: unknown,
+ refusalsField: string,
+ notePattern: RegExp,
+): boolean {
let parsed = body;
if (typeof body === "string") {
try {
@@ -18,20 +33,30 @@ export function firePdfRegionRefused(body: unknown): boolean {
}
}
if (typeof parsed !== "object" || parsed === null) return false;
- const { data_region_refusals, error_message } = parsed as Record<
- string,
- unknown
- >;
+ const fields = parsed as Record<string, unknown>;
+ const refusals = fields[refusalsField];
if (
- typeof data_region_refusals === "object" &&
- data_region_refusals !== null &&
- !Array.isArray(data_region_refusals) &&
- Object.keys(data_region_refusals).length > 0
+ typeof refusals === "object" &&
+ refusals !== null &&
+ !Array.isArray(refusals) &&
+ Object.keys(refusals).length > 0
)
return true;
- return (
- typeof error_message === "string" &&
- /\[data_region=\w+ refused: /.test(error_message)
+ const errorMessage = fields.error_message;
+ return typeof errorMessage === "string" && notePattern.test(errorMessage);
+}
+
+/**
+ * Whether a PDF service failure restricted to reserved capacity says the
+ * capacity it needed there was refused: `reserved_refusals` on a sync error
+ * body, or the refusal note a failed job appends to its `error_message`.
+ * Only call it on a failure.
+ */
+export function firePdfReservedRefused(body: unknown): boolean {
+ return firePdfRestrictionRefused(
+ body,
+ "reserved_refusals",
+ /\[reserved refused: /,
);
}
@@ -55,3 +80,17 @@ export function firePdfUsOnlyRefusal(
`${feature} is not available for teams restricted to US-only processing right now: ${detail}. Try again later.`,
);
}
+
+/** The error a team restricted to reserved capacity gets when the PDF service did not run, or did not confirm running, its document there. */
+export function firePdfReservedCapacityRefusal(
+ meta: Meta,
+ reason: "reserved_unavailable" | "reserved_unconfirmed",
+ sourceKind: FirePdfSourceKind = "pdf",
+): ReservedCapacityUnavailableError {
+ meta.logger.warn("FirePDF could not keep a request on reserved capacity", {
+ scrapeId: meta.id,
+ reason,
+ sourceKind,
+ });
+ return new ReservedCapacityUnavailableError();
+}
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.ts
index bb39bed16..77c5290af 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.ts
@@ -14,7 +14,7 @@ export const firePdfCacheRefusedWritesTotal = new Counter({
export const firePdfCacheBackgroundRefreshTotal = new Counter({
name: "firecrawl_fire_pdf_cache_background_refresh_total",
- help: "Background refreshes the fire-pdf cache service granted on a stale answer, by result: submitted, submit_failed, skipped_by_reference, skipped_zdr, skipped_image, skipped_us_only",
+ help: "Background refreshes the fire-pdf cache service granted on a stale answer, by result: submitted, submit_failed, skipped_by_reference, skipped_zdr, skipped_image, skipped_us_only, skipped_reserved_capacity",
labelNames: ["result"],
});
@@ -24,7 +24,8 @@ export type BackgroundRefreshResult =
| "skipped_by_reference"
| "skipped_zdr"
| "skipped_image"
- | "skipped_us_only";
+ | "skipped_us_only"
+ | "skipped_reserved_capacity";
export type CacheRefusedReason =
| "failed_pages"
@@ -124,4 +125,5 @@ export type FallbackReason =
| "terminal_cancelled"
| "polling_timeout"
| "result_503"
- | "region_unavailable";
+ | "region_unavailable"
+ | "reserved_unavailable";
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/poll.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/poll.ts
index 782851c50..8fb20d2ce 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/poll.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/poll.ts
@@ -2,7 +2,8 @@ import type { Meta } from "../../..";
import { fetch as undiciFetch } from "undici";
import { AbortManagerThrownError } from "../../../lib/abortManager";
import { isUsOnly } from "../../../../../lib/us-only";
-import { firePdfRegionRefused } from "./data-region";
+import { isReservedCapacity } from "../../../../../lib/reserved-capacity";
+import { firePdfReservedRefused, firePdfRegionRefused } from "./data-region";
import {
firePdfAsyncCompletedTotal,
firePdfAsyncLongPollTotal,
@@ -184,6 +185,20 @@ export async function pollUntilTerminal(deps: PollDeps): Promise<PollOk> {
failAsync(meta, "region_unavailable", { pollCount, body: pollBody });
}
+ if (
+ isReservedCapacity(meta.internalOptions.teamFlags ?? undefined) &&
+ (pollBody as { status?: unknown }).status === "failed" &&
+ firePdfReservedRefused(pollBody)
+ ) {
+ if (waitMs > 0) firePdfAsyncLongPollTotal.labels("terminal").inc();
+ firePdfAsyncPollCount.observe(pollCount);
+ firePdfAsyncCompletedTotal.labels("failed").inc();
+ failAsync(meta, "reserved_unavailable", {
+ pollCount,
+ body: pollBody,
+ });
+ }
+
if (pollStatus === 401) {
firePdfAsyncPollCount.observe(pollCount);
failAsync(meta, "http_401", { pollCount });
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts
index 37cb1feb3..4a50793f4 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts
@@ -302,6 +302,7 @@ export const resultResponseSchema = z.object({
// build does not understand never fails the scrape.
provenance: z.unknown().optional(),
data_region_applied: z.unknown().optional(),
+ reserved_applied: z.unknown().optional(),
});
export type PollResponse = z.infer<typeof pollResponseSchema>;
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts
index 2de06c019..c349d1c4e 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts
@@ -3,6 +3,7 @@ import type { PDFMode } from "../../../../../controllers/v2/types";
import { fetch as undiciFetch } from "undici";
import { AbortManagerThrownError } from "../../../lib/abortManager";
import { isUsOnly, US_DATA_REGION } from "../../../../../lib/us-only";
+import { isReservedCapacity } from "../../../../../lib/reserved-capacity";
import {
buildFirePdfJobTrainingCapture,
buildFirePdfRequestMetadata,
@@ -120,6 +121,9 @@ export async function submitJob(args: SubmitArgs): Promise<SubmitOutcome> {
} = args;
const scrapeId = meta.id;
const usOnly = isUsOnly(meta.internalOptions.teamFlags ?? undefined);
+ const reservedCapacity = isReservedCapacity(
+ meta.internalOptions.teamFlags ?? undefined,
+ );
if (
input.kind === "byReference" &&
@@ -166,6 +170,7 @@ export async function submitJob(args: SubmitArgs): Promise<SubmitOutcome> {
tableFormat: firePdfTableFormat(meta),
trainingCapture: buildFirePdfJobTrainingCapture(meta),
dataRegion: usOnly ? US_DATA_REGION : undefined,
+ reserved: reservedCapacity,
}),
};
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.ts
index ff953b0fc..950c7b87a 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.ts
@@ -162,6 +162,7 @@ export function buildFirePdfJobOptions(args: {
tableFormat: string | undefined;
trainingCapture: FirePdfTrainingCapture | undefined;
dataRegion: string | undefined;
+ reserved?: boolean;
}): Record<string, unknown> {
return {
// fire-pdf accepts only a positive integer here. When the router has no
@@ -188,6 +189,9 @@ export function buildFirePdfJobOptions(args: {
// An idempotency key on the PDF service, so a pinned lookup only adopts
// a pinned job.
...(args.dataRegion && { data_region: args.dataRegion }),
+ // Also an idempotency key, so a restricted lookup only adopts a job
+ // restricted the same way.
+ ...(args.reserved && { reserved: true }),
};
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts
index fca9490aa..3478b7104 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts
@@ -7,6 +7,7 @@ import type { PDFMode } from "../../../../controllers/v2/types";
import { safeMarkdownToHtml } from "./markdownToHtml";
import { createPdfCacheKey } from "../../../../lib/gcs-pdf-cache";
import { isUsOnly, US_DATA_REGION } from "../../../../lib/us-only";
+import { isReservedCapacity } from "../../../../lib/reserved-capacity";
import {
maybeSaveResult,
provenanceFromResponse,
@@ -20,6 +21,8 @@ import {
type FirePdfSourceKind,
} from "./fire-pdf/request-metadata";
import {
+ firePdfReservedRefused,
+ firePdfReservedCapacityRefusal,
firePdfRegionRefused,
firePdfUsOnlyRefusal,
} from "./fire-pdf/data-region";
@@ -113,6 +116,9 @@ export async function scrapePDFWithFirePDF(
const zdr = meta.internalOptions.zeroDataRetention === true;
const usOnly = isUsOnly(meta.internalOptions.teamFlags ?? undefined);
+ const reservedCapacity = isReservedCapacity(
+ meta.internalOptions.teamFlags ?? undefined,
+ );
const pdfSha256 = createPdfCacheKey(base64Content);
const tableFormat = firePdfTableFormat(meta);
@@ -164,6 +170,7 @@ export async function scrapePDFWithFirePDF(
training_capture: buildFirePdfTrainingCapture(meta),
...deadlineFields,
...(usOnly && { data_region: US_DATA_REGION }),
+ ...(reservedCapacity && { reserved: true }),
},
logger,
schema: z.object({
@@ -185,19 +192,23 @@ export async function scrapePDFWithFirePDF(
// option β€” older builds ignore unknown request fields and omit it.
page_markers: z.literal(true).optional(),
data_region_applied: z.unknown().optional(),
+ reserved_applied: z.unknown().optional(),
}),
mock: meta.mock,
abort: meta.abort.asSignal(),
}).catch(error => {
- if (
- usOnly &&
- firePdfRegionRefused(
- (error as { cause?: { response?: { body?: unknown } } })?.cause
- ?.response?.body,
- )
- ) {
+ const body = (error as { cause?: { response?: { body?: unknown } } })?.cause
+ ?.response?.body;
+ if (usOnly && firePdfRegionRefused(body)) {
throw firePdfUsOnlyRefusal(meta, "region_unavailable", sourceKind);
}
+ if (reservedCapacity && firePdfReservedRefused(body)) {
+ throw firePdfReservedCapacityRefusal(
+ meta,
+ "reserved_unavailable",
+ sourceKind,
+ );
+ }
throw error;
});
// A build that ignores the pin answers without the echo; its result may
@@ -205,6 +216,15 @@ export async function scrapePDFWithFirePDF(
if (usOnly && resp.data_region_applied !== US_DATA_REGION) {
throw firePdfUsOnlyRefusal(meta, "region_unconfirmed", sourceKind);
}
+ // Likewise, without the reserved echo the result may have come from other
+ // capacity.
+ if (reservedCapacity && resp.reserved_applied !== true) {
+ throw firePdfReservedCapacityRefusal(
+ meta,
+ "reserved_unconfirmed",
+ sourceKind,
+ );
+ }
const durationMs = Date.now() - startedAt;
if (includePageMarkdown && resp.pages === undefined) {
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
index a208996f9..eec65e497 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
@@ -17,6 +17,7 @@ import {
EngineUnsuccessfulError,
UnsupportedFileError,
UsOnlyUnavailableError,
+ ReservedCapacityUnavailableError,
} from "../../error";
import { open, readFile, stat, unlink } from "node:fs/promises";
import type { Response } from "undici";
@@ -47,6 +48,7 @@ import {
} from "../../../../lib/native-logging";
import { withSpan, setSpanAttributes } from "../../../../lib/otel-tracer";
import { isUsOnly } from "../../../../lib/us-only";
+import { isProcessingRestricted } from "../../../../lib/reserved-capacity";
import { scrapePDFWithRunPodMU } from "./runpodMU";
import { useFireEngine } from "../fire-engine/available";
import { reconcilePageCountWithFirePdf, scrapePDFWithFirePDF } from "./firePDF";
@@ -247,8 +249,12 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
// US-only teams get GPU parsing only from the PDF service, pinned to US
// capacity: no diversion to another engine, no serverless fallback, and no
- // sampling out of the PDF service.
+ // sampling out of the PDF service. Teams restricted to reserved capacity are
+ // routed the same way.
const usOnly = isUsOnly(meta.internalOptions.teamFlags ?? undefined);
+ const processingRestricted = isProcessingRestricted(
+ meta.internalOptions.teamFlags ?? undefined,
+ );
// The MinerU diversion is deterministic on the scrape id (see
// mineruDiverted) so this routing verdict is BY CONSTRUCTION the same
@@ -634,7 +640,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
fileSizeBytes <= FIRE_PDF_INLINE_HARD_MAX_FILE_SIZE));
const runpodMuUsable =
!forceFirePDF &&
- !usOnly &&
+ !processingRestricted &&
fileSizeBytes < MAX_FILE_SIZE &&
!!config.RUNPOD_MU_API_KEY &&
!!config.RUNPOD_MU_POD_ID;
@@ -682,7 +688,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
config.FIRE_PDF_BASE_URL &&
fileSizeBytes < FIRE_PDF_MAX_FILE_SIZE &&
(alwaysFirePdf ||
- usOnly ||
+ processingRestricted ||
Math.random() * 100 < config.FIRE_PDF_PERCENT)));
if (useFirePDF) {
@@ -749,7 +755,8 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
(!includePageMarkdown && !includeBlocks && !pageMarkers) ||
error instanceof RemoveFeatureError ||
error instanceof AbortManagerThrownError ||
- error instanceof UsOnlyUnavailableError
+ error instanceof UsOnlyUnavailableError ||
+ error instanceof ReservedCapacityUnavailableError
) {
throw error;
}
@@ -798,7 +805,8 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
if (
error instanceof RemoveFeatureError ||
error instanceof AbortManagerThrownError ||
- error instanceof UsOnlyUnavailableError
+ error instanceof UsOnlyUnavailableError ||
+ error instanceof ReservedCapacityUnavailableError
) {
throw error;
}
@@ -835,7 +843,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
if (
!result &&
!forceFirePDF &&
- !usOnly &&
+ !processingRestricted &&
fileSizeBytes >= MAX_FILE_SIZE &&
config.RUNPOD_MU_API_KEY &&
config.RUNPOD_MU_POD_ID
@@ -854,7 +862,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
if (
!result &&
!forceFirePDF &&
- !usOnly &&
+ !processingRestricted &&
fileSizeBytes < MAX_FILE_SIZE &&
base64Content !== undefined &&
config.RUNPOD_MU_API_KEY &&
@@ -972,20 +980,34 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
}
// Without GPU OCR a scanned document has no text layer to return.
if (
- usOnly &&
+ processingRestricted &&
!skipOCR &&
(detectedPdfType === "Scanned" ||
detectedPdfType === "ImageBased" ||
!result.markdown?.trim())
) {
- meta.logger.warn("US-only PDF needs OCR the PDF service did not run", {
- method: "scrapePDF",
- scrape_id: meta.id,
- pdf_type: detectedPdfType,
- });
- throw new UsOnlyUnavailableError(
- "This PDF needs OCR, which is not available for teams restricted to US-only processing right now. Try again later.",
+ if (usOnly) {
+ meta.logger.warn(
+ "US-only PDF needs OCR the PDF service did not run",
+ {
+ method: "scrapePDF",
+ scrape_id: meta.id,
+ pdf_type: detectedPdfType,
+ },
+ );
+ throw new UsOnlyUnavailableError(
+ "This PDF needs OCR, which is not available for teams restricted to US-only processing right now. Try again later.",
+ );
+ }
+ meta.logger.warn(
+ "PDF restricted to reserved capacity needs OCR the PDF service did not run",
+ {
+ method: "scrapePDF",
+ scrape_id: meta.id,
+ pdf_type: detectedPdfType,
+ },
);
+ throw new ReservedCapacityUnavailableError();
}
}
diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.ts
index 80cbb929a..0dd5ab7af 100644
--- a/apps/api/src/scraper/scrapeURL/error.ts
+++ b/apps/api/src/scraper/scrapeURL/error.ts
@@ -47,6 +47,24 @@ export class UsOnlyUnavailableError extends TransportableError {
}
}
+/** Parsing the team's reserved capacity restriction rules out, because the PDF service could not run, or did not confirm running, it there. */
+export class ReservedCapacityUnavailableError extends TransportableError {
+ constructor(
+ message = "PDF processing is unavailable for this team's configuration. Please retry later.",
+ ) {
+ super("RESERVED_CAPACITY_UNAVAILABLE", message);
+ }
+
+ static deserialize(
+ _: ErrorCodes,
+ data: ReturnType<typeof this.prototype.serialize>,
+ ) {
+ const x = new ReservedCapacityUnavailableError(data.message);
+ x.stack = data.stack;
+ return x;
+ }
+}
+
export class NoEnginesLeftError extends TransportableError {
public fallbackList: Engine[];
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 1ecad12c7..0afd13e55 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -61,6 +61,7 @@ import {
BrandingNotSupportedError,
XTwitterConfigurationError,
UsOnlyUnavailableError,
+ ReservedCapacityUnavailableError,
ExchangeRefusedError,
} from "./error";
import { ScrapeRetryTracker } from "./retryTracker";
@@ -1067,6 +1068,7 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
error.error instanceof AgentIndexOnlyError ||
error.error instanceof XTwitterConfigurationError ||
error.error instanceof UsOnlyUnavailableError ||
+ error.error instanceof ReservedCapacityUnavailableError ||
error.error instanceof ExchangeRefusedError ||
error.error instanceof ThirdPartyDataTermsRequiredError
) {