Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): reservedCapacity team flag for PDF parsing (#5707)
Abimael Martell1183fc786775
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.ymlindex 657f4a16c..ead32ad7e 100644--- a/.github/workflows/test-server.yml+++ b/.github/workflows/test-server.yml@@ -394,6 +394,17 @@ jobs:--reporter=defaultworking-directory: apps/api+ - name: Run reserved capacity PDF regression tests+ if: matrix.engine == 'fetch' && matrix.proxy == 'no-proxy' && matrix.nuq == 'postgres'+ run: >-+ pnpm exec vitest run+ src/scraper/scrapeURL/engines/pdf/__tests__/firePDFReservedCapacity.test.ts+ src/scraper/scrapeURL/engines/pdf/__tests__/reservedCapacityPdfRouting.test.ts+ src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts+ src/scraper/scrapeURL/engines/pdf/__tests__/indexPdfSupersede.test.ts+ --reporter=default+ working-directory: apps/api+- name: Run US-only page fetch regression testsif: matrix.engine == 'fetch' && matrix.proxy == 'no-proxy' && matrix.nuq == 'postgres'run: >-diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex 70489f63d..1ab28e811 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -372,7 +372,8 @@ async function scrapeControllerInner(if (e.code === "unsafe_domain_blocked" ||- e.code === "US_ONLY_UNAVAILABLE"+ e.code === "US_ONLY_UNAVAILABLE" ||+ e.code === "RESERVED_CAPACITY_UNAVAILABLE") {return res.status(403).json({success: false,diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 3ad00e9cd..a196161f6 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -1417,6 +1417,9 @@ export type TeamFlags = {// Every request is processed by US-based vendors only. A route with no US// vendor is refused or skipped with a warning.usOnly?: boolean;+ // PDF and image parsing runs only on reserved processing capacity. A parse+ // that cannot be kept there is refused.+ reservedCapacity?: boolean;// Per-team extensions past a deprecation's sunset (lib/deprecations.ts):// surface or entry key to the YYYY-MM-DD or UTC timestamp it runs until.deprecationExtensions?: Record<string, string>;diff --git a/apps/api/src/controllers/v2/parse.ts b/apps/api/src/controllers/v2/parse.tsindex 839933c80..2140ccfb5 100644--- a/apps/api/src/controllers/v2/parse.ts+++ b/apps/api/src/controllers/v2/parse.ts@@ -652,7 +652,10 @@ export async function parseController(});}- if (e.code === "US_ONLY_UNAVAILABLE") {+ if (+ e.code === "US_ONLY_UNAVAILABLE" ||+ e.code === "RESERVED_CAPACITY_UNAVAILABLE"+ ) {setSpanAttributes(span, {"parse.status_code": 403,});diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.tsindex 801987624..31ff53935 100644--- a/apps/api/src/controllers/v2/scrape.ts+++ b/apps/api/src/controllers/v2/scrape.ts@@ -544,7 +544,8 @@ export async function scrapeController(if (e.code === "unsafe_domain_blocked" ||- e.code === "US_ONLY_UNAVAILABLE"+ e.code === "US_ONLY_UNAVAILABLE" ||+ e.code === "RESERVED_CAPACITY_UNAVAILABLE") {setSpanAttributes(span, {"scrape.status_code": 403,diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 7680dc445..bc15c3b0b 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -2192,6 +2192,9 @@ export type TeamFlags = {// Every request is processed by US-based vendors only. A route with no US// vendor is refused or skipped with a warning.usOnly?: boolean;+ // PDF and image parsing runs only on reserved processing capacity. A parse+ // that cannot be kept there is refused.+ reservedCapacity?: boolean;// Per-team extensions past a deprecation's sunset (lib/deprecations.ts):// surface or entry key to the YYYY-MM-DD or UTC timestamp it runs until.deprecationExtensions?: Record<string, string>;diff --git a/apps/api/src/lib/error-serde.ts b/apps/api/src/lib/error-serde.tsindex 31fb73971..b9a2c3263 100644--- a/apps/api/src/lib/error-serde.ts+++ b/apps/api/src/lib/error-serde.ts@@ -43,6 +43,7 @@ import {JsonExtractionContentTooLargeError,XTwitterConfigurationError,UsOnlyUnavailableError,+ ReservedCapacityUnavailableError,ExchangeRefusedError,} from "../scraper/scrapeURL/error";import { UnsafeDomainBlockedError } from "./threat-protection/error";@@ -98,6 +99,7 @@ const errorMap: Record<ErrorCodes, any> = {SCRAPE_JSON_CONTENT_TOO_LARGE: JsonExtractionContentTooLargeError,SCRAPE_X_TWITTER_CONFIGURATION_ERROR: XTwitterConfigurationError,US_ONLY_UNAVAILABLE: UsOnlyUnavailableError,+ RESERVED_CAPACITY_UNAVAILABLE: ReservedCapacityUnavailableError,MAP_FAILED: MapFailedError,CONCURRENCY_QUEUE_TIMEOUT: ConcurrencyQueueTimeoutError,unsafe_domain_blocked: UnsafeDomainBlockedError,diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.tsindex f3952e70b..c6fde4732 100644--- a/apps/api/src/lib/error.ts+++ b/apps/api/src/lib/error.ts@@ -44,6 +44,7 @@ export type ErrorCodes =| "SCRAPE_JSON_CONTENT_TOO_LARGE"| "SCRAPE_X_TWITTER_CONFIGURATION_ERROR"| "US_ONLY_UNAVAILABLE"+ | "RESERVED_CAPACITY_UNAVAILABLE"| "PARSE_UNSUPPORTED_OPTIONS"| "CRAWL_DENIAL"| "UNSUPPORTED_SITE"diff --git a/apps/api/src/lib/reserved-capacity.ts b/apps/api/src/lib/reserved-capacity.tsnew file mode 100644index 000000000..bae9880a8--- /dev/null+++ b/apps/api/src/lib/reserved-capacity.ts@@ -0,0 +1,17 @@+import type { TeamFlags } from "../controllers/v2/types";+import { isUsOnly } from "./us-only";++/** Whether the team's PDF and image parsing must run on reserved processing capacity. Any value but `true` reads as unrestricted. */+export function isReservedCapacity(flags: TeamFlags | undefined): boolean {+ return flags?.reservedCapacity === true;+}++/**+ * Whether the team's PDF and image parsing carries a restriction the PDF+ * service must honor and confirm (US-only or reserved capacity). Such work+ * runs only on the PDF service: no other parsing engine, and no background+ * work the service would run without the restriction.+ */+export function isProcessingRestricted(flags: TeamFlags | undefined): boolean {+ return isUsOnly(flags) || isReservedCapacity(flags);+}diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.tsindex 053a76eae..eb211f27a 100644--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts@@ -52,6 +52,7 @@ import {} from "../../../../controllers/v2/types";import { hasFormatOfType } from "../../../../lib/format-utils";import { hasCustomRequestContext } from "../../lib/request-context";+import { isReservedCapacity } from "../../../../lib/reserved-capacity";export async function sendDocumentToIndex(meta: Meta, document: Document) {// Skip caching if screenshot format has custom viewport or quality settings@@ -588,6 +589,19 @@ export async function scrapeURLWithIndex(throw new IndexMissError();}+ // A parsed PDF or OCR'd image may have been produced on other capacity, so+ // a team restricted to reserved capacity parses it afresh. Raw PDF+ // bytes were never parsed and are still served.+ if (+ isReservedCapacity(meta.internalOptions.teamFlags ?? undefined) &&+ !isCachedPdfBase64 &&+ (doc.contentType === "application/pdf" ||+ doc.contentType?.startsWith("image/"))+ ) {+ logLookup("debug", "hit", { reservedCapacityMismatch: true });+ throw new IndexMissError();+ }+// fire-pdf may have re-parsed this PDF since it was indexed (a refresh or a// campaign). Only once a fresh entry exists is the index copy skipped, so// the request lands on that entry without a parse and is re-indexed.diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFBackgroundRefresh.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFBackgroundRefresh.test.tsindex 6f5d92113..5f6801657 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFBackgroundRefresh.test.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFBackgroundRefresh.test.ts@@ -320,4 +320,28 @@ describe("FirePDF cache background refresh", () => {expect(jobsFetch).not.toHaveBeenCalled();expect(await refreshCount("skipped_us_only")).toBe(1);});++ it("serves a team restricted to reserved capacity the stale entry without offering or submitting a refresh", async () => {+ const meta = makeMeta();+ meta.internalOptions.teamFlags = { reservedCapacity: true };+ answerWith(staleAnswer(true));+ expect(await lookupCachedResult(meta, base64, lookupArgs)).toEqual(+ staleResult,+ );+ await settle();+ expect(lookupFetch.mock.calls[0][0].body).not.toHaveProperty(+ "background_refresh",+ );+ expect(+ await submitBackgroundRefresh(meta, base64, {+ mode: undefined,+ includePageMarkdown: false,+ includeBlocks: false,+ pageMarkers: false,+ sourceKind: "pdf",+ }),+ ).toBe("skipped_reserved_capacity");+ expect(jobsFetch).not.toHaveBeenCalled();+ expect(await refreshCount("skipped_reserved_capacity")).toBe(1);+ });});diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFReservedCapacity.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFReservedCapacity.test.tsnew file mode 100644index 000000000..8e1928b3a--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFReservedCapacity.test.ts@@ -0,0 +1,628 @@+// Stub the GCS cache so the async client never reaches real cloud storage.+vi.mock("../../../../../lib/gcs-pdf-cache", () => ({+ pdfCacheConfigured: vi.fn(() => true),+ createPdfCacheKey: (s: string) => `sha-${s.length}`,+ resolvePdfCacheKey: (input: string | { key: string }) =>+ typeof input === "string" ? `sha-${input.length}` : input.key,+ getPdfResultFromCache: vi.fn(async () => null),+ savePdfResultToCache: vi.fn(async () => null),+}));+vi.mock("../../../lib/fetch", () => ({+ robustFetch: vi.fn(),+}));++import { robustFetch } from "../../../lib/fetch";+import { config } from "../../../../../config";+import {+ getPdfResultFromCache,+ savePdfResultToCache,+} from "../../../../../lib/gcs-pdf-cache";+import {+ ReservedCapacityUnavailableError,+ UsOnlyUnavailableError,+} from "../../../error";+import { scrapePDFWithFirePDF } from "../firePDF";+import { tryGetCached } from "../fire-pdf/cache";+import { scrapePDFWithFirePDFAsync } from "../fire-pdf/async";+import { lookupAdoptableFirePdfJob } from "../fire-pdf/lookup";+import { buildFirePdfJobOptions, FirePdfAsyncFailure } from "../fire-pdf/utils";+import {+ makeFetchFromSequence,+ makeMeta as makeAsyncMeta,+ noopSleep,+} from "./firePDFAsyncFixtures";++const mockedRobustFetch = vi.mocked(robustFetch);+const mockedGetCached = vi.mocked(getPdfResultFromCache);++const reservedFlags = { reservedCapacity: true };+const bothFlags = { usOnly: true, reservedCapacity: true };++function makeSyncMeta(teamFlags?: Record<string, unknown>) {+ const logger: any = {+ info: vi.fn(),+ warn: vi.fn(),+ error: vi.fn(),+ debug: vi.fn(),+ };+ return {+ id: "reserved-capacity-sync-test",+ url: "https://example.com/file.pdf",+ rewrittenUrl: undefined,+ options: {},+ logger,+ mock: null,+ abort: {+ throwIfAborted: vi.fn(),+ asSignal: vi.fn(() => new AbortController().signal),+ scrapeTimeout: vi.fn(() => 60_000),+ },+ internalOptions: {+ zeroDataRetention: false,+ teamId: "team-test",+ ...(teamFlags && { teamFlags }),+ },+ } as any;+}++function makeReservedAsyncMeta(teamFlags?: Record<string, unknown>) {+ return makeAsyncMeta({+ internalOptions: {+ zeroDataRetention: false,+ teamId: "team-x",+ teamConcurrency: 12,+ crawlId: undefined,+ ...(teamFlags && { teamFlags }),+ },+ });+}++function failureStatus(status: number, body: unknown) {+ return new Error("Request sent failure status", {+ cause: { response: { status, body: JSON.stringify(body) } },+ });+}++const syncResult = (extra: Record<string, unknown>) =>+ ({+ markdown: "Document",+ failed_pages: null,+ pages_processed: 1,+ ...extra,+ }) as any;++// The PDF service's sync answer when the reserved capacity it needed was refused.+const reservedNoCapacity = {+ error: "No OCR pods available",+ reserved_refusals: { reserved_unroutable: 3 },+};++const ORIGINAL_BASE_URL = config.FIRE_PDF_BASE_URL;++beforeAll(() => {+ config.FIRE_PDF_BASE_URL = "http://fire-pdf.test";+});++afterAll(() => {+ config.FIRE_PDF_BASE_URL = ORIGINAL_BASE_URL;+});++beforeEach(() => {+ vi.clearAllMocks();+});++describe("FirePDF sync parse for teams restricted to reserved capacity", () => {+ it("sends reserved only for a restricted team", async () => {+ mockedRobustFetch.mockResolvedValue(syncResult({ reserved_applied: true }));++ await scrapePDFWithFirePDF(makeSyncMeta(), "BASE64", 1);+ await scrapePDFWithFirePDF(makeSyncMeta(reservedFlags), "BASE64", 1);++ const unrestricted = mockedRobustFetch.mock.calls[0][0].body;+ const restricted = mockedRobustFetch.mock.calls[1][0].body;+ expect(Object.hasOwn(unrestricted, "reserved")).toBe(false);+ expect(restricted.reserved).toBe(true);+ expect(Object.hasOwn(restricted, "data_region")).toBe(false);+ });++ it("returns a restricted result that carries the reserved echo", async () => {+ mockedRobustFetch.mockResolvedValue(syncResult({ reserved_applied: true }));++ const result = await scrapePDFWithFirePDF(+ makeSyncMeta(reservedFlags),+ "BASE64",+ 1,+ );++ expect(result.markdown).toBe("Document");+ });++ it("refuses a restricted parse the PDF service could not place", async () => {+ mockedRobustFetch.mockRejectedValue(failureStatus(503, reservedNoCapacity));++ const error = await scrapePDFWithFirePDF(+ makeSyncMeta(reservedFlags),+ "BASE64",+ 1,+ ).catch(e => e);++ expect(error).toBeInstanceOf(ReservedCapacityUnavailableError);+ expect(error.code).toBe("RESERVED_CAPACITY_UNAVAILABLE");+ expect(error.message).toContain(+ "PDF processing is unavailable for this team's configuration. Please retry later.",+ );+ });++ it("keeps a restricted failure without reserved refusals on its own error", async () => {+ const failure = failureStatus(502, { error: "All 3 pages failed" });+ mockedRobustFetch.mockRejectedValue(failure);++ const error = await scrapePDFWithFirePDF(+ makeSyncMeta(reservedFlags),+ "BASE64",+ 1,+ ).catch(e => e);++ expect(error).toBe(failure);+ });++ it.each([+ ["missing", {}],+ ["null", { reserved_applied: null }],+ ["false", { reserved_applied: false }],+ ])(+ "refuses a restricted result whose reserved echo is %s",+ async (_name, echo) => {+ mockedRobustFetch.mockResolvedValue(syncResult(echo));++ const error = await scrapePDFWithFirePDF(+ makeSyncMeta(reservedFlags),+ "BASE64",+ 1,+ ).catch(e => e);++ expect(error).toBeInstanceOf(ReservedCapacityUnavailableError);+ expect(error.message).toContain(+ "PDF processing is unavailable for this team's configuration. Please retry later.",+ );+ },+ );++ it("uses the same error for a restricted image", async () => {+ mockedRobustFetch.mockRejectedValue(failureStatus(503, reservedNoCapacity));++ const error = await scrapePDFWithFirePDF(+ makeSyncMeta(reservedFlags),+ "BASE64",+ 1,+ 1,+ "ocr",+ false,+ false,+ false,+ "image",+ ).catch(e => e);++ expect(error).toBeInstanceOf(ReservedCapacityUnavailableError);+ expect(error.message).toContain(+ "PDF processing is unavailable for this team's configuration. Please retry later.",+ );+ });++ it("keeps an unrestricted failure and an unrestricted result without the echo unchanged", async () => {+ const failure = failureStatus(503, reservedNoCapacity);+ mockedRobustFetch.mockRejectedValueOnce(failure);+ expect(+ await scrapePDFWithFirePDF(makeSyncMeta(), "BASE64", 1).catch(e => e),+ ).toBe(failure);++ mockedRobustFetch.mockResolvedValueOnce(syncResult({}));+ const result = await scrapePDFWithFirePDF(makeSyncMeta(), "BASE64", 1);+ expect(result.markdown).toBe("Document");+ });++ describe("with US-only as well", () => {+ it("sends both fields and returns a result carrying both echoes", async () => {+ mockedRobustFetch.mockResolvedValue(+ syncResult({ data_region_applied: "us", reserved_applied: true }),+ );++ const result = await scrapePDFWithFirePDF(+ makeSyncMeta(bothFlags),+ "BASE64",+ 1,+ );++ const body = mockedRobustFetch.mock.calls[0][0].body;+ expect(body.data_region).toBe("us");+ expect(body.reserved).toBe(true);+ expect(result.markdown).toBe("Document");+ });++ it.each([+ ["the region echo", { reserved_applied: true }, UsOnlyUnavailableError],+ [+ "the reserved echo",+ { data_region_applied: "us" },+ ReservedCapacityUnavailableError,+ ],+ ])("refuses a result missing %s", async (_name, echo, errorClass) => {+ mockedRobustFetch.mockResolvedValue(syncResult(echo));++ const error = await scrapePDFWithFirePDF(+ makeSyncMeta(bothFlags),+ "BASE64",+ 1,+ ).catch(e => e);++ expect(error).toBeInstanceOf(errorClass);+ });++ it("keeps the US-only error when both restrictions were refused", async () => {+ mockedRobustFetch.mockRejectedValue(+ failureStatus(503, {+ ...reservedNoCapacity,+ data_region_refusals: { region_unroutable: 1 },+ }),+ );++ const error = await scrapePDFWithFirePDF(+ makeSyncMeta(bothFlags),+ "BASE64",+ 1,+ ).catch(e => e);++ expect(error).toBeInstanceOf(UsOnlyUnavailableError);+ });++ it("maps a reserved-only refusal to the reserved capacity error", async () => {+ mockedRobustFetch.mockRejectedValue(+ failureStatus(503, reservedNoCapacity),+ );++ const error = await scrapePDFWithFirePDF(+ makeSyncMeta(bothFlags),+ "BASE64",+ 1,+ ).catch(e => e);++ expect(error).toBeInstanceOf(ReservedCapacityUnavailableError);+ });+ });+});++const submitAccepted = {+ matchUrl: /\/jobs$/,+ matchMethod: "POST" as const,+ response: {+ status: 202,+ body: { scrape_id: "scrape-id-test", status: "queued", lane: "fast" },+ },+};+const pollDone = {+ matchUrl: /\/jobs\/scrape-id-test$/,+ matchMethod: "GET" as const,+ response: {+ status: 200,+ body: { scrape_id: "scrape-id-test", status: "done", pages_processed: 3 },+ },+};+function resultWith(extra: Record<string, unknown>) {+ return {+ matchUrl: /\/jobs\/scrape-id-test\/result$/,+ matchMethod: "GET" as const,+ response: {+ status: 200,+ body: {+ schema_version: 1,+ markdown: "# Async",+ pages_processed: 3,+ failed_pages: null,+ partial_pages: null,+ ...extra,+ },+ },+ };+}+function pollFailed(errorMessage: string) {+ return {+ matchUrl: /\/jobs\/scrape-id-test$/,+ matchMethod: "GET" as const,+ response: {+ status: 502,+ body: {+ scrape_id: "scrape-id-test",+ reserved_applied: true,+ status: "failed",+ error_class: "ocr_unavailable",+ error_message: errorMessage,+ },+ },+ };+}++function runAsync(meta: any, fetchImpl: any) {+ return scrapePDFWithFirePDFAsync(+ meta,+ "BASE64",+ undefined,+ undefined,+ undefined,+ { fetchImpl, fallbackImpl: vi.fn(), sleepImpl: noopSleep },+ );+}++describe("FirePDF async jobs for teams restricted to reserved capacity", () => {+ it("sends reserved in the job options only for a restricted team", async () => {+ const unrestricted = makeFetchFromSequence([+ submitAccepted,+ pollDone,+ resultWith({ reserved_applied: null }),+ ]);+ await runAsync(makeReservedAsyncMeta(), unrestricted.fetchImpl);++ const restricted = makeFetchFromSequence([+ submitAccepted,+ pollDone,+ resultWith({ reserved_applied: true }),+ ]);+ const result = await runAsync(+ makeReservedAsyncMeta(reservedFlags),+ restricted.fetchImpl,+ );++ expect(unrestricted.calls[0].body).not.toHaveProperty("reserved");+ expect(unrestricted.calls[0].body).not.toHaveProperty("options.reserved");+ expect(restricted.calls[0].body).not.toHaveProperty("reserved");+ expect(restricted.calls[0].body).toHaveProperty("options.reserved", true);+ expect(restricted.calls[0].body).not.toHaveProperty("options.data_region");+ expect(result.markdown).toBe("# Async");+ });++ it("refuses a restricted job that failed for lack of reserved capacity, without cancelling it", async () => {+ const { fetchImpl, calls } = makeFetchFromSequence([+ submitAccepted,+ pollFailed(+ "No OCR pods available [reserved refused: reserved_unroutable=4]",+ ),+ ]);++ const error = await runAsync(+ makeReservedAsyncMeta(reservedFlags),+ fetchImpl,+ ).catch(e => e);++ expect(error).toBeInstanceOf(ReservedCapacityUnavailableError);+ expect(calls.map(call => call.method)).toEqual(["POST", "GET"]);+ });++ it("keeps a restricted job failure without reserved refusals off the restriction error", async () => {+ const { fetchImpl } = makeFetchFromSequence([+ submitAccepted,+ pollFailed("Batch render failed"),+ ]);++ const error = await runAsync(+ makeReservedAsyncMeta(reservedFlags),+ fetchImpl,+ ).catch(e => e);++ expect(error).toBeInstanceOf(FirePdfAsyncFailure);+ expect(error.reason).toBe("terminal_failed");+ });++ it.each([+ ["missing", {}],+ ["null", { reserved_applied: null }],+ ["false", { reserved_applied: false }],+ ])(+ "refuses a restricted result whose reserved echo is %s and caches nothing",+ async (_name, echo) => {+ const { fetchImpl } = makeFetchFromSequence([+ submitAccepted,+ pollDone,+ resultWith(echo),+ ]);++ const error = await runAsync(+ makeReservedAsyncMeta(reservedFlags),+ fetchImpl,+ ).catch(e => e);++ expect(error).toBeInstanceOf(ReservedCapacityUnavailableError);+ expect(savePdfResultToCache).not.toHaveBeenCalled();+ },+ );++ describe("by reference", () => {+ const byReference = {+ gcsUri: "gs://bucket/inputs/scrape-id-test.pdf",+ sha256: "ab".repeat(32),+ sizeBytes: 200 * 1024 * 1024,+ };+ const runByReference = (meta: any, fetchImpl: any) =>+ scrapePDFWithFirePDFAsync(meta, byReference, undefined, 3, undefined, {+ fetchImpl,+ fallbackImpl: vi.fn(),+ sleepImpl: noopSleep,+ });++ it("sends reserved in the job options and accepts the reserved echo", async () => {+ const { fetchImpl, calls } = makeFetchFromSequence([+ submitAccepted,+ pollDone,+ resultWith({ reserved_applied: true }),+ ]);++ const result = await runByReference(+ makeReservedAsyncMeta(reservedFlags),+ fetchImpl,+ );++ expect(calls[0].body).toHaveProperty("input_gcs_uri", byReference.gcsUri);+ expect(calls[0].body).toHaveProperty("options.reserved", true);+ expect(result.markdown).toBe("# Async");+ });++ it("refuses a result without the reserved echo and caches nothing", async () => {+ const { fetchImpl } = makeFetchFromSequence([+ submitAccepted,+ pollDone,+ resultWith({}),+ ]);++ const error = await runByReference(+ makeReservedAsyncMeta(reservedFlags),+ fetchImpl,+ ).catch(e => e);++ expect(error).toBeInstanceOf(ReservedCapacityUnavailableError);+ expect(savePdfResultToCache).not.toHaveBeenCalled();+ });+ });++ it("sends both restrictions and requires both echoes for a team with both flags", async () => {+ const both = makeFetchFromSequence([+ submitAccepted,+ pollDone,+ resultWith({ data_region_applied: "us", reserved_applied: true }),+ ]);+ const result = await runAsync(+ makeReservedAsyncMeta(bothFlags),+ both.fetchImpl,+ );+ expect(both.calls[0].body).toHaveProperty("options.data_region", "us");+ expect(both.calls[0].body).toHaveProperty("options.reserved", true);+ expect(result.markdown).toBe("# Async");++ const regionOnly = makeFetchFromSequence([+ submitAccepted,+ pollDone,+ resultWith({ data_region_applied: "us" }),+ ]);+ const error = await runAsync(+ makeReservedAsyncMeta(bothFlags),+ regionOnly.fetchImpl,+ ).catch(e => e);+ expect(error).toBeInstanceOf(ReservedCapacityUnavailableError);+ });++ it("forwards reserved from the caller's job options to the content lookup", async () => {+ const lookupHit = {+ matchUrl: /\/jobs\/lookup$/,+ matchMethod: "POST" as const,+ response: {+ status: 200,+ body: { scrape_id: "found-1", status: "running" },+ },+ };+ const restricted = makeFetchFromSequence([lookupHit]);++ await lookupAdoptableFirePdfJob(+ makeReservedAsyncMeta(reservedFlags),+ "ef".repeat(32),+ buildFirePdfJobOptions({+ maxPages: undefined,+ pagesProcessed: undefined,+ mode: "auto",+ includePageMarkdown: false,+ includeBlocks: false,+ pageMarkers: false,+ tableFormat: undefined,+ trainingCapture: undefined,+ dataRegion: undefined,+ reserved: true,+ }),+ restricted.fetchImpl,+ );++ expect(restricted.calls[0].body).not.toHaveProperty("reserved");+ expect(restricted.calls[0].body).toHaveProperty("options", {+ mode: "auto",+ reserved: true,+ });+ });+});++describe("FirePDF content cache for teams restricted to reserved capacity", () => {+ const cachedEntry = { markdown: "cached", html: "<p>cached</p>" } as any;++ afterEach(() => {+ config.FIRE_PDF_CACHE_BASE_URL = undefined;+ });++ it("never reads the cache for a restricted team but still writes its result", async () => {+ mockedGetCached.mockResolvedValue(cachedEntry);+ mockedRobustFetch.mockResolvedValue(syncResult({ reserved_applied: true }));++ const result = await scrapePDFWithFirePDF(+ makeSyncMeta(reservedFlags),+ "BASE64",+ );++ expect(result.markdown).toBe("Document");+ expect(mockedGetCached).not.toHaveBeenCalled();+ expect(mockedRobustFetch).toHaveBeenCalledTimes(1);+ expect(savePdfResultToCache).toHaveBeenCalled();+ });++ it.each([+ ["an unrestricted team", undefined],+ ["a US-only team", { usOnly: true }],+ ])("still serves %s from the cache", async (_name, flags) => {+ mockedGetCached.mockResolvedValue(cachedEntry);++ const result = await scrapePDFWithFirePDF(makeSyncMeta(flags), "BASE64");++ expect(result.markdown).toBe("cached");+ expect(mockedRobustFetch).not.toHaveBeenCalled();+ });++ it("skips the cache service lookup for a restricted team", async () => {+ config.FIRE_PDF_CACHE_BASE_URL = "http://cache.test";+ mockedRobustFetch.mockResolvedValue(syncResult({ reserved_applied: true }));++ await scrapePDFWithFirePDF(makeSyncMeta(reservedFlags), "BASE64");++ const urls = mockedRobustFetch.mock.calls.map(call => call[0].url);+ expect(urls).toEqual(["http://fire-pdf.test/ocr"]);+ });++ it("still records the raw cache key for the URL index without a lookup", async () => {+ config.FIRE_PDF_CACHE_BASE_URL = "http://cache.test";+ const meta = { ...makeSyncMeta(reservedFlags), firePdfCacheKey: {} };+ const rawKey = `raw-${"cd".repeat(32)}`;++ const cached = await tryGetCached(+ meta,+ { key: rawKey },+ "auto",+ undefined,+ 3,+ false,+ false,+ );++ expect(cached).toBeNull();+ expect(meta.firePdfCacheKey).toEqual({ current: rawKey });+ expect(mockedRobustFetch).not.toHaveBeenCalled();+ });++ it("skips the content cache read on async jobs for a restricted team", async () => {+ mockedGetCached.mockResolvedValue(cachedEntry);+ const { fetchImpl, calls } = makeFetchFromSequence([+ submitAccepted,+ pollDone,+ resultWith({ reserved_applied: true }),+ ]);++ const result = await runAsync(+ makeReservedAsyncMeta(reservedFlags),+ fetchImpl,+ );++ expect(result.markdown).toBe("# Async");+ expect(mockedGetCached).not.toHaveBeenCalled();+ expect(calls[0].method).toBe("POST");+ });+});diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPdfSupersede.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPdfSupersede.test.tsindex c102ec137..422b8793d 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPdfSupersede.test.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPdfSupersede.test.ts@@ -40,7 +40,7 @@ const mutable = config as unknown as MutableConfig;const createdAt = "2026-10-01T00:00:00.000Z";const rawKey = `raw-${"b".repeat(64)}`;-function meta() {+function meta(teamFlags?: Record<string, unknown>) {return {id: "index-supersede",url: "https://example.com/doc.pdf",@@ -51,7 +51,7 @@ function meta() {error: vi.fn(),child: () => ({ info: vi.fn(), warn: vi.fn(), error: vi.fn() }),},- internalOptions: { teamId: "team-1" },+ internalOptions: { teamId: "team-1", ...(teamFlags && { teamFlags }) },featureFlags: new Set(),options: {maxAge: 7 * 24 * 60 * 60 * 1000,@@ -189,3 +189,59 @@ describe("URL index PDF supersede check", () => {expect(checkFirePdfSupersedes).not.toHaveBeenCalled();});});++describe("URL index for teams restricted to reserved capacity", () => {+ const reserved = { reservedCapacity: true };++ beforeEach(() => {+ vi.mocked(indexGetRecent5).mockResolvedValue([+ {+ id: "row-1",+ created_at: createdAt,+ status: 200,+ has_screenshot: false,+ has_screenshot_fullscreen: false,+ wait_time_ms: null,+ },+ ] as any);+ vi.mocked(getCachedIndexEntries).mockResolvedValue({ status: "miss" });+ });++ it("misses a parsed PDF entry for a restricted team", async () => {+ vi.mocked(getIndexFromGCS).mockResolvedValue(indexedPdf());+ await expect(scrapeURLWithIndex(meta(reserved))).rejects.toBeInstanceOf(+ IndexMissError,+ );+ });++ it("misses an OCR'd image entry for a restricted team", async () => {+ vi.mocked(getIndexFromGCS).mockResolvedValue(+ indexedPdf({ contentType: "image/png", pdfMetadata: undefined }),+ );+ const restricted = { ...meta(reserved), imageOcrEnabled: async () => true };+ await expect(scrapeURLWithIndex(restricted)).rejects.toBeInstanceOf(+ IndexMissError,+ );+ });++ it.each([+ ["an unrestricted team", undefined],+ ["a US-only team", { usOnly: true }],+ ])("still serves a parsed PDF entry to %s", async (_name, flags) => {+ vi.mocked(getIndexFromGCS).mockResolvedValue(indexedPdf());+ const served = await scrapeURLWithIndex(meta(flags));+ expect(served.html).toBe("<p>old parse</p>");+ });++ it("still serves a non-PDF entry to a restricted team", async () => {+ vi.mocked(getIndexFromGCS).mockResolvedValue(+ indexedPdf({+ url: "https://example.com/page",+ contentType: "text/html",+ pdfMetadata: undefined,+ }),+ );+ const served = await scrapeURLWithIndex(meta(reserved));+ expect(served.html).toBe("<p>old parse</p>");+ });+});diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/reservedCapacityPdfRouting.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/reservedCapacityPdfRouting.test.tsnew file mode 100644index 000000000..1923311c1--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/reservedCapacityPdfRouting.test.ts@@ -0,0 +1,239 @@+import { mkdtemp, rm, writeFile } from "node:fs/promises";+import { tmpdir } from "node:os";+import path from "node:path";+import { processPdf } from "@mendable/firecrawl-rs";+import { config } from "../../../../../config";+import {+ ReservedCapacityUnavailableError,+ UsOnlyUnavailableError,+} from "../../../error";+import { scrapePDFWithFirePDF } from "../firePDF";+import { scrapePDFWithParsePDF } from "../pdfParse";+import { scrapePDFWithRunPodMU } from "../runpodMU";+import { mineruDiverted } from "../fire-pdf/by-reference";+import { scrapePDF } from "../index";++vi.mock("@mendable/firecrawl-rs", () => ({+ processPdf: vi.fn(),+ detectPdf: vi.fn(),+}));++vi.mock("../firePDF", async importOriginal => ({+ ...(await importOriginal<typeof import("../firePDF")>()),+ scrapePDFWithFirePDF: vi.fn(),+}));++vi.mock("../pdfParse", () => ({+ scrapePDFWithParsePDF: vi.fn(),+}));++vi.mock("../runpodMU", () => ({+ scrapePDFWithRunPodMU: vi.fn(),+}));++vi.mock("../markdownToHtml", () => ({+ safeMarkdownToHtml: vi.fn(async (markdown: string) => `<p>${markdown}</p>`),+}));++const mockedProcessPdf = vi.mocked(processPdf);+const mockedFirePdf = vi.mocked(scrapePDFWithFirePDF);+const mockedPdfParse = vi.mocked(scrapePDFWithParsePDF);+const mockedRunPodMU = vi.mocked(scrapePDFWithRunPodMU);++const configKeys = [+ "PDF_RUST_EXTRACT_ENABLE",+ "FIRE_PDF_ENABLE",+ "FIRE_PDF_BASE_URL",+ "FIRE_PDF_PERCENT",+ "FIRE_PDF_ASYNC_PERCENT",+ "MINERU_PERCENT",+ "RUNPOD_MU_API_KEY",+ "RUNPOD_MU_POD_ID",+ "PARSE_PDF_ALWAYS_FIRE_PDF",+] as const;+const originalConfig = Object.fromEntries(+ configKeys.map(key => [key, config[key]]),+);++function setConfig(+ overrides: Partial<Record<(typeof configKeys)[number], unknown>>,+) {+ Object.assign(config as any, overrides);+}++const tempDirs: string[] = [];++async function makeMeta(teamFlags?: Record<string, unknown>) {+ const dir = await mkdtemp(path.join(tmpdir(), "reserved-capacity-pdf-"));+ tempDirs.push(dir);+ const filePath = path.join(dir, "doc.pdf");+ await writeFile(filePath, "%PDF-1.7\n%test\n");+ const logger: any = {+ info: vi.fn(),+ warn: vi.fn(),+ error: vi.fn(),+ debug: vi.fn(),+ };+ logger.child = vi.fn(() => logger);+ return {+ id: "01a00000-0000-7000-8000-000000000001",+ url: "https://example.com/doc.pdf",+ options: { parsers: ["pdf"] },+ logger,+ featureFlags: new Set(["pdf"]),+ pdfPrefetch: {+ filePath,+ url: "https://example.com/doc.pdf",+ status: 200,+ proxyUsed: "basic",+ },+ abort: {+ throwIfAborted: vi.fn(),+ asSignal: vi.fn(() => new AbortController().signal),+ scrapeTimeout: vi.fn(() => 60_000),+ },+ internalOptions: {+ teamId: "team-test",+ ...(teamFlags && { teamFlags }),+ },+ } as any;+}++function nativeResult(pdfType: string, markdown: string) {+ return {+ pdfType,+ confidence: 0.99,+ isComplex: true,+ pageCount: 1,+ markdown,+ title: null,+ pagesNeedingOcr: [],+ logs: [],+ } as any;+}++const reserved = { reservedCapacity: true };++describe("PDF routing for teams restricted to reserved capacity", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ setConfig({+ PDF_RUST_EXTRACT_ENABLE: true,+ FIRE_PDF_ENABLE: true,+ FIRE_PDF_BASE_URL: "http://fire-pdf.test",+ FIRE_PDF_PERCENT: 100,+ FIRE_PDF_ASYNC_PERCENT: 0,+ MINERU_PERCENT: 0,+ RUNPOD_MU_API_KEY: "test-key",+ RUNPOD_MU_POD_ID: "test-endpoint",+ PARSE_PDF_ALWAYS_FIRE_PDF: false,+ });+ mockedProcessPdf.mockResolvedValue(nativeResult("TextBased", "text layer"));+ mockedFirePdf.mockResolvedValue({+ markdown: "fire-pdf",+ html: "<p>fire-pdf</p>",+ pagesProcessed: 1,+ });+ mockedPdfParse.mockResolvedValue({+ markdown: "pdf-parse",+ html: "<p>pdf-parse</p>",+ });+ mockedRunPodMU.mockResolvedValue({+ markdown: "alternate",+ html: "<p>alternate</p>",+ });+ });++ afterEach(async () => {+ await Promise.all(+ tempDirs.splice(0).map(dir => rm(dir, { recursive: true, force: true })),+ );+ });++ afterAll(() => {+ Object.assign(config as any, originalConfig);+ });++ it("never diverts a restricted team to the alternate PDF engine", async () => {+ setConfig({ MINERU_PERCENT: 100 });++ expect(mineruDiverted(await makeMeta())).toBe(true);+ expect(mineruDiverted(await makeMeta(reserved))).toBe(false);++ const result = await scrapePDF(await makeMeta(reserved));++ expect(result.markdown).toBe("fire-pdf");+ expect(mockedFirePdf).toHaveBeenCalledTimes(1);+ expect(mockedRunPodMU).not.toHaveBeenCalled();+ });++ it("sends a restricted PDF to the PDF service regardless of FIRE_PDF_PERCENT", async () => {+ setConfig({ FIRE_PDF_PERCENT: 0, RUNPOD_MU_API_KEY: undefined });++ expect((await scrapePDF(await makeMeta())).markdown).toBe("pdf-parse");+ expect(mockedFirePdf).not.toHaveBeenCalled();++ expect((await scrapePDF(await makeMeta(reserved))).markdown).toBe(+ "fire-pdf",+ );+ expect(mockedFirePdf).toHaveBeenCalledTimes(1);+ });++ it("fails a restricted scrape the PDF service refuses, with no fallback", async () => {+ mockedFirePdf.mockRejectedValueOnce(+ new ReservedCapacityUnavailableError("no reserved capacity"),+ );++ const error = await scrapePDF(await makeMeta(reserved)).catch(e => e);++ expect(error).toBeInstanceOf(ReservedCapacityUnavailableError);+ expect(mockedRunPodMU).not.toHaveBeenCalled();+ expect(mockedPdfParse).not.toHaveBeenCalled();+ });++ it("skips the alternate engine fallback but keeps the local pdf-parse fallback for a restricted team", async () => {+ mockedFirePdf.mockRejectedValue(new Error("fire-pdf 500"));++ expect((await scrapePDF(await makeMeta())).markdown).toBe("alternate");+ expect(mockedRunPodMU).toHaveBeenCalledTimes(1);++ mockedRunPodMU.mockClear();+ expect((await scrapePDF(await makeMeta(reserved))).markdown).toBe(+ "pdf-parse",+ );+ expect(mockedRunPodMU).not.toHaveBeenCalled();+ });++ it.each([+ ["a scanned PDF", "Scanned", "page header"],+ ["a PDF with no text layer", "Mixed", " "],+ ])(+ "refuses %s for a restricted team instead of returning it without OCR",+ async (_name, pdfType, text) => {+ mockedProcessPdf.mockResolvedValue(nativeResult(pdfType, ""));+ mockedFirePdf.mockRejectedValue(new Error("fire-pdf 500"));+ mockedPdfParse.mockResolvedValue({ markdown: text, html: text });++ const error = await scrapePDF(await makeMeta(reserved)).catch(e => e);++ expect(error).toBeInstanceOf(ReservedCapacityUnavailableError);+ expect(error.message).toContain(+ "PDF processing is unavailable for this team's configuration. Please retry later.",+ );+ expect(mockedRunPodMU).not.toHaveBeenCalled();+ },+ );++ it("keeps the US-only OCR error for a team with both restrictions", async () => {+ mockedProcessPdf.mockResolvedValue(nativeResult("Scanned", ""));+ mockedFirePdf.mockRejectedValue(new Error("fire-pdf 500"));+ mockedPdfParse.mockResolvedValue({ markdown: "", html: "" });++ const error = await scrapePDF(+ await makeMeta({ usOnly: true, reservedCapacity: true }),+ ).catch(e => e);++ expect(error).toBeInstanceOf(UsOnlyUnavailableError);+ expect(mockedRunPodMU).not.toHaveBeenCalled();+ });+});diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.tsindex 0d7f51ef4..b6423c0a8 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts@@ -10,7 +10,11 @@ import { cancelJob } from "./cancel";import { maybeSaveResult, provenanceFromResponse, tryGetCached } from "./cache";import { resolvePdfCacheKey } from "../../../../../lib/gcs-pdf-cache";import { isUsOnly, US_DATA_REGION } from "../../../../../lib/us-only";-import { firePdfUsOnlyRefusal } from "./data-region";+import { isReservedCapacity } from "../../../../../lib/reserved-capacity";+import {+ firePdfReservedCapacityRefusal,+ firePdfUsOnlyRefusal,+} from "./data-region";import {firePdfAsyncAbandonedTotal,firePdfAsyncTotalDurationSeconds,@@ -330,7 +334,8 @@ export async function scrapePDFWithFirePDFAsync((error.reason === "terminal_failed" ||error.reason === "terminal_expired" ||error.reason === "terminal_cancelled" ||- error.reason === "region_unavailable");+ error.reason === "region_unavailable" ||+ error.reason === "reserved_unavailable");if ((jobAlreadyTerminal || terminalReached) && meta.largePdfProcessing) {// The job is dead β a "processing continues" message would lie.meta.largePdfProcessing.current = undefined;@@ -380,6 +385,12 @@ export async function scrapePDFWithFirePDFAsync() {throw firePdfUsOnlyRefusal(meta, "region_unavailable");}+ if (+ error instanceof FirePdfAsyncFailure &&+ error.reason === "reserved_unavailable"+ ) {+ throw firePdfReservedCapacityRefusal(meta, "reserved_unavailable");+ }throw submitMayHaveBeenAccepted ? error.originalError : error;}@@ -398,6 +409,12 @@ export async function scrapePDFWithFirePDFAsync() {throw firePdfUsOnlyRefusal(meta, "region_unconfirmed");}+ if (+ isReservedCapacity(meta.internalOptions.teamFlags ?? undefined) &&+ fetched.reserved_applied !== true+ ) {+ throw firePdfReservedCapacityRefusal(meta, "reserved_unconfirmed");+ }// ββ Assemble + cache save βββββββββββββββββββββββββββββββββββββββββββββconst pages =diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/background-refresh.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/background-refresh.tsindex ad62f5210..61a6c30a2 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/background-refresh.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/background-refresh.ts@@ -4,6 +4,7 @@ import type { Meta } from "../../..";import type { PDFMode } from "../../../../../controllers/v2/types";import { config } from "../../../../../config";import { isUsOnly } from "../../../../../lib/us-only";+import { isReservedCapacity } from "../../../../../lib/reserved-capacity";import type { PdfCacheKeyInput } from "../../../../../lib/gcs-pdf-cache";import {type BackgroundRefreshResult,@@ -112,6 +113,10 @@ async function submit(if (isUsOnly(meta.internalOptions.teamFlags ?? undefined)) {return "skipped_us_only";}+ // Nor its reserved capacity echo.+ if (isReservedCapacity(meta.internalOptions.teamFlags ?? undefined)) {+ return "skipped_reserved_capacity";+ }// A by-reference document's bytes are in fire-pdf's input bucket, not here.if (typeof input !== "string") return "skipped_by_reference";// Raster images are only ever parsed through the sync endpoint.@@ -144,8 +149,9 @@ async function submit(tableFormat: undefined,// Internal work under no team: never offered for training capture.trainingCapture: undefined,- // US-only teams never reach here: their refreshes are skipped above.+ // Restricted teams never reach here: their refreshes are skipped above.dataRegion: undefined,+ reserved: undefined,}),refresh: true,},diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference-flow.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference-flow.tsindex 2e9d400fc..3c3c1ff45 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference-flow.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference-flow.ts@@ -11,6 +11,7 @@ import { cacheKeyShape, tryGetCached } from "./cache";import { lookupAdoptableFirePdfJob } from "./lookup";import { buildFirePdfJobOptions, FirePdfAsyncFailure } from "./utils";import { isUsOnly, US_DATA_REGION } from "../../../../../lib/us-only";+import { isReservedCapacity } from "../../../../../lib/reserved-capacity";import {buildFirePdfJobTrainingCapture,firePdfTableFormat,@@ -155,6 +156,9 @@ export async function runFirePdfByReferenceAttempt(dataRegion: isUsOnly(meta.internalOptions.teamFlags ?? undefined)? US_DATA_REGION: undefined,+ reserved: isReservedCapacity(+ meta.internalOptions.teamFlags ?? undefined,+ ),}),);meta.abort.throwIfAborted();diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference.tsindex a2aacae38..6b4603fc1 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference.ts@@ -12,7 +12,7 @@ import {} from "../../../../../controllers/v2/types";import { deterministicPercentage } from "./routing";import { storage } from "../../../../../lib/gcs-jobs";-import { isUsOnly } from "../../../../../lib/us-only";+import { isProcessingRestricted } from "../../../../../lib/reserved-capacity";import { FIRE_PDF_BY_REFERENCE_MAX_FILE_SIZE } from "../types";import { parseAlwaysFirePdf } from "./request-metadata";@@ -202,12 +202,13 @@ export function largePdfLimitBytes(meta: Meta): number {* fire-engine handoff grant, the download admission, and the PDF engine's* routing β sees the SAME verdict; a per-call random draw would let them* drift. Keyed apart from the async-cohort hash. Parse requests that must- * reach FirePDF (parseAlwaysFirePdf) and US-only teams are never diverted. */+ * reach FirePDF (parseAlwaysFirePdf) and teams whose processing is restricted+ * (isProcessingRestricted) are never diverted. */export function mineruDiverted(meta: Meta): boolean {return (config.MINERU_PERCENT > 0 &&!parseAlwaysFirePdf(meta) &&- !isUsOnly(meta.internalOptions.teamFlags ?? undefined) &&+ !isProcessingRestricted(meta.internalOptions.teamFlags ?? undefined) &&deterministicPercentage(`mineru:${meta.id}`) < config.MINERU_PERCENT);}diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache-service.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache-service.tsindex 2ab92c08a..5440c8eeb 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache-service.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache-service.ts@@ -8,7 +8,7 @@ import {} from "../../../../../lib/gcs-pdf-cache";import type { PDFMode } from "../../../../../controllers/v2/types";import { robustFetch } from "../../../lib/fetch";-import { isUsOnly } from "../../../../../lib/us-only";+import { isProcessingRestricted } from "../../../../../lib/reserved-capacity";import {recordSpanException,type Span,@@ -45,6 +45,27 @@ async function cacheLookupKeys(input: PdfCacheKeyInput): Promise<string[]> {];}+/**+ * The keys a cache-service lookup asks with. A PDF's raw-byte key is also+ * recorded on the scrape, so the URL index can later tell whether the cache+ * holds a newer parse of this document.+ */+export async function recordFirePdfCacheKey(+ meta: Meta,+ input: PdfCacheKeyInput,+ sourceKind: FirePdfSourceKind,+): Promise<string[]> {+ const keys = await cacheLookupKeys(input);+ if (+ sourceKind === "pdf" &&+ keys[0].startsWith("raw-") &&+ meta.firePdfCacheKey !== undefined+ ) {+ meta.firePdfCacheKey.current = keys[0];+ }+ return keys;+}+// The service's own read bound, so a lookup never waits longer than the// service would; past it the lookup is a miss and the document is parsed. One// retry, within the same bound, covers a connection that dropped before the@@ -195,21 +216,14 @@ async function lookupWithinSpan(// Background refresh is only offered when this client could act on a// grant: it needs the bytes (inline, not by reference) of a PDF (images// are only parsed through the sync endpoint), outside zero data retention- // and outside US-only teams.+ // and outside teams whose processing is restricted.const canRefreshInBackground =config.FIRE_PDF_CACHE_BACKGROUND_REFRESH === true &&typeof input === "string" &&sourceKind === "pdf" &&meta.internalOptions.zeroDataRetention !== true &&- !isUsOnly(meta.internalOptions.teamFlags ?? undefined);- const keys = await cacheLookupKeys(input);- if (- sourceKind === "pdf" &&- keys[0].startsWith("raw-") &&- meta.firePdfCacheKey !== undefined- ) {- meta.firePdfCacheKey.current = keys[0];- }+ !isProcessingRestricted(meta.internalOptions.teamFlags ?? undefined);+ const keys = await recordFirePdfCacheKey(meta, input, sourceKind);let answer: z.infer<typeof lookupOutcomeSchema>;try {answer = await robustFetch({diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.tsindex af8976c9d..a812489c5 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts@@ -19,7 +19,12 @@ import {firePdfCacheEventsTotal,firePdfCacheRefusedWritesTotal,} from "./metrics";-import { cacheServiceConfigured, lookupCachedResult } from "./cache-service";+import {+ cacheServiceConfigured,+ lookupCachedResult,+ recordFirePdfCacheKey,+} from "./cache-service";+import { isReservedCapacity } from "../../../../../lib/reserved-capacity";import { firePdfTableFormat } from "./request-metadata";import { consumeRefresh, refreshDecisionFor } from "./refresh-budget";import {@@ -206,6 +211,15 @@ export async function tryGetCached(pageMarkers,);if (!cacheable) return null;+ // A stored result may have been produced on other capacity, so a team+ // restricted to reserved capacity is always parsed afresh. Its own+ // results are still written, and the key is still recorded for the index.+ if (isReservedCapacity(meta.internalOptions.teamFlags ?? undefined)) {+ if (viaService && !isRasterImagePayload(base64Content)) {+ await recordFirePdfCacheKey(meta, base64Content, "pdf");+ }+ return null;+ }if (viaService) {// The service probes the variants, applies the read-time rules and// budgets `refresh`; a failed lookup is a miss.diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/data-region.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/data-region.tsindex 784c72655..65e00b0ac 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/data-region.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/data-region.ts@@ -1,5 +1,8 @@import type { Meta } from "../../..";-import { UsOnlyUnavailableError } from "../../../error";+import {+ ReservedCapacityUnavailableError,+ UsOnlyUnavailableError,+} from "../../../error";import type { FirePdfSourceKind } from "./request-metadata";/**@@ -9,6 +12,18 @@ import type { FirePdfSourceKind } from "./request-metadata";* failure: a job that finished in the region can still report refused acquires.*/export function firePdfRegionRefused(body: unknown): boolean {+ return firePdfRestrictionRefused(+ body,+ "data_region_refusals",+ /\[data_region=\w+ refused: /,+ );+}++function firePdfRestrictionRefused(+ body: unknown,+ refusalsField: string,+ notePattern: RegExp,+): boolean {let parsed = body;if (typeof body === "string") {try {@@ -18,20 +33,30 @@ export function firePdfRegionRefused(body: unknown): boolean {}}if (typeof parsed !== "object" || parsed === null) return false;- const { data_region_refusals, error_message } = parsed as Record<- string,- unknown- >;+ const fields = parsed as Record<string, unknown>;+ const refusals = fields[refusalsField];if (- typeof data_region_refusals === "object" &&- data_region_refusals !== null &&- !Array.isArray(data_region_refusals) &&- Object.keys(data_region_refusals).length > 0+ typeof refusals === "object" &&+ refusals !== null &&+ !Array.isArray(refusals) &&+ Object.keys(refusals).length > 0)return true;- return (- typeof error_message === "string" &&- /\[data_region=\w+ refused: /.test(error_message)+ const errorMessage = fields.error_message;+ return typeof errorMessage === "string" && notePattern.test(errorMessage);+}++/**+ * Whether a PDF service failure restricted to reserved capacity says the+ * capacity it needed there was refused: `reserved_refusals` on a sync error+ * body, or the refusal note a failed job appends to its `error_message`.+ * Only call it on a failure.+ */+export function firePdfReservedRefused(body: unknown): boolean {+ return firePdfRestrictionRefused(+ body,+ "reserved_refusals",+ /\[reserved refused: /,);}@@ -55,3 +80,17 @@ export function firePdfUsOnlyRefusal(`${feature} is not available for teams restricted to US-only processing right now: ${detail}. Try again later.`,);}++/** The error a team restricted to reserved capacity gets when the PDF service did not run, or did not confirm running, its document there. */+export function firePdfReservedCapacityRefusal(+ meta: Meta,+ reason: "reserved_unavailable" | "reserved_unconfirmed",+ sourceKind: FirePdfSourceKind = "pdf",+): ReservedCapacityUnavailableError {+ meta.logger.warn("FirePDF could not keep a request on reserved capacity", {+ scrapeId: meta.id,+ reason,+ sourceKind,+ });+ return new ReservedCapacityUnavailableError();+}diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.tsindex bb39bed16..77c5290af 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.ts@@ -14,7 +14,7 @@ export const firePdfCacheRefusedWritesTotal = new Counter({export const firePdfCacheBackgroundRefreshTotal = new Counter({name: "firecrawl_fire_pdf_cache_background_refresh_total",- help: "Background refreshes the fire-pdf cache service granted on a stale answer, by result: submitted, submit_failed, skipped_by_reference, skipped_zdr, skipped_image, skipped_us_only",+ help: "Background refreshes the fire-pdf cache service granted on a stale answer, by result: submitted, submit_failed, skipped_by_reference, skipped_zdr, skipped_image, skipped_us_only, skipped_reserved_capacity",labelNames: ["result"],});@@ -24,7 +24,8 @@ export type BackgroundRefreshResult =| "skipped_by_reference"| "skipped_zdr"| "skipped_image"- | "skipped_us_only";+ | "skipped_us_only"+ | "skipped_reserved_capacity";export type CacheRefusedReason =| "failed_pages"@@ -124,4 +125,5 @@ export type FallbackReason =| "terminal_cancelled"| "polling_timeout"| "result_503"- | "region_unavailable";+ | "region_unavailable"+ | "reserved_unavailable";diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/poll.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/poll.tsindex 782851c50..8fb20d2ce 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/poll.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/poll.ts@@ -2,7 +2,8 @@ import type { Meta } from "../../..";import { fetch as undiciFetch } from "undici";import { AbortManagerThrownError } from "../../../lib/abortManager";import { isUsOnly } from "../../../../../lib/us-only";-import { firePdfRegionRefused } from "./data-region";+import { isReservedCapacity } from "../../../../../lib/reserved-capacity";+import { firePdfReservedRefused, firePdfRegionRefused } from "./data-region";import {firePdfAsyncCompletedTotal,firePdfAsyncLongPollTotal,@@ -184,6 +185,20 @@ export async function pollUntilTerminal(deps: PollDeps): Promise<PollOk> {failAsync(meta, "region_unavailable", { pollCount, body: pollBody });}+ if (+ isReservedCapacity(meta.internalOptions.teamFlags ?? undefined) &&+ (pollBody as { status?: unknown }).status === "failed" &&+ firePdfReservedRefused(pollBody)+ ) {+ if (waitMs > 0) firePdfAsyncLongPollTotal.labels("terminal").inc();+ firePdfAsyncPollCount.observe(pollCount);+ firePdfAsyncCompletedTotal.labels("failed").inc();+ failAsync(meta, "reserved_unavailable", {+ pollCount,+ body: pollBody,+ });+ }+if (pollStatus === 401) {firePdfAsyncPollCount.observe(pollCount);failAsync(meta, "http_401", { pollCount });diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.tsindex 37cb1feb3..4a50793f4 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts@@ -302,6 +302,7 @@ export const resultResponseSchema = z.object({// build does not understand never fails the scrape.provenance: z.unknown().optional(),data_region_applied: z.unknown().optional(),+ reserved_applied: z.unknown().optional(),});export type PollResponse = z.infer<typeof pollResponseSchema>;diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.tsindex 2de06c019..c349d1c4e 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts@@ -3,6 +3,7 @@ import type { PDFMode } from "../../../../../controllers/v2/types";import { fetch as undiciFetch } from "undici";import { AbortManagerThrownError } from "../../../lib/abortManager";import { isUsOnly, US_DATA_REGION } from "../../../../../lib/us-only";+import { isReservedCapacity } from "../../../../../lib/reserved-capacity";import {buildFirePdfJobTrainingCapture,buildFirePdfRequestMetadata,@@ -120,6 +121,9 @@ export async function submitJob(args: SubmitArgs): Promise<SubmitOutcome> {} = args;const scrapeId = meta.id;const usOnly = isUsOnly(meta.internalOptions.teamFlags ?? undefined);+ const reservedCapacity = isReservedCapacity(+ meta.internalOptions.teamFlags ?? undefined,+ );if (input.kind === "byReference" &&@@ -166,6 +170,7 @@ export async function submitJob(args: SubmitArgs): Promise<SubmitOutcome> {tableFormat: firePdfTableFormat(meta),trainingCapture: buildFirePdfJobTrainingCapture(meta),dataRegion: usOnly ? US_DATA_REGION : undefined,+ reserved: reservedCapacity,}),};diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.tsindex ff953b0fc..950c7b87a 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.ts@@ -162,6 +162,7 @@ export function buildFirePdfJobOptions(args: {tableFormat: string | undefined;trainingCapture: FirePdfTrainingCapture | undefined;dataRegion: string | undefined;+ reserved?: boolean;}): Record<string, unknown> {return {// fire-pdf accepts only a positive integer here. When the router has no@@ -188,6 +189,9 @@ export function buildFirePdfJobOptions(args: {// An idempotency key on the PDF service, so a pinned lookup only adopts// a pinned job....(args.dataRegion && { data_region: args.dataRegion }),+ // Also an idempotency key, so a restricted lookup only adopts a job+ // restricted the same way.+ ...(args.reserved && { reserved: true }),};}diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.tsindex fca9490aa..3478b7104 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts@@ -7,6 +7,7 @@ import type { PDFMode } from "../../../../controllers/v2/types";import { safeMarkdownToHtml } from "./markdownToHtml";import { createPdfCacheKey } from "../../../../lib/gcs-pdf-cache";import { isUsOnly, US_DATA_REGION } from "../../../../lib/us-only";+import { isReservedCapacity } from "../../../../lib/reserved-capacity";import {maybeSaveResult,provenanceFromResponse,@@ -20,6 +21,8 @@ import {type FirePdfSourceKind,} from "./fire-pdf/request-metadata";import {+ firePdfReservedRefused,+ firePdfReservedCapacityRefusal,firePdfRegionRefused,firePdfUsOnlyRefusal,} from "./fire-pdf/data-region";@@ -113,6 +116,9 @@ export async function scrapePDFWithFirePDF(const zdr = meta.internalOptions.zeroDataRetention === true;const usOnly = isUsOnly(meta.internalOptions.teamFlags ?? undefined);+ const reservedCapacity = isReservedCapacity(+ meta.internalOptions.teamFlags ?? undefined,+ );const pdfSha256 = createPdfCacheKey(base64Content);const tableFormat = firePdfTableFormat(meta);@@ -164,6 +170,7 @@ export async function scrapePDFWithFirePDF(training_capture: buildFirePdfTrainingCapture(meta),...deadlineFields,...(usOnly && { data_region: US_DATA_REGION }),+ ...(reservedCapacity && { reserved: true }),},logger,schema: z.object({@@ -185,19 +192,23 @@ export async function scrapePDFWithFirePDF(// option β older builds ignore unknown request fields and omit it.page_markers: z.literal(true).optional(),data_region_applied: z.unknown().optional(),+ reserved_applied: z.unknown().optional(),}),mock: meta.mock,abort: meta.abort.asSignal(),}).catch(error => {- if (- usOnly &&- firePdfRegionRefused(- (error as { cause?: { response?: { body?: unknown } } })?.cause- ?.response?.body,- )- ) {+ const body = (error as { cause?: { response?: { body?: unknown } } })?.cause+ ?.response?.body;+ if (usOnly && firePdfRegionRefused(body)) {throw firePdfUsOnlyRefusal(meta, "region_unavailable", sourceKind);}+ if (reservedCapacity && firePdfReservedRefused(body)) {+ throw firePdfReservedCapacityRefusal(+ meta,+ "reserved_unavailable",+ sourceKind,+ );+ }throw error;});// A build that ignores the pin answers without the echo; its result may@@ -205,6 +216,15 @@ export async function scrapePDFWithFirePDF(if (usOnly && resp.data_region_applied !== US_DATA_REGION) {throw firePdfUsOnlyRefusal(meta, "region_unconfirmed", sourceKind);}+ // Likewise, without the reserved echo the result may have come from other+ // capacity.+ if (reservedCapacity && resp.reserved_applied !== true) {+ throw firePdfReservedCapacityRefusal(+ meta,+ "reserved_unconfirmed",+ sourceKind,+ );+ }const durationMs = Date.now() - startedAt;if (includePageMarkdown && resp.pages === undefined) {diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex a208996f9..eec65e497 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@ -17,6 +17,7 @@ import {EngineUnsuccessfulError,UnsupportedFileError,UsOnlyUnavailableError,+ ReservedCapacityUnavailableError,} from "../../error";import { open, readFile, stat, unlink } from "node:fs/promises";import type { Response } from "undici";@@ -47,6 +48,7 @@ import {} from "../../../../lib/native-logging";import { withSpan, setSpanAttributes } from "../../../../lib/otel-tracer";import { isUsOnly } from "../../../../lib/us-only";+import { isProcessingRestricted } from "../../../../lib/reserved-capacity";import { scrapePDFWithRunPodMU } from "./runpodMU";import { useFireEngine } from "../fire-engine/available";import { reconcilePageCountWithFirePdf, scrapePDFWithFirePDF } from "./firePDF";@@ -247,8 +249,12 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {// US-only teams get GPU parsing only from the PDF service, pinned to US// capacity: no diversion to another engine, no serverless fallback, and no- // sampling out of the PDF service.+ // sampling out of the PDF service. Teams restricted to reserved capacity are+ // routed the same way.const usOnly = isUsOnly(meta.internalOptions.teamFlags ?? undefined);+ const processingRestricted = isProcessingRestricted(+ meta.internalOptions.teamFlags ?? undefined,+ );// The MinerU diversion is deterministic on the scrape id (see// mineruDiverted) so this routing verdict is BY CONSTRUCTION the same@@ -634,7 +640,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {fileSizeBytes <= FIRE_PDF_INLINE_HARD_MAX_FILE_SIZE));const runpodMuUsable =!forceFirePDF &&- !usOnly &&+ !processingRestricted &&fileSizeBytes < MAX_FILE_SIZE &&!!config.RUNPOD_MU_API_KEY &&!!config.RUNPOD_MU_POD_ID;@@ -682,7 +688,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {config.FIRE_PDF_BASE_URL &&fileSizeBytes < FIRE_PDF_MAX_FILE_SIZE &&(alwaysFirePdf ||- usOnly ||+ processingRestricted ||Math.random() * 100 < config.FIRE_PDF_PERCENT)));if (useFirePDF) {@@ -749,7 +755,8 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {(!includePageMarkdown && !includeBlocks && !pageMarkers) ||error instanceof RemoveFeatureError ||error instanceof AbortManagerThrownError ||- error instanceof UsOnlyUnavailableError+ error instanceof UsOnlyUnavailableError ||+ error instanceof ReservedCapacityUnavailableError) {throw error;}@@ -798,7 +805,8 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {if (error instanceof RemoveFeatureError ||error instanceof AbortManagerThrownError ||- error instanceof UsOnlyUnavailableError+ error instanceof UsOnlyUnavailableError ||+ error instanceof ReservedCapacityUnavailableError) {throw error;}@@ -835,7 +843,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {if (!result &&!forceFirePDF &&- !usOnly &&+ !processingRestricted &&fileSizeBytes >= MAX_FILE_SIZE &&config.RUNPOD_MU_API_KEY &&config.RUNPOD_MU_POD_ID@@ -854,7 +862,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {if (!result &&!forceFirePDF &&- !usOnly &&+ !processingRestricted &&fileSizeBytes < MAX_FILE_SIZE &&base64Content !== undefined &&config.RUNPOD_MU_API_KEY &&@@ -972,20 +980,34 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {}// Without GPU OCR a scanned document has no text layer to return.if (- usOnly &&+ processingRestricted &&!skipOCR &&(detectedPdfType === "Scanned" ||detectedPdfType === "ImageBased" ||!result.markdown?.trim())) {- meta.logger.warn("US-only PDF needs OCR the PDF service did not run", {- method: "scrapePDF",- scrape_id: meta.id,- pdf_type: detectedPdfType,- });- throw new UsOnlyUnavailableError(- "This PDF needs OCR, which is not available for teams restricted to US-only processing right now. Try again later.",+ if (usOnly) {+ meta.logger.warn(+ "US-only PDF needs OCR the PDF service did not run",+ {+ method: "scrapePDF",+ scrape_id: meta.id,+ pdf_type: detectedPdfType,+ },+ );+ throw new UsOnlyUnavailableError(+ "This PDF needs OCR, which is not available for teams restricted to US-only processing right now. Try again later.",+ );+ }+ meta.logger.warn(+ "PDF restricted to reserved capacity needs OCR the PDF service did not run",+ {+ method: "scrapePDF",+ scrape_id: meta.id,+ pdf_type: detectedPdfType,+ },);+ throw new ReservedCapacityUnavailableError();}}diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.tsindex 80cbb929a..0dd5ab7af 100644--- a/apps/api/src/scraper/scrapeURL/error.ts+++ b/apps/api/src/scraper/scrapeURL/error.ts@@ -47,6 +47,24 @@ export class UsOnlyUnavailableError extends TransportableError {}}+/** Parsing the team's reserved capacity restriction rules out, because the PDF service could not run, or did not confirm running, it there. */+export class ReservedCapacityUnavailableError extends TransportableError {+ constructor(+ message = "PDF processing is unavailable for this team's configuration. Please retry later.",+ ) {+ super("RESERVED_CAPACITY_UNAVAILABLE", message);+ }++ static deserialize(+ _: ErrorCodes,+ data: ReturnType<typeof this.prototype.serialize>,+ ) {+ const x = new ReservedCapacityUnavailableError(data.message);+ x.stack = data.stack;+ return x;+ }+}+export class NoEnginesLeftError extends TransportableError {public fallbackList: Engine[];diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 1ecad12c7..0afd13e55 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -61,6 +61,7 @@ import {BrandingNotSupportedError,XTwitterConfigurationError,UsOnlyUnavailableError,+ ReservedCapacityUnavailableError,ExchangeRefusedError,} from "./error";import { ScrapeRetryTracker } from "./retryTracker";@@ -1067,6 +1068,7 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {error.error instanceof AgentIndexOnlyError ||error.error instanceof XTwitterConfigurationError ||error.error instanceof UsOnlyUnavailableError ||+ error.error instanceof ReservedCapacityUnavailableError ||error.error instanceof ExchangeRefusedError ||error.error instanceof ThirdPartyDataTermsRequiredError) {