Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

feat(api): fire-pdf async client behind opt-in flag (#3526)

Abimael Martell63eeeedd08cd
diff --git a/apps/api/src/__tests__/snips/v2/parsers-fire-pdf-async.test.ts b/apps/api/src/__tests__/snips/v2/parsers-fire-pdf-async.test.ts
new file mode 100644
index 000000000..d753c8572
--- /dev/null
+++ b/apps/api/src/__tests__/snips/v2/parsers-fire-pdf-async.test.ts
@@ -0,0 +1,54 @@
+import {
+ ALLOW_TEST_SUITE_WEBSITE,
+ describeIf,
+ TEST_SUITE_WEBSITE,
+} from "../lib";
+import { config } from "../../../config";
+import { scrape, scrapeTimeout, idmux, Identity } from "./lib";
+
+let identity: Identity;
+
+beforeAll(async () => {
+ identity = await idmux({
+ name: "parsers-fire-pdf-async",
+ concurrency: 100,
+ credits: 1000000,
+ });
+}, 10000 + scrapeTimeout);
+
+// Requires fire-engine (PDF is downloaded via the same path that needs it in
+// CI) AND a fire-pdf base URL β€” the async client is only reached when
+// `useFirePDF` is true in the call site, which gates on FIRE_PDF_BASE_URL.
+//
+// Behavior: the async client falls back to sync /ocr on any failure (including
+// 404 when /jobs isn't deployed), so this test asserts the user-visible
+// contract (markdown comes back), not which underlying engine served it.
+// That keeps the test stable in environments where fire-pdf staging hasn't
+// flipped /jobs on yet, while still verifying the flag is wired all the way
+// through.
+const SHOULD_RUN =
+ ALLOW_TEST_SUITE_WEBSITE &&
+ !process.env.TEST_SUITE_SELF_HOSTED &&
+ !!config.FIRE_PDF_BASE_URL;
+
+describeIf(SHOULD_RUN)("fire-pdf async parser flag", () => {
+ const pdfUrl = `${TEST_SUITE_WEBSITE}/example.pdf`;
+
+ it.concurrent(
+ "parsers: [{type:'pdf', __firePdfAsync: true}] returns markdown (async path or fallback)",
+ async () => {
+ const response = await scrape(
+ {
+ url: pdfUrl,
+ parsers: [{ type: "pdf", __firePdfAsync: true }],
+ },
+ identity,
+ );
+
+ expect(response.markdown).toBeDefined();
+ expect(response.markdown).toContain("PDF Test File");
+ expect(response.metadata.numPages).toBeGreaterThan(0);
+ },
+ scrapeTimeout * 3,
+ );
+});
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index c86ad9832..9023468ec 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -449,6 +449,11 @@ const pdfParserWithOptions = z.strictObject({
type: z.literal("pdf"),
mode: pdfModeSchema.optional(),
maxPages: z.int().positive().finite().max(10000).optional(),
+ // Experimental: route this request through the fire-pdf async pipeline
+ // (POST /jobs + poll) instead of the sync POST /ocr endpoint. Falls back
+ // to sync on any async-path failure, so user-visible behavior is unchanged
+ // beyond latency variance. Underscored to mark as internal/experimental.
+ __firePdfAsync: z.boolean().optional(),
});
const parsersSchema = z
@@ -493,6 +498,17 @@ export function getPDFMode(parsers?: Parsers): PDFMode {
return "auto";
}
+export function getFirePdfAsync(parsers?: Parsers): boolean {
+ if (!parsers) return false;
+ for (const parser of parsers) {
+ if (parser === "pdf") return false;
+ if (typeof parser === "object" && parser.type === "pdf") {
+ return parser.__firePdfAsync === true;
+ }
+ }
+ return false;
+}
+
function transformIframeSelector(selector: string): string {
return selector.replace(/(?:^|[\s,])iframe(?=\s|$|[.#\[:,])/g, match => {
const prefix = match.match(/^[\s,]/)?.[0] || "";
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts
new file mode 100644
index 000000000..dca6581f7
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts
@@ -0,0 +1,565 @@
+// Stub the GCS cache so unit tests never reach real cloud storage. The async
+// client calls these on the happy path; without the stub the first cache lookup
+// blows up trying to download from GCS using the credentials in .env.
+jest.mock("../../../../../lib/gcs-pdf-cache", () => ({
+ createPdfCacheKey: (s: string) => `sha-${s.length}`,
+ getPdfResultFromCache: jest.fn(async () => null),
+ savePdfResultToCache: jest.fn(async () => null),
+}));
+
+import { scrapePDFWithFirePDFAsync } from "../fire-pdf/async";
+import { config } from "../../../../../config";
+
+// ── Fixtures ─────────────────────────────────────────────────────────────
+
+const BASE_URL_ENV = "FIRE_PDF_BASE_URL";
+const ORIGINAL_BASE_URL = process.env[BASE_URL_ENV];
+
+beforeAll(() => {
+ // Tests build URLs against this; the config object reads the env via zod
+ // at module init, so set both for safety.
+ process.env[BASE_URL_ENV] = "http://fire-pdf.test";
+ (config as { FIRE_PDF_BASE_URL?: string }).FIRE_PDF_BASE_URL =
+ "http://fire-pdf.test";
+});
+
+afterAll(() => {
+ if (ORIGINAL_BASE_URL === undefined) {
+ delete process.env[BASE_URL_ENV];
+ } else {
+ process.env[BASE_URL_ENV] = ORIGINAL_BASE_URL;
+ }
+});
+
+type FakeResponse = {
+ status: number;
+ body: unknown;
+};
+
+function jsonResp({ status, body }: FakeResponse) {
+ return {
+ status,
+ json: async () => body,
+ } as any;
+}
+
+function makeMeta(overrides: Record<string, unknown> = {}) {
+ const noopLogger: any = {
+ info: jest.fn(),
+ warn: jest.fn(),
+ error: jest.fn(),
+ debug: jest.fn(),
+ child: jest.fn(function child() {
+ return noopLogger;
+ }),
+ };
+
+ return {
+ id: "scrape-id-test",
+ url: "https://example.com/doc.pdf",
+ rewrittenUrl: undefined,
+ logger: noopLogger,
+ mock: null,
+ abort: {
+ throwIfAborted: jest.fn(),
+ asSignal: jest.fn(() => new AbortController().signal),
+ scrapeTimeout: jest.fn(() => 60_000),
+ },
+ internalOptions: {
+ zeroDataRetention: false,
+ teamId: "team-x",
+ crawlId: undefined,
+ },
+ options: {
+ parsers: [{ type: "pdf", __firePdfAsync: true }],
+ },
+ ...overrides,
+ } as any;
+}
+
+function makeFetchFromSequence(
+ matchers: Array<{
+ matchUrl: RegExp;
+ matchMethod?: "GET" | "POST";
+ response: FakeResponse | (() => FakeResponse);
+ }>,
+) {
+ const calls: Array<{ url: string; method: string }> = [];
+ const cursor = { idx: 0 };
+ const fetchImpl: any = async (url: string, init: any) => {
+ const method = (init?.method ?? "GET").toUpperCase();
+ calls.push({ url, method });
+ const matcher = matchers[cursor.idx++];
+ if (!matcher) {
+ throw new Error(
+ `unexpected request #${cursor.idx} to ${method} ${url} (no matcher left)`,
+ );
+ }
+ if (!matcher.matchUrl.test(url)) {
+ throw new Error(
+ `request ${cursor.idx} url mismatch: got ${url}, expected ${matcher.matchUrl}`,
+ );
+ }
+ if (matcher.matchMethod && matcher.matchMethod !== method) {
+ throw new Error(
+ `request ${cursor.idx} method mismatch: got ${method}, expected ${matcher.matchMethod}`,
+ );
+ }
+ const r =
+ typeof matcher.response === "function"
+ ? matcher.response()
+ : matcher.response;
+ return jsonResp(r);
+ };
+ return { fetchImpl, calls };
+}
+
+const noopSleep = async () => {};
+
+// ── Tests ────────────────────────────────────────────────────────────────
+
+describe("scrapePDFWithFirePDFAsync", () => {
+ it("happy path: POST 202 queued β†’ poll done β†’ result returns markdown", async () => {
+ const { fetchImpl, calls } = makeFetchFromSequence([
+ {
+ matchUrl: /\/jobs$/,
+ matchMethod: "POST",
+ response: {
+ status: 202,
+ body: {
+ scrape_id: "scrape-id-test",
+ status: "queued",
+ lane: "fast",
+ retry_after_ms: 50,
+ },
+ },
+ },
+ {
+ matchUrl: /\/jobs\/scrape-id-test$/,
+ matchMethod: "GET",
+ response: { status: 202, body: { scrape_id: "x", status: "running" } },
+ },
+ {
+ matchUrl: /\/jobs\/scrape-id-test$/,
+ matchMethod: "GET",
+ response: {
+ status: 200,
+ body: {
+ scrape_id: "x",
+ status: "done",
+ pages_processed: 12,
+ },
+ },
+ },
+ {
+ matchUrl: /\/jobs\/scrape-id-test\/result$/,
+ matchMethod: "GET",
+ response: {
+ status: 200,
+ body: {
+ schema_version: 1,
+ markdown: "# Hello async",
+ pages_processed: 12,
+ failed_pages: null,
+ partial_pages: null,
+ },
+ },
+ },
+ ]);
+ const fallback = jest.fn();
+
+ const result = await scrapePDFWithFirePDFAsync(
+ makeMeta(),
+ "BASE64",
+ undefined,
+ undefined,
+ undefined,
+ { fetchImpl, fallbackImpl: fallback, sleepImpl: noopSleep },
+ );
+
+ expect(result.markdown).toBe("# Hello async");
+ expect(result.pagesProcessed).toBe(12);
+ expect(fallback).not.toHaveBeenCalled();
+ expect(calls).toHaveLength(4);
+ });
+
+ it("idempotent replay: POST 200 done skips polling and fetches result", async () => {
+ const { fetchImpl, calls } = makeFetchFromSequence([
+ {
+ matchUrl: /\/jobs$/,
+ matchMethod: "POST",
+ response: {
+ status: 200,
+ body: { scrape_id: "scrape-id-test", status: "done" },
+ },
+ },
+ {
+ matchUrl: /\/jobs\/scrape-id-test\/result$/,
+ matchMethod: "GET",
+ response: {
+ status: 200,
+ body: { markdown: "cached", pages_processed: 3 },
+ },
+ },
+ ]);
+ const fallback = jest.fn();
+
+ const result = await scrapePDFWithFirePDFAsync(
+ makeMeta(),
+ "BASE64",
+ undefined,
+ undefined,
+ undefined,
+ { fetchImpl, fallbackImpl: fallback, sleepImpl: noopSleep },
+ );
+
+ expect(result.markdown).toBe("cached");
+ expect(fallback).not.toHaveBeenCalled();
+ expect(calls).toHaveLength(2);
+ });
+
+ it.each([
+ ["404", 404],
+ ["413", 413],
+ ["429", 429],
+ ["503", 503],
+ ["generic 5xx", 500],
+ ])("falls back to sync when POST /jobs returns %s", async (_, status) => {
+ const { fetchImpl } = makeFetchFromSequence([
+ {
+ matchUrl: /\/jobs$/,
+ matchMethod: "POST",
+ response: { status, body: { error: "x" } },
+ },
+ ]);
+ const fallback = jest.fn(async () => ({
+ markdown: "sync-result",
+ html: "<p>sync-result</p>",
+ pagesProcessed: 1,
+ }));
+
+ const result = await scrapePDFWithFirePDFAsync(
+ makeMeta(),
+ "BASE64",
+ undefined,
+ undefined,
+ undefined,
+ { fetchImpl, fallbackImpl: fallback, sleepImpl: noopSleep },
+ );
+
+ expect(fallback).toHaveBeenCalledTimes(1);
+ expect(result.markdown).toBe("sync-result");
+ });
+
+ it("falls back to sync when POST /jobs throws a network error", async () => {
+ const fetchImpl: any = async () => {
+ throw new Error("connect ECONNREFUSED");
+ };
+ const fallback = jest.fn(async () => ({
+ markdown: "sync",
+ html: "",
+ pagesProcessed: 0,
+ }));
+
+ const result = await scrapePDFWithFirePDFAsync(
+ makeMeta(),
+ "BASE64",
+ undefined,
+ undefined,
+ undefined,
+ { fetchImpl, fallbackImpl: fallback, sleepImpl: noopSleep },
+ );
+
+ expect(fallback).toHaveBeenCalledTimes(1);
+ expect(result.markdown).toBe("sync");
+ });
+
+ it("throws on POST 409 scrape_id conflict (fatal, no fallback)", async () => {
+ const { fetchImpl } = makeFetchFromSequence([
+ {
+ matchUrl: /\/jobs$/,
+ matchMethod: "POST",
+ response: {
+ status: 409,
+ body: { error: "scrape_id_conflict", conflict_fields: ["pdf_b64"] },
+ },
+ },
+ ]);
+ const fallback = jest.fn();
+
+ await expect(
+ scrapePDFWithFirePDFAsync(
+ makeMeta(),
+ "BASE64",
+ undefined,
+ undefined,
+ undefined,
+ { fetchImpl, fallbackImpl: fallback, sleepImpl: noopSleep },
+ ),
+ ).rejects.toThrow(/conflict/);
+ expect(fallback).not.toHaveBeenCalled();
+ });
+
+ it("falls back when polling returns terminal failed (502)", async () => {
+ const { fetchImpl } = makeFetchFromSequence([
+ {
+ matchUrl: /\/jobs$/,
+ response: {
+ status: 202,
+ body: { scrape_id: "x", status: "queued" },
+ },
+ },
+ {
+ matchUrl: /\/jobs\/scrape-id-test$/,
+ response: {
+ status: 502,
+ body: {
+ scrape_id: "x",
+ status: "failed",
+ error_class: "worker_oom",
+ error_message: "ran out of memory",
+ },
+ },
+ },
+ ]);
+ const fallback = jest.fn(async () => ({
+ markdown: "sync",
+ html: "",
+ pagesProcessed: 0,
+ }));
+
+ const result = await scrapePDFWithFirePDFAsync(
+ makeMeta(),
+ "BASE64",
+ undefined,
+ undefined,
+ undefined,
+ { fetchImpl, fallbackImpl: fallback, sleepImpl: noopSleep },
+ );
+
+ expect(fallback).toHaveBeenCalledTimes(1);
+ expect(result.markdown).toBe("sync");
+ });
+
+ it("falls back when polling returns 410 (expired/cancelled)", async () => {
+ const { fetchImpl } = makeFetchFromSequence([
+ {
+ matchUrl: /\/jobs$/,
+ response: {
+ status: 202,
+ body: { scrape_id: "x", status: "queued" },
+ },
+ },
+ {
+ matchUrl: /\/jobs\/scrape-id-test$/,
+ response: {
+ status: 410,
+ body: { scrape_id: "x", status: "expired" },
+ },
+ },
+ ]);
+ const fallback = jest.fn(async () => ({
+ markdown: "sync",
+ html: "",
+ pagesProcessed: 0,
+ }));
+
+ const result = await scrapePDFWithFirePDFAsync(
+ makeMeta(),
+ "BASE64",
+ undefined,
+ undefined,
+ undefined,
+ { fetchImpl, fallbackImpl: fallback, sleepImpl: noopSleep },
+ );
+
+ expect(fallback).toHaveBeenCalledTimes(1);
+ expect(result.markdown).toBe("sync");
+ });
+
+ it("falls back when polling exceeds deadline + buffer", async () => {
+ let virtualNow = 1_000_000;
+ const advance = (ms: number) => {
+ virtualNow += ms;
+ };
+
+ const { fetchImpl } = makeFetchFromSequence([
+ {
+ matchUrl: /\/jobs$/,
+ response: {
+ status: 202,
+ body: {
+ scrape_id: "x",
+ status: "queued",
+ retry_after_ms: 1000,
+ },
+ },
+ },
+ // Subsequent polls β€” won't be reached if timeout triggers correctly,
+ // but provide one just in case the loop runs one iteration.
+ {
+ matchUrl: /\/jobs\/scrape-id-test$/,
+ response: { status: 202, body: { scrape_id: "x", status: "running" } },
+ },
+ ]);
+ const fallback = jest.fn(async () => ({
+ markdown: "sync",
+ html: "",
+ pagesProcessed: 0,
+ }));
+
+ // 5s scrape budget β†’ deadline 5s, polling deadline = submit + 5s + 30s = 35s.
+ // Each sleep advances time by 60s, blowing past the polling deadline.
+ const meta = makeMeta({
+ abort: {
+ throwIfAborted: jest.fn(),
+ asSignal: jest.fn(() => new AbortController().signal),
+ scrapeTimeout: jest.fn(() => 5_000),
+ },
+ });
+
+ const result = await scrapePDFWithFirePDFAsync(
+ meta,
+ "BASE64",
+ undefined,
+ undefined,
+ undefined,
+ {
+ fetchImpl,
+ fallbackImpl: fallback,
+ sleepImpl: async ms => advance(ms + 60_000),
+ nowImpl: () => virtualNow,
+ },
+ );
+
+ expect(fallback).toHaveBeenCalledTimes(1);
+ expect(result.markdown).toBe("sync");
+ });
+
+ it("falls back when result endpoint returns 503", async () => {
+ const { fetchImpl } = makeFetchFromSequence([
+ {
+ matchUrl: /\/jobs$/,
+ response: {
+ status: 202,
+ body: { scrape_id: "x", status: "queued" },
+ },
+ },
+ {
+ matchUrl: /\/jobs\/scrape-id-test$/,
+ response: {
+ status: 200,
+ body: { scrape_id: "x", status: "done", pages_processed: 5 },
+ },
+ },
+ {
+ matchUrl: /\/jobs\/scrape-id-test\/result$/,
+ response: { status: 503, body: { error: "gcs_unreachable" } },
+ },
+ ]);
+ const fallback = jest.fn(async () => ({
+ markdown: "sync",
+ html: "",
+ pagesProcessed: 0,
+ }));
+
+ const result = await scrapePDFWithFirePDFAsync(
+ makeMeta(),
+ "BASE64",
+ undefined,
+ undefined,
+ undefined,
+ { fetchImpl, fallbackImpl: fallback, sleepImpl: noopSleep },
+ );
+
+ expect(fallback).toHaveBeenCalledTimes(1);
+ expect(result.markdown).toBe("sync");
+ });
+
+ it("re-polls once on result 409, then succeeds", async () => {
+ const { fetchImpl, calls } = makeFetchFromSequence([
+ {
+ matchUrl: /\/jobs$/,
+ matchMethod: "POST",
+ response: {
+ status: 202,
+ body: { scrape_id: "x", status: "queued" },
+ },
+ },
+ {
+ matchUrl: /\/jobs\/scrape-id-test$/,
+ matchMethod: "GET",
+ response: {
+ status: 200,
+ body: { scrape_id: "x", status: "done", pages_processed: 7 },
+ },
+ },
+ {
+ matchUrl: /\/jobs\/scrape-id-test\/result$/,
+ matchMethod: "GET",
+ response: { status: 409, body: { error: "status_flipped" } },
+ },
+ {
+ matchUrl: /\/jobs\/scrape-id-test\/result$/,
+ matchMethod: "GET",
+ response: {
+ status: 200,
+ body: { markdown: "ok", pages_processed: 7 },
+ },
+ },
+ ]);
+ const fallback = jest.fn();
+
+ const result = await scrapePDFWithFirePDFAsync(
+ makeMeta(),
+ "BASE64",
+ undefined,
+ undefined,
+ undefined,
+ { fetchImpl, fallbackImpl: fallback, sleepImpl: noopSleep },
+ );
+
+ expect(result.markdown).toBe("ok");
+ expect(fallback).not.toHaveBeenCalled();
+ expect(calls).toHaveLength(4);
+ });
+
+ it("deadline_at is within the spec'd [5s, 30min] window", async () => {
+ let submittedBody: any;
+ const fetchImpl: any = async (url: string, init: any) => {
+ if (/\/jobs$/.test(url) && (init?.method ?? "GET") === "POST") {
+ submittedBody = JSON.parse(init.body as string);
+ return jsonResp({
+ status: 202,
+ body: { scrape_id: "x", status: "queued" },
+ });
+ }
+ if (/\/jobs\/scrape-id-test$/.test(url)) {
+ return jsonResp({
+ status: 200,
+ body: { scrape_id: "x", status: "done", pages_processed: 1 },
+ });
+ }
+ return jsonResp({
+ status: 200,
+ body: { markdown: "ok", pages_processed: 1 },
+ });
+ };
+
+ await scrapePDFWithFirePDFAsync(
+ makeMeta(),
+ "BASE64",
+ undefined,
+ undefined,
+ undefined,
+ { fetchImpl, fallbackImpl: jest.fn(), sleepImpl: noopSleep },
+ );
+
+ const deadlineMs = new Date(submittedBody.deadline_at).getTime();
+ const delta = deadlineMs - Date.now();
+ // Within the spec: 5s < delta < 30min. Loose lower bound because clock
+ // can drift slightly during the test.
+ expect(delta).toBeGreaterThanOrEqual(5_000 - 100);
+ expect(delta).toBeLessThanOrEqual(30 * 60 * 1_000);
+ });
+});
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts
new file mode 100644
index 000000000..c9d8a25eb
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts
@@ -0,0 +1,155 @@
+import { Meta } from "../../..";
+import { config } from "../../../../../config";
+import { fetch as undiciFetch } from "undici";
+import type { PDFProcessorResult } from "../types";
+import type { PDFMode } from "../../../../../controllers/v2/types";
+import { safeMarkdownToHtml } from "../markdownToHtml";
+import { scrapePDFWithFirePDF } from "../firePDF";
+import { firePdfAsyncTotalDurationSeconds } from "./metrics";
+import { POLL_FLOOR_MS, POLL_TIMEOUT_BUFFER_MS } from "./schema";
+import {
+ defaultSleep,
+ computeDeadlineMs,
+ makeFallback,
+} from "./utils";
+import { tryGetCached, maybeSaveResult } from "./cache";
+import { submitJob } from "./submit";
+import { pollUntilTerminal, type PollOutcome } from "./poll";
+import { fetchResult } from "./result";
+
+type FirePdfAsyncDeps = {
+ fetchImpl?: typeof undiciFetch;
+ fallbackImpl?: typeof scrapePDFWithFirePDF;
+ sleepImpl?: (ms: number, signal?: AbortSignal) => Promise<void>;
+ nowImpl?: () => number;
+};
+
+export async function scrapePDFWithFirePDFAsync(
+ meta: Meta,
+ base64Content: string,
+ maxPages?: number,
+ pagesProcessed?: number,
+ mode?: PDFMode,
+ deps: FirePdfAsyncDeps = {},
+): Promise<PDFProcessorResult> {
+ const fetchImpl = deps.fetchImpl ?? undiciFetch;
+ const fallbackImpl = deps.fallbackImpl ?? scrapePDFWithFirePDF;
+ const sleep = deps.sleepImpl ?? defaultSleep;
+ const now = deps.nowImpl ?? Date.now;
+
+ const cached = await tryGetCached(
+ meta,
+ base64Content,
+ mode,
+ maxPages,
+ pagesProcessed,
+ );
+ if (cached) return cached;
+
+ meta.abort.throwIfAborted();
+
+ const baseUrl = config.FIRE_PDF_BASE_URL;
+ if (!baseUrl) {
+ // Should be unreachable β€” call site checks this β€” but fall back rather
+ // than crash if a route somehow bypasses the gate.
+ return fallbackImpl(meta, base64Content, maxPages, pagesProcessed, mode);
+ }
+
+ const overallStartedAt = now();
+ const submitTime = now();
+ const deadlineFromNow = computeDeadlineMs(meta.abort.scrapeTimeout());
+ const deadlineAt = new Date(submitTime + deadlineFromNow).toISOString();
+ const pollingDeadline = submitTime + deadlineFromNow + POLL_TIMEOUT_BUFFER_MS;
+
+ const fallback = makeFallback({
+ meta,
+ fallbackImpl,
+ base64Content,
+ maxPages,
+ pagesProcessed,
+ mode,
+ });
+
+ // ── Step 1: POST /jobs ────────────────────────────────────────────────
+ const submit = await submitJob({
+ meta,
+ baseUrl,
+ base64Content,
+ maxPages,
+ pagesProcessed,
+ mode,
+ deadlineAt,
+ fetchImpl,
+ fallback,
+ });
+ if (submit.kind === "fallback") return submit.result;
+
+ // ── Step 2: poll until terminal (skip on idempotent-replay done) ──────
+ const polled: PollOutcome = submit.alreadyDone
+ ? {
+ kind: "done",
+ poll: { scrape_id: meta.id, status: "done" },
+ pollCount: 0,
+ }
+ : await pollUntilTerminal({
+ baseUrl,
+ scrapeId: meta.id,
+ initialDelay: submit.retryAfterMs ?? POLL_FLOOR_MS,
+ pollingDeadline,
+ meta,
+ fetchImpl,
+ sleep,
+ now,
+ fallback,
+ });
+ if (polled.kind === "fallback") return polled.result;
+
+ // ── Step 3: GET /jobs/:id/result ──────────────────────────────────────
+ const fetched = await fetchResult({
+ baseUrl,
+ scrapeId: meta.id,
+ meta,
+ fetchImpl,
+ sleep,
+ fallback,
+ });
+ if (fetched.kind === "fallback") return fetched.result;
+
+ // ── Assemble + cache save ─────────────────────────────────────────────
+ const pages =
+ fetched.result.pages_processed ??
+ polled.poll.pages_processed ??
+ pagesProcessed;
+ const durationMs = now() - overallStartedAt;
+ firePdfAsyncTotalDurationSeconds.observe(durationMs / 1000);
+
+ meta.logger.info("FirePDF async completed", {
+ scrapeId: meta.id,
+ durationMs,
+ markdownLength: fetched.result.markdown.length,
+ pagesProcessed: pages,
+ failedPages: fetched.result.failed_pages,
+ partialPages: fetched.result.partial_pages,
+ pollCount: polled.pollCount,
+ });
+
+ const processorResult: PDFProcessorResult & { markdown: string } = {
+ markdown: fetched.result.markdown,
+ html: await safeMarkdownToHtml(
+ fetched.result.markdown,
+ meta.logger,
+ meta.id,
+ ),
+ pagesProcessed: pages,
+ };
+
+ await maybeSaveResult({
+ meta,
+ base64Content,
+ mode,
+ maxPages,
+ result: processorResult,
+ });
+
+ return processorResult;
+}
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts
new file mode 100644
index 000000000..4a49f2c88
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts
@@ -0,0 +1,78 @@
+import type { Meta } from "../../..";
+import type { PDFMode } from "../../../../../controllers/v2/types";
+import type { PDFProcessorResult } from "../types";
+import {
+ getPdfResultFromCache,
+ savePdfResultToCache,
+} from "../../../../../lib/gcs-pdf-cache";
+
+// Cache layout mirrors the sync `scrapePDFWithFirePDF` so async/sync share
+// entries. `fast` mode bypasses entirely (hard cost ceiling β€” must fail on
+// scanned PDFs, not serve a cached OCR result), as does any call with
+// `maxPages` (the cached entry may have been written with a different cap).
+function cacheKeyShape(mode: PDFMode | undefined, maxPages: number | undefined) {
+ const cacheable = mode !== "fast" && !maxPages;
+ const ownVariant: string | undefined = mode === "ocr" ? "ocr" : undefined;
+ const lookupVariants: (string | undefined)[] =
+ mode === "ocr" ? ["ocr"] : [undefined, "ocr"];
+ return { cacheable, ownVariant, lookupVariants };
+}
+
+export async function tryGetCached(
+ meta: Meta,
+ base64Content: string,
+ mode: PDFMode | undefined,
+ maxPages: number | undefined,
+ pagesProcessed: number | undefined,
+): Promise<PDFProcessorResult | null> {
+ const { cacheable, lookupVariants } = cacheKeyShape(mode, maxPages);
+ if (!cacheable) return null;
+
+ for (const variant of lookupVariants) {
+ try {
+ const cached = await getPdfResultFromCache(
+ base64Content,
+ "firepdf",
+ variant,
+ );
+ if (cached) {
+ meta.logger.info("Using cached FirePDF result (async path)", {
+ scrapeId: meta.id,
+ requestedMode: mode,
+ cacheVariant: variant ?? "base",
+ });
+ return {
+ ...cached,
+ pagesProcessed: cached.pagesProcessed ?? pagesProcessed,
+ };
+ }
+ } catch (error) {
+ meta.logger.warn(
+ "Error checking FirePDF cache (async path), proceeding",
+ { error, cacheVariant: variant ?? "base" },
+ );
+ }
+ }
+ return null;
+}
+
+export async function maybeSaveResult(args: {
+ meta: Meta;
+ base64Content: string;
+ mode: PDFMode | undefined;
+ maxPages: number | undefined;
+ result: PDFProcessorResult & { markdown: string };
+}): Promise<void> {
+ const { meta, base64Content, mode, maxPages, result } = args;
+ const { cacheable, ownVariant } = cacheKeyShape(mode, maxPages);
+ if (!cacheable) return;
+
+ try {
+ await savePdfResultToCache(base64Content, result, "firepdf", ownVariant);
+ } catch (error) {
+ meta.logger.warn(
+ "Error saving FirePDF async result to cache (continuing)",
+ { error },
+ );
+ }
+}
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.ts
new file mode 100644
index 000000000..73ae7eb35
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.ts
@@ -0,0 +1,44 @@
+import { Counter, Histogram } from "prom-client";
+
+export const firePdfAsyncSubmittedTotal = new Counter({
+ name: "firecrawl_fire_pdf_async_submitted_total",
+ help: "Count of POST /jobs requests successfully submitted to fire-pdf async",
+ labelNames: ["lane"],
+});
+
+export const firePdfAsyncCompletedTotal = new Counter({
+ name: "firecrawl_fire_pdf_async_completed_total",
+ help: "Count of fire-pdf async jobs that reached a terminal status",
+ labelNames: ["terminal_status"],
+});
+
+export const firePdfAsyncFallbackTotal = new Counter({
+ name: "firecrawl_fire_pdf_async_fallback_total",
+ help: "Count of fire-pdf async requests that fell back to the sync /ocr path",
+ labelNames: ["reason"],
+});
+
+export const firePdfAsyncTotalDurationSeconds = new Histogram({
+ name: "firecrawl_fire_pdf_async_total_duration_seconds",
+ help: "End-to-end duration from 'decide to use async' to 'result available'",
+ buckets: [0.5, 1, 2.5, 5, 10, 30, 60, 120, 300, 600, 1200, 1800],
+});
+
+export const firePdfAsyncPollCount = new Histogram({
+ name: "firecrawl_fire_pdf_async_poll_count",
+ help: "Number of GET /jobs/:id polls performed per fire-pdf async job",
+ buckets: [1, 2, 5, 10, 20, 50, 100, 200, 500],
+});
+
+export type FallbackReason =
+ | "http_404"
+ | "http_413"
+ | "http_503"
+ | "http_429"
+ | "http_5xx"
+ | "network_error"
+ | "terminal_failed"
+ | "terminal_expired"
+ | "terminal_cancelled"
+ | "polling_timeout"
+ | "result_503";
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/poll.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/poll.ts
new file mode 100644
index 000000000..90c846baf
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/poll.ts
@@ -0,0 +1,166 @@
+import type { Meta } from "../../..";
+import type { PDFProcessorResult } from "../types";
+import { fetch as undiciFetch } from "undici";
+import { AbortManagerThrownError } from "../../../lib/abortManager";
+import {
+ firePdfAsyncCompletedTotal,
+ firePdfAsyncPollCount,
+ type FallbackReason,
+} from "./metrics";
+import {
+ pollResponseSchema,
+ TERMINAL_STATUSES,
+ type PollResponse,
+} from "./schema";
+import { nextPollDelay, type Fallback } from "./utils";
+
+export type PollDeps = {
+ baseUrl: string;
+ scrapeId: string;
+ initialDelay: number;
+ pollingDeadline: number;
+ meta: Meta;
+ fetchImpl: typeof undiciFetch;
+ sleep: (ms: number, signal?: AbortSignal) => Promise<void>;
+ now: () => number;
+ fallback: Fallback;
+};
+
+export type PollOutcome =
+ | { kind: "done"; poll: PollResponse; pollCount: number }
+ | { kind: "fallback"; result: PDFProcessorResult };
+
+export async function pollUntilTerminal(
+ deps: PollDeps,
+): Promise<PollOutcome> {
+ const {
+ baseUrl,
+ scrapeId,
+ pollingDeadline,
+ meta,
+ fetchImpl,
+ sleep,
+ now,
+ fallback,
+ } = deps;
+ let pollCount = 0;
+ let lastDelay = deps.initialDelay;
+
+ while (true) {
+ if (now() > pollingDeadline) {
+ firePdfAsyncPollCount.observe(pollCount);
+ return {
+ kind: "fallback",
+ result: await fallback("polling_timeout", { pollCount }),
+ };
+ }
+
+ meta.abort.throwIfAborted();
+ await sleep(lastDelay, meta.abort.asSignal());
+ pollCount++;
+
+ let pollResp;
+ try {
+ pollResp = await fetchImpl(`${baseUrl}/jobs/${scrapeId}`, {
+ method: "GET",
+ signal: meta.abort.asSignal(),
+ });
+ } catch (error) {
+ if (error instanceof AbortManagerThrownError) throw error;
+ firePdfAsyncPollCount.observe(pollCount);
+ return {
+ kind: "fallback",
+ result: await fallback("network_error", {
+ error: String(error),
+ pollCount,
+ }),
+ };
+ }
+
+ const pollStatus = pollResp.status;
+ const pollBody = await pollResp.json().catch(() => ({}));
+
+ if (pollStatus === 404) {
+ firePdfAsyncPollCount.observe(pollCount);
+ throw new Error(
+ "fire-pdf async GET /jobs/:id 404: scrape_id missing after successful submit",
+ );
+ }
+
+ if (pollStatus === 410) {
+ firePdfAsyncPollCount.observe(pollCount);
+ const parsed = pollResponseSchema.safeParse(pollBody);
+ const status = parsed.success ? parsed.data.status : "expired";
+ firePdfAsyncCompletedTotal.labels(status).inc();
+ return {
+ kind: "fallback",
+ result: await fallback(
+ status === "cancelled" ? "terminal_cancelled" : "terminal_expired",
+ { status, pollCount, body: pollBody },
+ ),
+ };
+ }
+
+ if (pollStatus === 502) {
+ firePdfAsyncPollCount.observe(pollCount);
+ firePdfAsyncCompletedTotal.labels("failed").inc();
+ return {
+ kind: "fallback",
+ result: await fallback("terminal_failed", {
+ pollCount,
+ body: pollBody,
+ }),
+ };
+ }
+
+ if (pollStatus !== 200 && pollStatus !== 202) {
+ firePdfAsyncPollCount.observe(pollCount);
+ return {
+ kind: "fallback",
+ result: await fallback("http_5xx", {
+ status: pollStatus,
+ body: pollBody,
+ pollCount,
+ }),
+ };
+ }
+
+ const parsed = pollResponseSchema.safeParse(pollBody);
+ if (!parsed.success) {
+ firePdfAsyncPollCount.observe(pollCount);
+ return {
+ kind: "fallback",
+ result: await fallback("http_5xx", {
+ error: String(parsed.error),
+ body: pollBody,
+ pollCount,
+ }),
+ };
+ }
+
+ if (TERMINAL_STATUSES.has(parsed.data.status)) {
+ firePdfAsyncPollCount.observe(pollCount);
+ firePdfAsyncCompletedTotal.labels(parsed.data.status).inc();
+ if (parsed.data.status !== "done") {
+ const reason: FallbackReason =
+ parsed.data.status === "failed"
+ ? "terminal_failed"
+ : parsed.data.status === "expired"
+ ? "terminal_expired"
+ : "terminal_cancelled";
+ return {
+ kind: "fallback",
+ result: await fallback(reason, {
+ status: parsed.data.status,
+ errorClass: parsed.data.error_class,
+ errorMessage: parsed.data.error_message,
+ pollCount,
+ }),
+ };
+ }
+ return { kind: "done", poll: parsed.data, pollCount };
+ }
+
+ lastDelay = nextPollDelay(lastDelay, parsed.data.retry_after_ms);
+ }
+}
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/result.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/result.ts
new file mode 100644
index 000000000..633b19ecd
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/result.ts
@@ -0,0 +1,92 @@
+import type { Meta } from "../../..";
+import type { PDFProcessorResult } from "../types";
+import { fetch as undiciFetch } from "undici";
+import { AbortManagerThrownError } from "../../../lib/abortManager";
+import {
+ POLL_FLOOR_MS,
+ resultResponseSchema,
+ type ResultResponse,
+} from "./schema";
+import type { Fallback } from "./utils";
+
+export type ResultDeps = {
+ baseUrl: string;
+ scrapeId: string;
+ meta: Meta;
+ fetchImpl: typeof undiciFetch;
+ sleep: (ms: number, signal?: AbortSignal) => Promise<void>;
+ fallback: Fallback;
+};
+
+export type ResultOutcome =
+ | { kind: "ok"; result: ResultResponse }
+ | { kind: "fallback"; result: PDFProcessorResult };
+
+export async function fetchResult(deps: ResultDeps): Promise<ResultOutcome> {
+ const { baseUrl, scrapeId, meta, fetchImpl, sleep, fallback } = deps;
+ let retried409 = 0;
+
+ while (true) {
+ let resp;
+ try {
+ resp = await fetchImpl(`${baseUrl}/jobs/${scrapeId}/result`, {
+ method: "GET",
+ signal: meta.abort.asSignal(),
+ });
+ } catch (error) {
+ if (error instanceof AbortManagerThrownError) throw error;
+ return {
+ kind: "fallback",
+ result: await fallback("network_error", { error: String(error) }),
+ };
+ }
+
+ const status = resp.status;
+ const body = await resp.json().catch(() => ({}));
+
+ if (status === 503) {
+ return {
+ kind: "fallback",
+ result: await fallback("result_503", { body }),
+ };
+ }
+
+ if (status === 409) {
+ retried409++;
+ if (retried409 > 1) {
+ return {
+ kind: "fallback",
+ result: await fallback("http_5xx", {
+ status: 409,
+ body,
+ note: "result endpoint kept returning 409",
+ }),
+ };
+ }
+ meta.logger.info("FirePDF async result returned 409, re-polling once", {
+ scrapeId,
+ });
+ await sleep(POLL_FLOOR_MS, meta.abort.asSignal());
+ continue;
+ }
+
+ if (status !== 200) {
+ return {
+ kind: "fallback",
+ result: await fallback("http_5xx", { status, body }),
+ };
+ }
+
+ const parsed = resultResponseSchema.safeParse(body);
+ if (!parsed.success) {
+ return {
+ kind: "fallback",
+ result: await fallback("http_5xx", {
+ error: String(parsed.error),
+ body,
+ }),
+ };
+ }
+ return { kind: "ok", result: parsed.data };
+ }
+}
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts
new file mode 100644
index 000000000..9d19f1056
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts
@@ -0,0 +1,56 @@
+import { z } from "zod";
+
+// Deadline constraints (`deadline_at - now` must fall in this window per the
+// /jobs contract). Polling cadence β€” start at the response's `retry_after_ms`
+// floor, exponential backoff capped at POLL_CAP_MS. Polling deadline budget
+// = computed `deadline_at` + this buffer (defense in depth on top of the
+// worker's own expiration handling).
+export const MIN_DEADLINE_MS = 5_000;
+export const MAX_DEADLINE_MS = 30 * 60 * 1_000;
+export const POLL_FLOOR_MS = 1_000;
+export const POLL_CAP_MS = 5_000;
+export const POLL_TIMEOUT_BUFFER_MS = 30_000;
+
+export const TERMINAL_STATUSES = new Set([
+ "done",
+ "failed",
+ "expired",
+ "cancelled",
+]);
+
+export const submitResponseSchema = z.object({
+ scrape_id: z.string(),
+ status: z.enum(["queued", "published", "running", "done"]),
+ lane: z.string().optional(),
+ retry_after_ms: z.number().optional(),
+});
+
+export const pollResponseSchema = z.object({
+ scrape_id: z.string(),
+ status: z.enum([
+ "queued",
+ "published",
+ "running",
+ "done",
+ "failed",
+ "expired",
+ "cancelled",
+ ]),
+ retry_after_ms: z.number().optional(),
+ pages_processed: z.number().optional(),
+ failed_pages: z.array(z.number()).nullable().optional(),
+ partial_pages: z.array(z.number()).nullable().optional(),
+ error_class: z.string().optional(),
+ error_message: z.string().optional(),
+});
+
+export const resultResponseSchema = z.object({
+ schema_version: z.literal(1).optional(),
+ markdown: z.string(),
+ pages_processed: z.number().optional(),
+ failed_pages: z.array(z.number()).nullable().optional(),
+ partial_pages: z.array(z.number()).nullable().optional(),
+});
+
+export type PollResponse = z.infer<typeof pollResponseSchema>;
+export type ResultResponse = z.infer<typeof resultResponseSchema>;
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts
new file mode 100644
index 000000000..e239ed31c
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts
@@ -0,0 +1,137 @@
+import type { Meta } from "../../..";
+import type { PDFMode } from "../../../../../controllers/v2/types";
+import type { PDFProcessorResult } from "../types";
+import { fetch as undiciFetch } from "undici";
+import { AbortManagerThrownError } from "../../../lib/abortManager";
+import { firePdfAsyncSubmittedTotal } from "./metrics";
+import { submitResponseSchema } from "./schema";
+import type { Fallback } from "./utils";
+
+export type SubmitOutcome =
+ | {
+ kind: "ok";
+ lane: string | undefined;
+ retryAfterMs: number | undefined;
+ alreadyDone: boolean;
+ }
+ | { kind: "fallback"; result: PDFProcessorResult };
+
+export type SubmitArgs = {
+ meta: Meta;
+ baseUrl: string;
+ base64Content: string;
+ maxPages: number | undefined;
+ pagesProcessed: number | undefined;
+ mode: PDFMode | undefined;
+ deadlineAt: string;
+ fetchImpl: typeof undiciFetch;
+ fallback: Fallback;
+};
+
+export async function submitJob(args: SubmitArgs): Promise<SubmitOutcome> {
+ const {
+ meta,
+ baseUrl,
+ base64Content,
+ maxPages,
+ pagesProcessed,
+ mode,
+ deadlineAt,
+ fetchImpl,
+ fallback,
+ } = args;
+ const scrapeId = meta.id;
+
+ const body = {
+ pdf_b64: base64Content,
+ scrape_id: scrapeId,
+ source: "firecrawl" as const,
+ zdr: false as const,
+ deadline_at: deadlineAt,
+ ...(meta.internalOptions.teamId && {
+ team_id: meta.internalOptions.teamId,
+ }),
+ ...(meta.internalOptions.crawlId && {
+ crawl_id: meta.internalOptions.crawlId,
+ }),
+ options: {
+ ...(pagesProcessed !== undefined && { pages_estimate: pagesProcessed }),
+ ...(maxPages !== undefined && { max_pages: maxPages }),
+ ...(mode !== undefined && { mode }),
+ },
+ };
+
+ let status: number;
+ let json: unknown;
+ try {
+ const resp = await fetchImpl(`${baseUrl}/jobs`, {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify(body),
+ signal: meta.abort.asSignal(),
+ });
+ status = resp.status;
+ json = await resp.json().catch(() => ({}));
+ } catch (error) {
+ if (error instanceof AbortManagerThrownError) throw error;
+ return { kind: "fallback", result: await fallback("network_error", { error: String(error) }) };
+ }
+
+ if (status === 404) return { kind: "fallback", result: await fallback("http_404") };
+ if (status === 413) return { kind: "fallback", result: await fallback("http_413") };
+ if (status === 429) return { kind: "fallback", result: await fallback("http_429") };
+ if (status === 503) return { kind: "fallback", result: await fallback("http_503") };
+
+ if (status === 409) {
+ meta.logger.error("FirePDF async POST /jobs returned 409 scrape_id_conflict", {
+ scrapeId,
+ body: json,
+ });
+ throw new Error(
+ "fire-pdf async POST /jobs conflict: scrape_id reused with different inputs",
+ );
+ }
+
+ if (status === 400) {
+ meta.logger.error("FirePDF async POST /jobs returned 400 validation error", {
+ scrapeId,
+ body: json,
+ });
+ throw new Error("fire-pdf async POST /jobs validation error");
+ }
+
+ if (status !== 200 && status !== 202) {
+ return {
+ kind: "fallback",
+ result: await fallback("http_5xx", { status, body: json }),
+ };
+ }
+
+ const parsed = submitResponseSchema.safeParse(json);
+ if (!parsed.success) {
+ return {
+ kind: "fallback",
+ result: await fallback("http_5xx", {
+ error: String(parsed.error),
+ body: json,
+ status,
+ }),
+ };
+ }
+
+ firePdfAsyncSubmittedTotal.labels(parsed.data.lane ?? "unknown").inc();
+ meta.logger.info("FirePDF async POST /jobs accepted", {
+ scrapeId,
+ status: parsed.data.status,
+ httpStatus: status,
+ lane: parsed.data.lane,
+ deadlineAt,
+ });
+
+ return {
+ kind: "ok",
+ lane: parsed.data.lane,
+ retryAfterMs: parsed.data.retry_after_ms,
+ alreadyDone: status === 200 && parsed.data.status === "done",
+ };
+}
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.ts
new file mode 100644
index 000000000..a96b75cab
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.ts
@@ -0,0 +1,89 @@
+import type { Meta } from "../../..";
+import type { PDFMode } from "../../../../../controllers/v2/types";
+import type { PDFProcessorResult } from "../types";
+import type { scrapePDFWithFirePDF } from "../firePDF";
+import {
+ MIN_DEADLINE_MS,
+ MAX_DEADLINE_MS,
+ POLL_FLOOR_MS,
+ POLL_CAP_MS,
+} from "./schema";
+import {
+ firePdfAsyncFallbackTotal,
+ type FallbackReason,
+} from "./metrics";
+
+export function defaultSleep(
+ ms: number,
+ signal?: AbortSignal,
+): Promise<void> {
+ return new Promise((resolve, reject) => {
+ const handle = setTimeout(() => {
+ signal?.removeEventListener("abort", onAbort);
+ resolve();
+ }, ms);
+ const onAbort = () => {
+ clearTimeout(handle);
+ reject(
+ signal?.reason instanceof Error ? signal.reason : new Error("Aborted"),
+ );
+ };
+ if (signal) {
+ if (signal.aborted) {
+ clearTimeout(handle);
+ reject(
+ signal.reason instanceof Error
+ ? signal.reason
+ : new Error("Aborted"),
+ );
+ return;
+ }
+ signal.addEventListener("abort", onAbort, { once: true });
+ }
+ });
+}
+
+export function nextPollDelay(
+ prev: number,
+ retryAfterMs: number | undefined,
+): number {
+ const candidate = retryAfterMs ?? Math.max(prev * 2, POLL_FLOOR_MS);
+ return Math.min(POLL_CAP_MS, Math.max(POLL_FLOOR_MS, candidate));
+}
+
+export function computeDeadlineMs(scrapeTimeoutMs: number | undefined): number {
+ // 5min default when there's no scrape budget (CLI/tests). Anything outside
+ // [5s, 30min] is clamped to satisfy the /jobs contract.
+ const fallback = 5 * 60 * 1_000;
+ const candidate = scrapeTimeoutMs ?? fallback;
+ return Math.min(MAX_DEADLINE_MS, Math.max(MIN_DEADLINE_MS, candidate));
+}
+
+export type Fallback = (
+ reason: FallbackReason,
+ extra?: Record<string, unknown>,
+) => Promise<PDFProcessorResult>;
+
+export type FallbackCtx = {
+ meta: Meta;
+ fallbackImpl: typeof scrapePDFWithFirePDF;
+ base64Content: string;
+ maxPages?: number;
+ pagesProcessed?: number;
+ mode?: PDFMode;
+};
+
+export function makeFallback(ctx: FallbackCtx): Fallback {
+ const { meta, fallbackImpl, base64Content, maxPages, pagesProcessed, mode } =
+ ctx;
+ const scrapeId = meta.id;
+ return async (reason, extra = {}) => {
+ firePdfAsyncFallbackTotal.labels(reason).inc();
+ meta.logger.warn("FirePDF async falling back to sync /ocr", {
+ scrapeId,
+ reason,
+ ...extra,
+ });
+ return fallbackImpl(meta, base64Content, maxPages, pagesProcessed, mode);
+ };
+}
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
index 019e2db00..b6c652708 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
@@ -18,6 +18,7 @@ import {
shouldParsePDF,
getPDFMaxPages,
getPDFMode,
+ getFirePdfAsync,
} from "../../../../controllers/v2/types";
import type { PDFMode } from "../../../../controllers/v2/types";
import { processPdf, detectPdf } from "@mendable/firecrawl-rs";
@@ -34,6 +35,7 @@ import {
import { withSpan, setSpanAttributes } from "../../../../lib/otel-tracer";
import { scrapePDFWithRunPodMU } from "./runpodMU";
import { reconcilePageCountWithFirePdf, scrapePDFWithFirePDF } from "./firePDF";
+import { scrapePDFWithFirePDFAsync } from "./fire-pdf/async";
import { scrapePDFWithParsePDF } from "./pdfParse";
import { captureExceptionWithZdrCheck } from "../../../../services/sentry";
import { isPdfBuffer, PDF_SNIFF_WINDOW } from "./pdfUtils";
@@ -399,12 +401,20 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
Math.random() * 100 < config.FIRE_PDF_PERCENT);
if (useFirePDF) {
+ // Per-request opt-in to async fire-pdf pipeline. Async client falls
+ // back to sync on any failure, so behaviour stays bounded by the
+ // sync path.
+ const useAsync = getFirePdfAsync(meta.options.parsers);
try {
- result = await scrapePDFWithFirePDF(
+ result = await (useAsync
+ ? scrapePDFWithFirePDFAsync
+ : scrapePDFWithFirePDF)(
{
...meta,
logger: meta.logger.child({
- method: "scrapePDF/firePDF",
+ method: useAsync
+ ? "scrapePDF/firePDFAsync"
+ : "scrapePDF/firePDF",
}),
},
base64Content,