Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

fire-pdf: send explicit {timeout, created_at} for deadline contract (#3399)

Abimael Martell8ad9b81a4ab8
diff --git a/apps/api/src/lib/gcs-pdf-cache.ts b/apps/api/src/lib/gcs-pdf-cache.ts
index ec9c7651b..6a1deeda6 100644
--- a/apps/api/src/lib/gcs-pdf-cache.ts
+++ b/apps/api/src/lib/gcs-pdf-cache.ts
@@ -11,7 +11,7 @@ const PROVIDER_PREFIXES: Record<PdfCacheProvider, string> = {
firepdf: "pdf-cache-firepdf/",
};
-function createPdfCacheKey(pdfContent: string | Buffer): string {
+export function createPdfCacheKey(pdfContent: string | Buffer): string {
return crypto.createHash("sha256").update(pdfContent).digest("hex");
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts
index 70ea3d60e..86fa38c70 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts
@@ -5,6 +5,7 @@ import { z } from "zod";
import type { PDFProcessorResult } from "./types";
import { safeMarkdownToHtml } from "./markdownToHtml";
import {
+ createPdfCacheKey,
getPdfResultFromCache,
savePdfResultToCache,
} from "../../../../lib/gcs-pdf-cache";
@@ -42,6 +43,29 @@ export async function scrapePDFWithFirePDF(
pagesProcessed,
});
+ const zdr = meta.internalOptions.zeroDataRetention === true;
+ const pdfSha256 = createPdfCacheKey(base64Content);
+
+ // Explicit deadline contract with fire-pdf (mirrors mineru-api):
+ // timeout β€” remaining scrape-tier budget in ms (from AbortManager)
+ // created_at β€” epoch ms when we handed the budget over to fire-pdf
+ //
+ // fire-pdf computes remaining = timeout - (now - created_at) and can
+ // return 503 if the budget is spent. Previously it only saw the abort
+ // signal from the HTTP connection, which it didn't observe β€” so work
+ // kept running past the caller's timeout and the user got a late
+ // failure instead of a fast deadline-exceeded response.
+ //
+ // scrapeTimeout() returns undefined if no scrape-tier deadline is set
+ // (e.g., internal tests, CLI). Don't send timeout in that case so
+ // fire-pdf applies its own default.
+ const fireScrapeTimeout = meta.abort.scrapeTimeout();
+ const deadlineFields: { timeout?: number; created_at?: number } = {};
+ if (fireScrapeTimeout !== undefined && fireScrapeTimeout > 0) {
+ deadlineFields.timeout = Math.floor(fireScrapeTimeout);
+ deadlineFields.created_at = Date.now();
+ }
+
const resp = await robustFetch({
url: `${config.FIRE_PDF_BASE_URL}/ocr`,
method: "POST",
@@ -52,6 +76,17 @@ export async function scrapePDFWithFirePDF(
pdf: base64Content,
scrape_id: meta.id,
...(maxPages !== undefined && { max_pages: maxPages }),
+ // Enrichment for the fire-pdf jobs DB / dashboard. fire-pdf treats
+ // these as optional β€” older fire-pdf builds will ignore unknown fields.
+ team_id: meta.internalOptions.teamId,
+ ...(meta.internalOptions.crawlId && {
+ crawl_id: meta.internalOptions.crawlId,
+ }),
+ ...(zdr ? {} : { url: meta.rewrittenUrl ?? meta.url }),
+ pdf_sha256: pdfSha256,
+ source: "firecrawl",
+ zdr,
+ ...deadlineFields,
},
logger,
schema: z.object({
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
index 53c96fa73..1a4e1b09b 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
@@ -404,6 +404,22 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
meta.logger.warn("FirePDF failed -- falling back to MinerU", {
method: "scrapePDF/firePDF",
error,
+ event: "pdf_engine_fallback",
+ scrape_id: meta.id,
+ team_id: meta.internalOptions.teamId,
+ from_engine: "firepdf",
+ to_engine: "mineru",
+ // Coerce both to strings defensively β€” if someone throws a
+ // non-Error (e.g. a plain object or primitive), `.name` /
+ // `.message` could be undefined or non-string, and `.slice` would
+ // throw inside the fallback logger, masking the original failure.
+ error_class:
+ (error as { name?: unknown })?.name != null
+ ? String((error as { name?: unknown }).name)
+ : undefined,
+ error_message: String(
+ (error as { message?: unknown })?.message ?? "",
+ ).slice(0, 500),
});
}
}