Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fire-pdf: send explicit {timeout, created_at} for deadline contract (#3399)
Abimael Martell8ad9b81a4ab8
diff --git a/apps/api/src/lib/gcs-pdf-cache.ts b/apps/api/src/lib/gcs-pdf-cache.tsindex ec9c7651b..6a1deeda6 100644--- a/apps/api/src/lib/gcs-pdf-cache.ts+++ b/apps/api/src/lib/gcs-pdf-cache.ts@@ -11,7 +11,7 @@ const PROVIDER_PREFIXES: Record<PdfCacheProvider, string> = {firepdf: "pdf-cache-firepdf/",};-function createPdfCacheKey(pdfContent: string | Buffer): string {+export function createPdfCacheKey(pdfContent: string | Buffer): string {return crypto.createHash("sha256").update(pdfContent).digest("hex");}diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.tsindex 70ea3d60e..86fa38c70 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts@@ -5,6 +5,7 @@ import { z } from "zod";import type { PDFProcessorResult } from "./types";import { safeMarkdownToHtml } from "./markdownToHtml";import {+ createPdfCacheKey,getPdfResultFromCache,savePdfResultToCache,} from "../../../../lib/gcs-pdf-cache";@@ -42,6 +43,29 @@ export async function scrapePDFWithFirePDF(pagesProcessed,});+ const zdr = meta.internalOptions.zeroDataRetention === true;+ const pdfSha256 = createPdfCacheKey(base64Content);++ // Explicit deadline contract with fire-pdf (mirrors mineru-api):+ // timeout β remaining scrape-tier budget in ms (from AbortManager)+ // created_at β epoch ms when we handed the budget over to fire-pdf+ //+ // fire-pdf computes remaining = timeout - (now - created_at) and can+ // return 503 if the budget is spent. Previously it only saw the abort+ // signal from the HTTP connection, which it didn't observe β so work+ // kept running past the caller's timeout and the user got a late+ // failure instead of a fast deadline-exceeded response.+ //+ // scrapeTimeout() returns undefined if no scrape-tier deadline is set+ // (e.g., internal tests, CLI). Don't send timeout in that case so+ // fire-pdf applies its own default.+ const fireScrapeTimeout = meta.abort.scrapeTimeout();+ const deadlineFields: { timeout?: number; created_at?: number } = {};+ if (fireScrapeTimeout !== undefined && fireScrapeTimeout > 0) {+ deadlineFields.timeout = Math.floor(fireScrapeTimeout);+ deadlineFields.created_at = Date.now();+ }+const resp = await robustFetch({url: `${config.FIRE_PDF_BASE_URL}/ocr`,method: "POST",@@ -52,6 +76,17 @@ export async function scrapePDFWithFirePDF(pdf: base64Content,scrape_id: meta.id,...(maxPages !== undefined && { max_pages: maxPages }),+ // Enrichment for the fire-pdf jobs DB / dashboard. fire-pdf treats+ // these as optional β older fire-pdf builds will ignore unknown fields.+ team_id: meta.internalOptions.teamId,+ ...(meta.internalOptions.crawlId && {+ crawl_id: meta.internalOptions.crawlId,+ }),+ ...(zdr ? {} : { url: meta.rewrittenUrl ?? meta.url }),+ pdf_sha256: pdfSha256,+ source: "firecrawl",+ zdr,+ ...deadlineFields,},logger,schema: z.object({diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex 53c96fa73..1a4e1b09b 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@ -404,6 +404,22 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {meta.logger.warn("FirePDF failed -- falling back to MinerU", {method: "scrapePDF/firePDF",error,+ event: "pdf_engine_fallback",+ scrape_id: meta.id,+ team_id: meta.internalOptions.teamId,+ from_engine: "firepdf",+ to_engine: "mineru",+ // Coerce both to strings defensively β if someone throws a+ // non-Error (e.g. a plain object or primitive), `.name` /+ // `.message` could be undefined or non-string, and `.slice` would+ // throw inside the fallback logger, masking the original failure.+ error_class:+ (error as { name?: unknown })?.name != null+ ? String((error as { name?: unknown }).name)+ : undefined,+ error_message: String(+ (error as { message?: unknown })?.message ?? "",+ ).slice(0, 500),});}}