Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(scrape): forward pdf parser mode to fire-pdf + scope cache (#3475)
Abimael Martellddbbfc702f25
diff --git a/apps/api/src/lib/gcs-pdf-cache.ts b/apps/api/src/lib/gcs-pdf-cache.tsindex c6c44ac32..fd6fbbeb4 100644--- a/apps/api/src/lib/gcs-pdf-cache.ts+++ b/apps/api/src/lib/gcs-pdf-cache.ts@@ -28,6 +28,7 @@ export async function savePdfResultToCache(pdfContent: string,result: CachedPdfResult,provider: PdfCacheProvider = "runpod",+ variant?: string,): Promise<string | null> {try {if (!config.GCS_BUCKET_NAME) {@@ -36,8 +37,9 @@ export async function savePdfResultToCache(const prefix = PROVIDER_PREFIXES[provider];const cacheKey = createPdfCacheKey(pdfContent);+ const objectKey = variant ? `${cacheKey}-${variant}` : cacheKey;const bucket = storage.bucket(config.GCS_BUCKET_NAME);- const blob = bucket.file(`${prefix}${cacheKey}.json`);+ const blob = bucket.file(`${prefix}${objectKey}.json`);for (let i = 0; i < 3; i++) {try {@@ -83,6 +85,7 @@ export async function savePdfResultToCache(export async function getPdfResultFromCache(pdfContent: string,provider: PdfCacheProvider = "runpod",+ variant?: string,): Promise<CachedPdfResult | null> {try {if (!config.GCS_BUCKET_NAME) {@@ -91,8 +94,9 @@ export async function getPdfResultFromCache(const prefix = PROVIDER_PREFIXES[provider];const cacheKey = createPdfCacheKey(pdfContent);+ const objectKey = variant ? `${cacheKey}-${variant}` : cacheKey;const bucket = storage.bucket(config.GCS_BUCKET_NAME);- const blob = bucket.file(`${prefix}${cacheKey}.json`);+ const blob = bucket.file(`${prefix}${objectKey}.json`);const [content] = await blob.download();const result = JSON.parse(content.toString());diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.tsindex 3ea97d250..fa8dec3b2 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts@@ -3,6 +3,7 @@ import { config } from "../../../../config";import { robustFetch } from "../../lib/fetch";import { z } from "zod";import type { PDFProcessorResult } from "./types";+import type { PDFMode } from "../../../../controllers/v2/types";import { safeMarkdownToHtml } from "./markdownToHtml";import {createPdfCacheKey,@@ -45,26 +46,56 @@ export async function scrapePDFWithFirePDF(base64Content: string,maxPages?: number,pagesProcessed?: number,+ mode?: PDFMode,): Promise<PDFProcessorResult> {const logger = meta.logger;- if (!maxPages && !meta.internalOptions.zeroDataRetention) {- try {- const cached = await getPdfResultFromCache(base64Content, "firepdf");- if (cached) {- logger.info("Using cached FirePDF result", {- scrapeId: meta.id,+ // Cache layout:+ // - `ocr` mode reads/writes a dedicated `β¦-ocr.json` bucket. ocr+ // requests explicitly want forced layout-mode OCR, so they must+ // not be served a base-cache entry that was written by `auto`.+ // - `auto` (and legacy undefined-mode) reads/writes the base+ // `firepdf-<sha>.json` bucket β same key main has always used,+ // so existing entries keep working. As a free upgrade, auto also+ // reads the ocr bucket as a fallback: if some prior `ocr` run+ // already produced markdown for this PDF, reuse it rather than+ // running fire-pdf again.+ // - `fast` is bypassed entirely (hard cost ceiling β must fail on+ // scanned PDFs, not serve a cached OCR result).+ const cacheable =+ mode !== "fast" && !maxPages && !meta.internalOptions.zeroDataRetention;+ const ownVariant: string | undefined = mode === "ocr" ? "ocr" : undefined;+ const lookupVariants: (string | undefined)[] =+ mode === "ocr" ? ["ocr"] : [undefined, "ocr"];++ if (cacheable) {+ for (const variant of lookupVariants) {+ try {+ const cached = await getPdfResultFromCache(+ base64Content,+ "firepdf",+ variant,+ );+ if (cached) {+ logger.info("Using cached FirePDF result", {+ scrapeId: meta.id,+ requestedMode: mode,+ cacheVariant: variant ?? "base",+ });+ // Cache entries written before pagesProcessed existed don't carry+ // the field. Fall back to the caller's pagesProcessed argument so+ // billing on a stale hit doesn't silently regress to 0.+ return {+ ...cached,+ pagesProcessed: cached.pagesProcessed ?? pagesProcessed,+ };+ }+ } catch (error) {+ logger.warn("Error checking FirePDF cache, proceeding", {+ error,+ cacheVariant: variant ?? "base",});- // Cache entries written before pagesProcessed existed don't carry- // the field. Fall back to the caller's pagesProcessed argument so- // billing on a stale hit doesn't silently regress to 0.- return {- ...cached,- pagesProcessed: cached.pagesProcessed ?? pagesProcessed,- };}- } catch (error) {- logger.warn("Error checking FirePDF cache, proceeding", { error });}}@@ -112,6 +143,7 @@ export async function scrapePDFWithFirePDF(pdf: base64Content,scrape_id: meta.id,...(maxPages !== undefined && { max_pages: maxPages }),+ ...(mode !== undefined && { mode }),// Enrichment for the fire-pdf jobs DB / dashboard. fire-pdf treats// these as optional β older fire-pdf builds will ignore unknown fields.team_id: meta.internalOptions.teamId,@@ -153,9 +185,14 @@ export async function scrapePDFWithFirePDF(pagesProcessed: pages,};- if (!maxPages && !meta.internalOptions.zeroDataRetention) {+ if (cacheable) {try {- await savePdfResultToCache(base64Content, processorResult, "firepdf");+ await savePdfResultToCache(+ base64Content,+ processorResult,+ "firepdf",+ ownVariant,+ );} catch (error) {logger.warn("Error saving FirePDF result to cache", { error });}diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex ad4ef8583..019e2db00 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@ -410,6 +410,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {base64Content,maxPages,effectivePageCount,+ mode,);effectivePageCount = reconcilePageCountWithFirePdf(effectivePageCount,