Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

feat(scrape): forward pdf parser mode to fire-pdf + scope cache (#3475)

Abimael Martellddbbfc702f25
diff --git a/apps/api/src/lib/gcs-pdf-cache.ts b/apps/api/src/lib/gcs-pdf-cache.ts
index c6c44ac32..fd6fbbeb4 100644
--- a/apps/api/src/lib/gcs-pdf-cache.ts
+++ b/apps/api/src/lib/gcs-pdf-cache.ts
@@ -28,6 +28,7 @@ export async function savePdfResultToCache(
pdfContent: string,
result: CachedPdfResult,
provider: PdfCacheProvider = "runpod",
+ variant?: string,
): Promise<string | null> {
try {
if (!config.GCS_BUCKET_NAME) {
@@ -36,8 +37,9 @@ export async function savePdfResultToCache(
const prefix = PROVIDER_PREFIXES[provider];
const cacheKey = createPdfCacheKey(pdfContent);
+ const objectKey = variant ? `${cacheKey}-${variant}` : cacheKey;
const bucket = storage.bucket(config.GCS_BUCKET_NAME);
- const blob = bucket.file(`${prefix}${cacheKey}.json`);
+ const blob = bucket.file(`${prefix}${objectKey}.json`);
for (let i = 0; i < 3; i++) {
try {
@@ -83,6 +85,7 @@ export async function savePdfResultToCache(
export async function getPdfResultFromCache(
pdfContent: string,
provider: PdfCacheProvider = "runpod",
+ variant?: string,
): Promise<CachedPdfResult | null> {
try {
if (!config.GCS_BUCKET_NAME) {
@@ -91,8 +94,9 @@ export async function getPdfResultFromCache(
const prefix = PROVIDER_PREFIXES[provider];
const cacheKey = createPdfCacheKey(pdfContent);
+ const objectKey = variant ? `${cacheKey}-${variant}` : cacheKey;
const bucket = storage.bucket(config.GCS_BUCKET_NAME);
- const blob = bucket.file(`${prefix}${cacheKey}.json`);
+ const blob = bucket.file(`${prefix}${objectKey}.json`);
const [content] = await blob.download();
const result = JSON.parse(content.toString());
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts
index 3ea97d250..fa8dec3b2 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts
@@ -3,6 +3,7 @@ import { config } from "../../../../config";
import { robustFetch } from "../../lib/fetch";
import { z } from "zod";
import type { PDFProcessorResult } from "./types";
+import type { PDFMode } from "../../../../controllers/v2/types";
import { safeMarkdownToHtml } from "./markdownToHtml";
import {
createPdfCacheKey,
@@ -45,26 +46,56 @@ export async function scrapePDFWithFirePDF(
base64Content: string,
maxPages?: number,
pagesProcessed?: number,
+ mode?: PDFMode,
): Promise<PDFProcessorResult> {
const logger = meta.logger;
- if (!maxPages && !meta.internalOptions.zeroDataRetention) {
- try {
- const cached = await getPdfResultFromCache(base64Content, "firepdf");
- if (cached) {
- logger.info("Using cached FirePDF result", {
- scrapeId: meta.id,
+ // Cache layout:
+ // - `ocr` mode reads/writes a dedicated `…-ocr.json` bucket. ocr
+ // requests explicitly want forced layout-mode OCR, so they must
+ // not be served a base-cache entry that was written by `auto`.
+ // - `auto` (and legacy undefined-mode) reads/writes the base
+ // `firepdf-<sha>.json` bucket β€” same key main has always used,
+ // so existing entries keep working. As a free upgrade, auto also
+ // reads the ocr bucket as a fallback: if some prior `ocr` run
+ // already produced markdown for this PDF, reuse it rather than
+ // running fire-pdf again.
+ // - `fast` is bypassed entirely (hard cost ceiling β€” must fail on
+ // scanned PDFs, not serve a cached OCR result).
+ const cacheable =
+ mode !== "fast" && !maxPages && !meta.internalOptions.zeroDataRetention;
+ const ownVariant: string | undefined = mode === "ocr" ? "ocr" : undefined;
+ const lookupVariants: (string | undefined)[] =
+ mode === "ocr" ? ["ocr"] : [undefined, "ocr"];
+
+ if (cacheable) {
+ for (const variant of lookupVariants) {
+ try {
+ const cached = await getPdfResultFromCache(
+ base64Content,
+ "firepdf",
+ variant,
+ );
+ if (cached) {
+ logger.info("Using cached FirePDF result", {
+ scrapeId: meta.id,
+ requestedMode: mode,
+ cacheVariant: variant ?? "base",
+ });
+ // Cache entries written before pagesProcessed existed don't carry
+ // the field. Fall back to the caller's pagesProcessed argument so
+ // billing on a stale hit doesn't silently regress to 0.
+ return {
+ ...cached,
+ pagesProcessed: cached.pagesProcessed ?? pagesProcessed,
+ };
+ }
+ } catch (error) {
+ logger.warn("Error checking FirePDF cache, proceeding", {
+ error,
+ cacheVariant: variant ?? "base",
});
- // Cache entries written before pagesProcessed existed don't carry
- // the field. Fall back to the caller's pagesProcessed argument so
- // billing on a stale hit doesn't silently regress to 0.
- return {
- ...cached,
- pagesProcessed: cached.pagesProcessed ?? pagesProcessed,
- };
}
- } catch (error) {
- logger.warn("Error checking FirePDF cache, proceeding", { error });
}
}
@@ -112,6 +143,7 @@ export async function scrapePDFWithFirePDF(
pdf: base64Content,
scrape_id: meta.id,
...(maxPages !== undefined && { max_pages: maxPages }),
+ ...(mode !== undefined && { mode }),
// Enrichment for the fire-pdf jobs DB / dashboard. fire-pdf treats
// these as optional β€” older fire-pdf builds will ignore unknown fields.
team_id: meta.internalOptions.teamId,
@@ -153,9 +185,14 @@ export async function scrapePDFWithFirePDF(
pagesProcessed: pages,
};
- if (!maxPages && !meta.internalOptions.zeroDataRetention) {
+ if (cacheable) {
try {
- await savePdfResultToCache(base64Content, processorResult, "firepdf");
+ await savePdfResultToCache(
+ base64Content,
+ processorResult,
+ "firepdf",
+ ownVariant,
+ );
} catch (error) {
logger.warn("Error saving FirePDF result to cache", { error });
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
index ad4ef8583..019e2db00 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
@@ -410,6 +410,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
base64Content,
maxPages,
effectivePageCount,
+ mode,
);
effectivePageCount = reconcilePageCountWithFirePdf(
effectivePageCount,