Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): never serve or store an empty FirePDF result for a raster image (#4515)
Abimael Martell70d992234730
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.tsindex 24c88fcd7..e812bab65 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts@@ -611,3 +611,112 @@ describe("FirePDF page-markdown cache capabilities", () => {);});});++describe("FirePDF cache and empty raster-image results", () => {+ // Base64 of a PNG signature followed by padding: sniffs as image/png.+ const PNG_BASE64 = Buffer.from([+ 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 0, 0, 0, 0, 0,+ ]).toString("base64");+ // Base64 of "%PDF-1.7": not an image, so the PDF rules apply.+ const PDF_BASE64 = Buffer.from(+ "%PDF-1.7\n%\xe2\xe3\xcf\xd3\n",+ "latin1",+ ).toString("base64");++ beforeEach(() => {+ vi.clearAllMocks();+ getCached.mockResolvedValue(null);+ saveCached.mockResolvedValue(null);+ });++ it("treats a cached empty result for an image as a miss", async () => {+ getCached.mockResolvedValue({ markdown: "", html: "" });+ const meta = makeMeta();++ const result = await tryGetCached(+ meta,+ PNG_BASE64,+ "ocr",+ undefined,+ 1,+ false,+ false,+ );++ expect(result).toBeNull();+ expect(meta.logger.info).toHaveBeenCalledWith(+ "Ignoring cached empty FirePDF result for a raster image",+ expect.objectContaining({ cacheVariant: "ocr" }),+ );+ });++ it("still serves a cached image result that has text", async () => {+ getCached.mockResolvedValueOnce({+ markdown: "# Title",+ html: "<h1>Title</h1>",+ });++ const result = await tryGetCached(+ makeMeta(),+ PNG_BASE64,+ "ocr",+ undefined,+ 1,+ false,+ false,+ );++ expect(result).toMatchObject({ markdown: "# Title" });+ });++ it("keeps serving cached empty results for PDFs", async () => {+ getCached.mockResolvedValueOnce({ markdown: "", html: "" });++ const result = await tryGetCached(+ makeMeta(),+ PDF_BASE64,+ "auto",+ undefined,+ 1,+ false,+ false,+ );++ expect(result).toMatchObject({ markdown: "", html: "" });+ });++ it("never writes an empty image result, but writes empty PDF and non-empty image results", async () => {+ const save = (base64Content: string, markdown: string) =>+ maybeSaveResult({+ meta: makeMeta(),+ base64Content,+ mode: "ocr",+ maxPages: undefined,+ includePageMarkdown: false,+ includeBlocks: false,+ result: {+ markdown,+ html: markdown ? `<p>${markdown}</p>` : "",+ pagesProcessed: 1,+ },+ });++ await save(PNG_BASE64, "");+ expect(saveCached).not.toHaveBeenCalled();++ await save(PNG_BASE64, " \n");+ expect(saveCached).not.toHaveBeenCalled();++ await save(PDF_BASE64, "");+ expect(saveCached).toHaveBeenCalledTimes(1);++ await save(PNG_BASE64, "text");+ expect(saveCached).toHaveBeenCalledTimes(2);+ expect(saveCached).toHaveBeenLastCalledWith(+ PNG_BASE64,+ expect.objectContaining({ markdown: "text" }),+ "firepdf",+ "ocr",+ );+ });+});diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.tsindex 9c963edad..270f46aea 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts@@ -6,8 +6,28 @@ import {savePdfResultToCache,type PdfCacheKeyInput,} from "../../../../../lib/gcs-pdf-cache";+import { sniffImageContentTypeFromBase64 } from "../../../../../lib/image-formats";import { firePdfBlockPagesSchema } from "./schema";+// Raster images ride the same cache as PDFs (the image engine posts their+// bytes to the same FirePDF endpoint), but an EMPTY result for an image is+// not worth remembering: it is cheap to recompute, and it usually records a+// no-text gate decision or a transient failure rather than a property of the+// bytes. Cached empties outlived a gate change once and kept legible images+// blank for every later scrape of the same file, so they are neither served+// nor written for images. PDFs keep their empty results — a blank scan is a+// real, expensive-to-redo answer there. By-reference payloads (`{ key }`)+// are always PDFs.+function isRasterImagePayload(input: PdfCacheKeyInput): boolean {+ return (+ typeof input === "string" && sniffImageContentTypeFromBase64(input) !== null+ );+}++function isEmptyMarkdown(markdown: string): boolean {+ return markdown.trim().length === 0;+}+// Cache layout mirrors the sync `scrapePDFWithFirePDF` so async/sync share// entries. `fast` mode bypasses entirely (hard cost ceiling — must fail on// scanned PDFs, not serve a cached OCR result), as does any call with@@ -179,6 +199,23 @@ export async function tryGetCached(// never let a malformed/old artifact satisfy a cache lookup.continue;}+ if (+ isEmptyMarkdown(cached.markdown) &&+ isRasterImagePayload(base64Content)+ ) {+ // See isRasterImagePayload: an empty image result is a stale+ // verdict, not an answer. Re-run OCR and let the fresh result+ // decide (an empty one is not written back either).+ meta.logger.info(+ "Ignoring cached empty FirePDF result for a raster image",+ {+ scrapeId: meta.id,+ requestedMode: mode,+ cacheVariant: variant ?? "base",+ },+ );+ continue;+ }meta.logger.info("Using cached FirePDF result", {scrapeId: meta.id,requestedMode: mode,@@ -233,6 +270,9 @@ export async function maybeSaveResult(args: {pageMarkers,);if (!cacheable) return;+ // See isRasterImagePayload: never remember an empty result for an image.+ if (isEmptyMarkdown(result.markdown) && isRasterImagePayload(base64Content))+ return;try {await savePdfResultToCache(base64Content, result, "firepdf", ownVariant);