Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): never serve or store an empty FirePDF result for a raster image (#4515)

Abimael Martell70d992234730
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts
index 24c88fcd7..e812bab65 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts
@@ -611,3 +611,112 @@ describe("FirePDF page-markdown cache capabilities", () => {
);
});
});
+
+describe("FirePDF cache and empty raster-image results", () => {
+ // Base64 of a PNG signature followed by padding: sniffs as image/png.
+ const PNG_BASE64 = Buffer.from([
+ 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 0, 0, 0, 0, 0,
+ ]).toString("base64");
+ // Base64 of "%PDF-1.7": not an image, so the PDF rules apply.
+ const PDF_BASE64 = Buffer.from(
+ "%PDF-1.7\n%\xe2\xe3\xcf\xd3\n",
+ "latin1",
+ ).toString("base64");
+
+ beforeEach(() => {
+ vi.clearAllMocks();
+ getCached.mockResolvedValue(null);
+ saveCached.mockResolvedValue(null);
+ });
+
+ it("treats a cached empty result for an image as a miss", async () => {
+ getCached.mockResolvedValue({ markdown: "", html: "" });
+ const meta = makeMeta();
+
+ const result = await tryGetCached(
+ meta,
+ PNG_BASE64,
+ "ocr",
+ undefined,
+ 1,
+ false,
+ false,
+ );
+
+ expect(result).toBeNull();
+ expect(meta.logger.info).toHaveBeenCalledWith(
+ "Ignoring cached empty FirePDF result for a raster image",
+ expect.objectContaining({ cacheVariant: "ocr" }),
+ );
+ });
+
+ it("still serves a cached image result that has text", async () => {
+ getCached.mockResolvedValueOnce({
+ markdown: "# Title",
+ html: "<h1>Title</h1>",
+ });
+
+ const result = await tryGetCached(
+ makeMeta(),
+ PNG_BASE64,
+ "ocr",
+ undefined,
+ 1,
+ false,
+ false,
+ );
+
+ expect(result).toMatchObject({ markdown: "# Title" });
+ });
+
+ it("keeps serving cached empty results for PDFs", async () => {
+ getCached.mockResolvedValueOnce({ markdown: "", html: "" });
+
+ const result = await tryGetCached(
+ makeMeta(),
+ PDF_BASE64,
+ "auto",
+ undefined,
+ 1,
+ false,
+ false,
+ );
+
+ expect(result).toMatchObject({ markdown: "", html: "" });
+ });
+
+ it("never writes an empty image result, but writes empty PDF and non-empty image results", async () => {
+ const save = (base64Content: string, markdown: string) =>
+ maybeSaveResult({
+ meta: makeMeta(),
+ base64Content,
+ mode: "ocr",
+ maxPages: undefined,
+ includePageMarkdown: false,
+ includeBlocks: false,
+ result: {
+ markdown,
+ html: markdown ? `<p>${markdown}</p>` : "",
+ pagesProcessed: 1,
+ },
+ });
+
+ await save(PNG_BASE64, "");
+ expect(saveCached).not.toHaveBeenCalled();
+
+ await save(PNG_BASE64, " \n");
+ expect(saveCached).not.toHaveBeenCalled();
+
+ await save(PDF_BASE64, "");
+ expect(saveCached).toHaveBeenCalledTimes(1);
+
+ await save(PNG_BASE64, "text");
+ expect(saveCached).toHaveBeenCalledTimes(2);
+ expect(saveCached).toHaveBeenLastCalledWith(
+ PNG_BASE64,
+ expect.objectContaining({ markdown: "text" }),
+ "firepdf",
+ "ocr",
+ );
+ });
+});
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts
index 9c963edad..270f46aea 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts
@@ -6,8 +6,28 @@ import {
savePdfResultToCache,
type PdfCacheKeyInput,
} from "../../../../../lib/gcs-pdf-cache";
+import { sniffImageContentTypeFromBase64 } from "../../../../../lib/image-formats";
import { firePdfBlockPagesSchema } from "./schema";
+// Raster images ride the same cache as PDFs (the image engine posts their
+// bytes to the same FirePDF endpoint), but an EMPTY result for an image is
+// not worth remembering: it is cheap to recompute, and it usually records a
+// no-text gate decision or a transient failure rather than a property of the
+// bytes. Cached empties outlived a gate change once and kept legible images
+// blank for every later scrape of the same file, so they are neither served
+// nor written for images. PDFs keep their empty results — a blank scan is a
+// real, expensive-to-redo answer there. By-reference payloads (`{ key }`)
+// are always PDFs.
+function isRasterImagePayload(input: PdfCacheKeyInput): boolean {
+ return (
+ typeof input === "string" && sniffImageContentTypeFromBase64(input) !== null
+ );
+}
+
+function isEmptyMarkdown(markdown: string): boolean {
+ return markdown.trim().length === 0;
+}
+
// Cache layout mirrors the sync `scrapePDFWithFirePDF` so async/sync share
// entries. `fast` mode bypasses entirely (hard cost ceiling — must fail on
// scanned PDFs, not serve a cached OCR result), as does any call with
@@ -179,6 +199,23 @@ export async function tryGetCached(
// never let a malformed/old artifact satisfy a cache lookup.
continue;
}
+ if (
+ isEmptyMarkdown(cached.markdown) &&
+ isRasterImagePayload(base64Content)
+ ) {
+ // See isRasterImagePayload: an empty image result is a stale
+ // verdict, not an answer. Re-run OCR and let the fresh result
+ // decide (an empty one is not written back either).
+ meta.logger.info(
+ "Ignoring cached empty FirePDF result for a raster image",
+ {
+ scrapeId: meta.id,
+ requestedMode: mode,
+ cacheVariant: variant ?? "base",
+ },
+ );
+ continue;
+ }
meta.logger.info("Using cached FirePDF result", {
scrapeId: meta.id,
requestedMode: mode,
@@ -233,6 +270,9 @@ export async function maybeSaveResult(args: {
pageMarkers,
);
if (!cacheable) return;
+ // See isRasterImagePayload: never remember an empty result for an image.
+ if (isEmptyMarkdown(result.markdown) && isRasterImagePayload(base64Content))
+ return;
try {
await savePdfResultToCache(base64Content, result, "firepdf", ownVariant);