Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

feat(pdf): store fire-pdf provenance with cache entries, count cache events, refuse degraded writes, add refresh option (#4687)

Abimael Martell791bebb4c915
diff --git a/apps/api/src/__tests__/pdf-parser-options.test.ts b/apps/api/src/__tests__/pdf-parser-options.test.ts
index 118317407..dce6926de 100644
--- a/apps/api/src/__tests__/pdf-parser-options.test.ts
+++ b/apps/api/src/__tests__/pdf-parser-options.test.ts
@@ -3,6 +3,7 @@ import {
getPDFMode,
getPDFPageMarkdown,
getPDFPageMarkers,
+ getPDFRefresh,
scrapeOptions,
shouldParseImages,
shouldParsePDF,
@@ -111,3 +112,20 @@ describe("image parser", () => {
).toBe(false);
});
});
+
+describe("pdf parser refresh option", () => {
+ it("is off unless the pdf parser object sets it", () => {
+ expect(getPDFRefresh(undefined)).toBe(false);
+ expect(getPDFRefresh(["pdf"])).toBe(false);
+ expect(getPDFRefresh([{ type: "pdf" }])).toBe(false);
+ expect(getPDFRefresh([{ type: "pdf", refresh: false }])).toBe(false);
+ expect(getPDFRefresh([{ type: "pdf", refresh: true }])).toBe(true);
+ });
+
+ it("parses through the strict schema", () => {
+ expect(parsePdfParser({ type: "pdf", refresh: true })).toMatchObject({
+ type: "pdf",
+ refresh: true,
+ });
+ });
+});
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index 8d3d9436a..a00b526b6 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -388,6 +388,10 @@ const configSchema = z.object({
FIRE_PDF_PERCENT: z.coerce.number().min(0).max(100).default(10),
FIRE_PDF_BASE_URL: z.string().optional(),
FIRE_PDF_API_KEY: z.string().optional(),
+ // `parsers: [{ type: "pdf", refresh: true }]` skips the content cache and
+ // forces a fresh parse. Per team, per minute; beyond the budget the
+ // request is served normally. 0 disables the option.
+ FIRE_PDF_CACHE_REFRESH_PER_MINUTE: z.coerce.number().int().min(0).default(10),
// Raster image OCR of image URLs and parse uploads through FirePDF (see
// lib/image-ocr-gate.ts). Needs FIRE_PDF_BASE_URL.
IMAGE_OCR_ENABLED: z.stringbool().default(false),
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 05cf16dc5..1ff4ef484 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -508,6 +508,11 @@ const pdfParserWithOptions = z
* cross-page stitching β€” callers that need every physical page should
* use `pages: true` instead. No new response field. */
pageMarkers: z.boolean().optional(),
+ /** Skip the cached conversion for this document and parse it again with
+ * the current pipeline; the fresh result overwrites the cache entry for
+ * everyone. Billed like a fresh parse. Use it when a cached result is
+ * wrong or outdated. */
+ refresh: z.boolean().optional(),
// Experimental: route this request through the fire-pdf async pipeline
// (POST /jobs + poll) instead of the sync POST /ocr endpoint. Falls back
// to sync on any async-path failure, so user-visible behavior is unchanged
@@ -639,6 +644,17 @@ export function getPDFPageMarkers(parsers?: Parsers): boolean {
return false;
}
+/** `parsers: [{ type: "pdf", refresh: true }]`: bypass the content cache for this request. */
+export function getPDFRefresh(parsers?: Parsers): boolean {
+ if (!parsers) return false;
+ for (const parser of parsers) {
+ if (typeof parser === "object" && parser.type === "pdf") {
+ return parser.refresh === true;
+ }
+ }
+ return false;
+}
+
export function getFirePdfAsync(parsers?: Parsers): boolean {
if (!parsers) return false;
for (const parser of parsers) {
diff --git a/apps/api/src/lib/gcs-pdf-cache.ts b/apps/api/src/lib/gcs-pdf-cache.ts
index 4f2465af5..6fb0a9684 100644
--- a/apps/api/src/lib/gcs-pdf-cache.ts
+++ b/apps/api/src/lib/gcs-pdf-cache.ts
@@ -1,5 +1,6 @@
import { ApiError } from "@google-cloud/storage";
import type { FirePdfPageBlocks } from "../scraper/scrapeURL/engines/pdf/types";
+import type { FirePdfProvenance } from "../scraper/scrapeURL/engines/pdf/fire-pdf/schema";
import { logger } from "./logger";
import { config } from "../config";
import crypto from "crypto";
@@ -10,7 +11,7 @@ type PdfCacheProvider = "runpod" | "firepdf";
// Cache shape β€” markdown/html are required; pagesProcessed is optional so
// pre-existing entries (written before the field existed) round-trip cleanly
// and the caller can fall back to its own page-count signal on a stale hit.
-type CachedPdfResult = {
+export type CachedPdfResult = {
markdown: string;
html: string;
pagesProcessed?: number;
@@ -19,6 +20,13 @@ type CachedPdfResult = {
/** Typed layout blocks (fire-pdf wire shape); present only in
* block-capable cache variants. */
blocks?: FirePdfPageBlocks[];
+ /** fire-pdf's stamp for the result this entry holds. Absent on entries
+ * written before it existed, which a cache policy reads as "unknown". */
+ provenance?: FirePdfProvenance;
+ /** When this entry was written (ISO-8601). */
+ cachedAt?: string;
+ /** The variant this entry was written under ("base" for the bare key). */
+ variant?: string;
};
const PROVIDER_PREFIXES: Record<PdfCacheProvider, string> = {
@@ -37,10 +45,20 @@ export function createPdfCacheKey(pdfContent: string | Buffer): string {
* distinct. */
export type PdfCacheKeyInput = string | { key: string };
-function resolvePdfCacheKey(input: PdfCacheKeyInput): string {
+/** The cache key for an input: sha256 of the inline base64 payload, or the
+ * caller's precomputed `raw-<sha256>` key. Logged on every cache event so a
+ * team report can be turned into the keys to purge. */
+export function resolvePdfCacheKey(input: PdfCacheKeyInput): string {
return typeof input === "string" ? createPdfCacheKey(input) : input.key;
}
+/** Whether the content cache exists at all (self-hosted deployments may run
+ * without a bucket). Callers check this before hashing a payload or spending
+ * a refresh token for a cache that would neither read nor write. */
+export function pdfCacheConfigured(): boolean {
+ return !!config.GCS_BUCKET_NAME;
+}
+
export async function savePdfResultToCache(
pdfContent: PdfCacheKeyInput,
result: CachedPdfResult,
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts
index 008e14084..fffb7bcd5 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts
@@ -2,7 +2,10 @@
// client calls these on the happy path; without the stub the first cache lookup
// blows up trying to download from GCS using the credentials in .env.
vi.mock("../../../../../lib/gcs-pdf-cache", () => ({
+ pdfCacheConfigured: vi.fn(() => true),
createPdfCacheKey: (s: string) => `sha-${s.length}`,
+ resolvePdfCacheKey: (input: string | { key: string }) =>
+ typeof input === "string" ? `sha-${input.length}` : input.key,
getPdfResultFromCache: vi.fn(async () => null),
savePdfResultToCache: vi.fn(async () => null),
}));
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncDeadline.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncDeadline.test.ts
index be11af646..ceef41988 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncDeadline.test.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncDeadline.test.ts
@@ -1,7 +1,10 @@
// Stub the GCS cache so unit tests never reach real cloud storage β€” same
// setup as firePDFAsync.test.ts, which covers the orchestration end to end.
vi.mock("../../../../../lib/gcs-pdf-cache", () => ({
+ pdfCacheConfigured: vi.fn(() => true),
createPdfCacheKey: (s: string) => `sha-${s.length}`,
+ resolvePdfCacheKey: (input: string | { key: string }) =>
+ typeof input === "string" ? `sha-${input.length}` : input.key,
getPdfResultFromCache: vi.fn(async () => null),
savePdfResultToCache: vi.fn(async () => null),
}));
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts
index e812bab65..c34da25c5 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts
@@ -1,22 +1,39 @@
import {
getPdfResultFromCache,
+ pdfCacheConfigured,
savePdfResultToCache,
} from "../../../../../lib/gcs-pdf-cache";
import {
cacheKeyShape,
maybeSaveResult,
+ provenanceFromResponse,
tryGetCached,
} from "../fire-pdf/cache";
+import { consumeRefresh, refreshDecisionFor } from "../fire-pdf/refresh-budget";
+import { firePdfProvenanceSchema } from "../fire-pdf/schema";
+import {
+ firePdfCacheEventsTotal,
+ firePdfCacheRefusedWritesTotal,
+} from "../fire-pdf/metrics";
+
+vi.mock("../fire-pdf/refresh-budget", () => ({
+ consumeRefresh: vi.fn(async () => "allowed"),
+ refreshDecisionFor: vi.fn(() => undefined),
+}));
vi.mock("../../../../../lib/gcs-pdf-cache", () => ({
+ pdfCacheConfigured: vi.fn(() => true),
getPdfResultFromCache: vi.fn(),
savePdfResultToCache: vi.fn(),
+ resolvePdfCacheKey: vi.fn((input: string | { key: string }) =>
+ typeof input === "string" ? `key-of-${input}` : input.key,
+ ),
}));
const getCached = vi.mocked(getPdfResultFromCache);
const saveCached = vi.mocked(savePdfResultToCache);
-function makeMeta(zeroDataRetention = false) {
+function makeMeta(zeroDataRetention = false, parsers?: unknown[]) {
return {
id: "page-cache-test",
logger: {
@@ -24,14 +41,32 @@ function makeMeta(zeroDataRetention = false) {
warn: vi.fn(),
},
internalOptions: { zeroDataRetention },
+ ...(parsers ? { options: { parsers } } : {}),
} as any;
}
+const provenance = {
+ generation: "2026-09-16.1",
+ build_sha: "25c376ac15489dc6d5cab9af8728de41357ffc69",
+ built_at: "2026-09-17T20:11:39Z",
+ produced_at: "2026-09-17T20:12:00.000Z",
+ stages: ["native_text", "layout", "ocr"],
+ quality: {
+ total_pages: 3,
+ failed_pages: 0,
+ partial_pages: 0,
+ degraded_pages: 0,
+ ocr_pages: 1,
+ },
+};
+
describe("FirePDF page-markdown cache capabilities", () => {
beforeEach(() => {
vi.clearAllMocks();
getCached.mockResolvedValue(null);
- saveCached.mockResolvedValue(null);
+ // The real save returns the cache key on success and null when it could
+ // not persist; a null default would read as "not written".
+ saveCached.mockResolvedValue("saved-key");
});
it("uses versioned page-capable variants", () => {
@@ -626,7 +661,9 @@ describe("FirePDF cache and empty raster-image results", () => {
beforeEach(() => {
vi.clearAllMocks();
getCached.mockResolvedValue(null);
- saveCached.mockResolvedValue(null);
+ // The real save returns the cache key on success and null when it could
+ // not persist; a null default would read as "not written".
+ saveCached.mockResolvedValue("saved-key");
});
it("treats a cached empty result for an image as a miss", async () => {
@@ -720,3 +757,594 @@ describe("FirePDF cache and empty raster-image results", () => {
);
});
});
+
+describe("FirePDF cache provenance and write rules", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ getCached.mockResolvedValue(null);
+ // The real save returns the cache key on success and null when it could
+ // not persist; a null default would read as "not written".
+ saveCached.mockResolvedValue("saved-key");
+ });
+
+ it("stores the stamp, the write time and the variant with the entry", async () => {
+ await maybeSaveResult({
+ meta: makeMeta(),
+ base64Content: "BASE64",
+ mode: "auto",
+ maxPages: undefined,
+ includePageMarkdown: false,
+ includeBlocks: false,
+ result: { markdown: "whole", html: "<p>whole</p>", pagesProcessed: 3 },
+ provenance,
+ failedPages: null,
+ });
+
+ expect(saveCached).toHaveBeenCalledTimes(1);
+ const entry = saveCached.mock.calls[0][1];
+ expect(entry).toMatchObject({
+ markdown: "whole",
+ provenance,
+ variant: "base",
+ });
+ expect(typeof entry.cachedAt).toBe("string");
+ expect(Number.isNaN(Date.parse(entry.cachedAt!))).toBe(false);
+ });
+
+ it("stamps the compact legacy entry it back-fills beside a sidecar", async () => {
+ await maybeSaveResult({
+ meta: makeMeta(),
+ base64Content: "BASE64",
+ mode: "auto",
+ maxPages: undefined,
+ includePageMarkdown: true,
+ includeBlocks: false,
+ result: {
+ markdown: "whole",
+ html: "<p>whole</p>",
+ pagesProcessed: 2,
+ pageMarkdown: [{ page: 1, markdown: "one" }],
+ },
+ provenance,
+ });
+
+ expect(saveCached).toHaveBeenCalledTimes(2);
+ expect(saveCached.mock.calls[0][1]).toMatchObject({
+ provenance,
+ variant: "page-markdown-v1",
+ });
+ expect(saveCached.mock.calls[1][1]).toMatchObject({
+ provenance,
+ variant: "base",
+ });
+ });
+
+ it("writes entries without a stamp when fire-pdf sent none", async () => {
+ await maybeSaveResult({
+ meta: makeMeta(),
+ base64Content: "BASE64",
+ mode: "auto",
+ maxPages: undefined,
+ includePageMarkdown: false,
+ includeBlocks: false,
+ result: { markdown: "whole", html: "<p>whole</p>" },
+ });
+ expect(saveCached).toHaveBeenCalledTimes(1);
+ expect(saveCached.mock.calls[0][1]).not.toHaveProperty("provenance");
+ expect(saveCached.mock.calls[0][1]).toMatchObject({ variant: "base" });
+ });
+
+ it("does not cache a result with failed pages, and says why", async () => {
+ const meta = makeMeta();
+ await maybeSaveResult({
+ meta,
+ base64Content: "BASE64",
+ mode: "auto",
+ maxPages: undefined,
+ includePageMarkdown: false,
+ includeBlocks: false,
+ result: { markdown: "most of it", html: "<p>most of it</p>" },
+ provenance,
+ failedPages: [4],
+ });
+ expect(saveCached).not.toHaveBeenCalled();
+ expect(meta.logger.info).toHaveBeenCalledWith(
+ "FirePDF result not cached",
+ expect.objectContaining({
+ reason: "failed_pages",
+ cacheKey: "key-of-BASE64",
+ }),
+ );
+ });
+
+ it("does not cache a result whose pages lost layout (degraded), but does cache partial pages", async () => {
+ await maybeSaveResult({
+ meta: makeMeta(),
+ base64Content: "BASE64",
+ mode: "auto",
+ maxPages: undefined,
+ includePageMarkdown: false,
+ includeBlocks: false,
+ result: { markdown: "text only", html: "<p>text only</p>" },
+ provenance: {
+ ...provenance,
+ quality: { ...provenance.quality, degraded_pages: 2 },
+ },
+ failedPages: null,
+ });
+ expect(saveCached).not.toHaveBeenCalled();
+
+ await maybeSaveResult({
+ meta: makeMeta(),
+ base64Content: "BASE64",
+ mode: "auto",
+ maxPages: undefined,
+ includePageMarkdown: false,
+ includeBlocks: false,
+ result: { markdown: "nearly all", html: "<p>nearly all</p>" },
+ provenance: {
+ ...provenance,
+ quality: { ...provenance.quality, partial_pages: 1 },
+ },
+ failedPages: null,
+ });
+ expect(saveCached).toHaveBeenCalledTimes(1);
+ expect(saveCached.mock.calls[0][1]).toMatchObject({
+ provenance: expect.objectContaining({
+ quality: expect.objectContaining({ partial_pages: 1 }),
+ }),
+ });
+ });
+
+ it("serves a stamped entry without leaking the bookkeeping fields, and logs the key and generation", async () => {
+ getCached.mockResolvedValueOnce({
+ markdown: "whole",
+ html: "<p>whole</p>",
+ pagesProcessed: 3,
+ provenance,
+ cachedAt: "2026-09-17T20:12:01.000Z",
+ variant: "base",
+ });
+ const meta = makeMeta();
+ const result = await tryGetCached(
+ meta,
+ "BASE64",
+ "auto",
+ undefined,
+ undefined,
+ false,
+ false,
+ );
+ expect(result).toEqual({
+ markdown: "whole",
+ html: "<p>whole</p>",
+ pagesProcessed: 3,
+ });
+ expect(meta.logger.info).toHaveBeenCalledWith(
+ "Using cached FirePDF result",
+ expect.objectContaining({
+ cacheKey: "key-of-BASE64",
+ generation: "2026-09-16.1",
+ buildSha: provenance.build_sha,
+ cachedAt: "2026-09-17T20:12:01.000Z",
+ }),
+ );
+ });
+
+ it("reads an entry written before the stamp existed as unknown provenance", async () => {
+ getCached.mockResolvedValueOnce({ markdown: "old", html: "<p>old</p>" });
+ const meta = makeMeta();
+ await tryGetCached(meta, "BASE64", "auto", undefined, 1, false, false);
+ expect(meta.logger.info).toHaveBeenCalledWith(
+ "Using cached FirePDF result",
+ expect.objectContaining({
+ generation: "unknown",
+ buildSha: "unknown",
+ cachedAt: null,
+ }),
+ );
+ });
+
+ it("bypasses the read, but not the write, when the pdf parser asks for a refresh", async () => {
+ getCached.mockResolvedValue({ markdown: "stale", html: "<p>stale</p>" });
+ const meta = makeMeta(false, [{ type: "pdf", refresh: true }]);
+ const result = await tryGetCached(
+ meta,
+ "BASE64",
+ "auto",
+ undefined,
+ 1,
+ false,
+ false,
+ );
+ expect(result).toBeNull();
+ expect(getCached).not.toHaveBeenCalled();
+ expect(meta.logger.info).toHaveBeenCalledWith(
+ "FirePDF cache bypassed by refresh",
+ expect.objectContaining({ cacheKey: "key-of-BASE64" }),
+ );
+
+ await maybeSaveResult({
+ meta,
+ base64Content: "BASE64",
+ mode: "auto",
+ maxPages: undefined,
+ includePageMarkdown: false,
+ includeBlocks: false,
+ result: { markdown: "fresh", html: "<p>fresh</p>" },
+ provenance,
+ });
+ expect(saveCached).toHaveBeenCalledTimes(1);
+ });
+
+ it("serves the cache normally when the team's refresh budget is spent, and says so", async () => {
+ vi.mocked(consumeRefresh).mockResolvedValueOnce("limited");
+ getCached.mockResolvedValue({ markdown: "cached", html: "<p>cached</p>" });
+ const meta = makeMeta(false, [{ type: "pdf", refresh: true }]);
+ const result = await tryGetCached(
+ meta,
+ "BASE64",
+ "auto",
+ undefined,
+ 1,
+ false,
+ false,
+ );
+ expect(result?.markdown).toBe("cached");
+ expect(getCached).toHaveBeenCalled();
+ expect(meta.logger.warn).toHaveBeenCalledWith(
+ "FirePDF cache refresh not applied",
+ expect.objectContaining({
+ decision: "limited",
+ cacheKey: "key-of-BASE64",
+ }),
+ );
+ });
+ it("does not report a write the cache layer could not persist", async () => {
+ saveCached.mockResolvedValueOnce(null);
+ const meta = makeMeta();
+ await maybeSaveResult({
+ meta,
+ base64Content: "BASE64",
+ mode: "auto",
+ maxPages: undefined,
+ includePageMarkdown: false,
+ includeBlocks: false,
+ result: { markdown: "whole", html: "<p>whole</p>" },
+ provenance,
+ });
+ expect(saveCached).toHaveBeenCalledTimes(1);
+ expect(meta.logger.warn).toHaveBeenCalledWith(
+ "FirePDF result not persisted to cache",
+ expect.objectContaining({ cacheKey: "key-of-BASE64" }),
+ );
+ expect(meta.logger.info).not.toHaveBeenCalledWith(
+ "Saved FirePDF result to cache",
+ expect.anything(),
+ );
+ });
+
+ it("refuses a result whose stamp reports failed pages when the list is absent", async () => {
+ const meta = makeMeta();
+ await maybeSaveResult({
+ meta,
+ base64Content: "BASE64",
+ mode: "auto",
+ maxPages: undefined,
+ includePageMarkdown: false,
+ includeBlocks: false,
+ result: { markdown: "whole", html: "<p>whole</p>" },
+ provenance: {
+ ...provenance,
+ quality: { ...provenance.quality, failed_pages: 2 },
+ },
+ failedPages: undefined,
+ });
+ expect(saveCached).not.toHaveBeenCalled();
+ expect(meta.logger.info).toHaveBeenCalledWith(
+ "FirePDF result not cached",
+ expect.objectContaining({ reason: "failed_pages", failedPages: 2 }),
+ );
+ });
+
+ it("refuses a result whose stamp could not be read", async () => {
+ const meta = makeMeta();
+ await maybeSaveResult({
+ meta,
+ base64Content: "BASE64",
+ mode: "auto",
+ maxPages: undefined,
+ includePageMarkdown: false,
+ includeBlocks: false,
+ result: { markdown: "whole", html: "<p>whole</p>" },
+ provenance: null,
+ failedPages: [],
+ });
+ expect(saveCached).not.toHaveBeenCalled();
+ expect(meta.logger.info).toHaveBeenCalledWith(
+ "FirePDF result not cached",
+ expect.objectContaining({ reason: "malformed_provenance" }),
+ );
+ });
+
+ it("reads the stamp apart from the document and never throws", () => {
+ const logger = { warn: vi.fn(), info: vi.fn() } as any;
+ const context = { scrapeId: "s1", cacheKey: "key-of-BASE64" };
+ expect(provenanceFromResponse(provenance, logger, context)).toEqual(
+ provenance,
+ );
+ expect(provenanceFromResponse(undefined, logger, context)).toBeUndefined();
+ expect(logger.warn).not.toHaveBeenCalled();
+ expect(
+ provenanceFromResponse(
+ {
+ ...provenance,
+ quality: { ...provenance.quality, failed_pages: "2" },
+ },
+ logger,
+ context,
+ ),
+ ).toBeNull();
+ expect(provenanceFromResponse("garbage", logger, context)).toBeNull();
+ expect(logger.warn).toHaveBeenCalledWith(
+ "FirePDF provenance stamp not understood",
+ expect.objectContaining({
+ cacheKey: "key-of-BASE64",
+ issue: expect.stringContaining("quality.failed_pages"),
+ }),
+ );
+ });
+
+ it("rewrites the base alias on an allowed refresh even when one exists", async () => {
+ vi.mocked(refreshDecisionFor).mockReturnValueOnce("allowed");
+ const meta = makeMeta(false, [{ type: "pdf", refresh: true }]);
+ await maybeSaveResult({
+ meta,
+ base64Content: "BASE64",
+ mode: "auto",
+ maxPages: undefined,
+ includePageMarkdown: true,
+ includeBlocks: false,
+ result: {
+ markdown: "fresh",
+ html: "<p>fresh</p>",
+ pageMarkdown: [{ page: 1, markdown: "fresh" }],
+ },
+ provenance,
+ });
+ // No read of the existing alias: it is overwritten regardless.
+ expect(getCached).not.toHaveBeenCalled();
+ expect(saveCached).toHaveBeenCalledTimes(2);
+ expect(saveCached).toHaveBeenLastCalledWith(
+ "BASE64",
+ expect.objectContaining({ markdown: "fresh", variant: "base" }),
+ "firepdf",
+ undefined,
+ );
+ });
+
+ it("refuses a stamped result that carries no quality counts", async () => {
+ const meta = makeMeta();
+ const { quality: _quality, ...withoutQuality } = provenance;
+ await maybeSaveResult({
+ meta,
+ base64Content: "BASE64",
+ mode: "auto",
+ maxPages: undefined,
+ includePageMarkdown: false,
+ includeBlocks: false,
+ result: { markdown: "whole", html: "<p>whole</p>" },
+ provenance: withoutQuality,
+ failedPages: [],
+ });
+ expect(saveCached).not.toHaveBeenCalled();
+ expect(meta.logger.info).toHaveBeenCalledWith(
+ "FirePDF result not cached",
+ expect.objectContaining({ reason: "missing_quality" }),
+ );
+ });
+
+ it("treats an explicit null stamp as unreadable, not as absent", () => {
+ const logger = { warn: vi.fn(), info: vi.fn() } as any;
+ expect(
+ provenanceFromResponse(null, logger, {
+ scrapeId: "s1",
+ cacheKey: "key-of-BASE64",
+ }),
+ ).toBeNull();
+ expect(logger.warn).toHaveBeenCalledWith(
+ "FirePDF provenance stamp not understood",
+ expect.objectContaining({ issue: expect.stringContaining("null") }),
+ );
+ });
+
+ it("keeps fields a newer fire-pdf adds to a contributing build", () => {
+ const parsed = firePdfProvenanceSchema.parse({
+ ...provenance,
+ contributing_builds: [
+ {
+ generation: "2026-09-16.1",
+ build_sha: "abc",
+ built_at: null,
+ lane: "heavy",
+ },
+ ],
+ });
+ expect(parsed.contributing_builds?.[0]).toMatchObject({ lane: "heavy" });
+ });
+
+ it("logs an alias write the cache layer could not persist", async () => {
+ saveCached.mockResolvedValueOnce("saved-key").mockResolvedValueOnce(null);
+ const meta = makeMeta();
+ await maybeSaveResult({
+ meta,
+ base64Content: "BASE64",
+ mode: "auto",
+ maxPages: undefined,
+ includePageMarkdown: true,
+ includeBlocks: false,
+ result: {
+ markdown: "whole",
+ html: "<p>whole</p>",
+ pageMarkdown: [{ page: 1, markdown: "whole" }],
+ },
+ provenance,
+ });
+ expect(saveCached).toHaveBeenCalledTimes(2);
+ expect(meta.logger.info).toHaveBeenCalledWith(
+ "Saved FirePDF result to cache",
+ expect.objectContaining({
+ cacheVariant: "page-markdown-v1",
+ alias: false,
+ cacheKey: "key-of-BASE64",
+ }),
+ );
+ expect(meta.logger.warn).toHaveBeenCalledWith(
+ "FirePDF result not persisted to cache",
+ expect.objectContaining({
+ cacheVariant: "base",
+ alias: true,
+ cacheKey: "key-of-BASE64",
+ }),
+ );
+ });
+
+ it("rejects a stamp whose page counts are negative or fractional", () => {
+ expect(firePdfProvenanceSchema.safeParse(provenance).success).toBe(true);
+ for (const bad of [
+ { ...provenance.quality, degraded_pages: -1 },
+ { ...provenance.quality, failed_pages: 0.5 },
+ ]) {
+ expect(
+ firePdfProvenanceSchema.safeParse({ ...provenance, quality: bad })
+ .success,
+ ).toBe(false);
+ }
+ });
+});
+
+// The counters are the headline deliverable: spy on the real prom-client
+// counters so a change to an event name, a label or a refusal reason fails
+// here instead of silently on a dashboard.
+describe("FirePDF cache counters", () => {
+ const events = vi.spyOn(firePdfCacheEventsTotal, "inc");
+ const refused = vi.spyOn(firePdfCacheRefusedWritesTotal, "inc");
+
+ beforeEach(() => {
+ vi.clearAllMocks();
+ getCached.mockResolvedValue(null);
+ saveCached.mockResolvedValue("saved-key");
+ vi.mocked(consumeRefresh).mockResolvedValue("allowed");
+ });
+
+ const save = (
+ overrides: Partial<Parameters<typeof maybeSaveResult>[0]> = {},
+ ) =>
+ maybeSaveResult({
+ meta: makeMeta(),
+ base64Content: "BASE64",
+ mode: "auto",
+ maxPages: undefined,
+ includePageMarkdown: false,
+ includeBlocks: false,
+ result: { markdown: "whole", html: "<p>whole</p>" },
+ provenance,
+ failedPages: [],
+ ...overrides,
+ });
+
+ const read = (meta = makeMeta()) =>
+ tryGetCached(meta, "BASE64", "auto", undefined, 1, false, false);
+
+ it("counts a persisted write, and a write that was not persisted", async () => {
+ await save();
+ expect(events).toHaveBeenCalledWith({ event: "write", variant: "base" });
+ saveCached.mockResolvedValueOnce(null);
+ await save();
+ expect(events).toHaveBeenCalledWith({
+ event: "write_failed",
+ variant: "base",
+ });
+ });
+
+ it("counts the alias write under its own variant", async () => {
+ saveCached.mockResolvedValueOnce("saved-key").mockResolvedValueOnce(null);
+ await save({
+ includePageMarkdown: true,
+ result: {
+ markdown: "whole",
+ html: "<p>whole</p>",
+ pageMarkdown: [{ page: 1, markdown: "whole" }],
+ },
+ });
+ expect(events).toHaveBeenCalledWith({
+ event: "write",
+ variant: "page-markdown-v1",
+ });
+ expect(events).toHaveBeenCalledWith({
+ event: "write_failed",
+ variant: "base",
+ });
+ });
+
+ it("counts every refusal reason", async () => {
+ await save({ failedPages: [2] });
+ await save({
+ provenance: {
+ ...provenance,
+ quality: { ...provenance.quality, degraded_pages: 1 },
+ },
+ });
+ const { quality: _quality, ...withoutQuality } = provenance;
+ await save({ provenance: withoutQuality });
+ await save({ provenance: null });
+ for (const reason of [
+ "failed_pages",
+ "degraded_pages",
+ "missing_quality",
+ "malformed_provenance",
+ ]) {
+ expect(refused).toHaveBeenCalledWith({ reason });
+ }
+ expect(
+ events.mock.calls.filter(
+ ([labels]) => (labels as { event?: string }).event === "refused_write",
+ ),
+ ).toHaveLength(4);
+ expect(saveCached).not.toHaveBeenCalled();
+ });
+
+ it("does nothing, and counts nothing, when no cache is configured", async () => {
+ vi.mocked(pdfCacheConfigured).mockReturnValue(false);
+ try {
+ const meta = makeMeta(false, [{ type: "pdf", refresh: true }]);
+ await expect(read(meta)).resolves.toBeNull();
+ await save({ meta });
+ expect(getCached).not.toHaveBeenCalled();
+ expect(saveCached).not.toHaveBeenCalled();
+ expect(consumeRefresh).not.toHaveBeenCalled();
+ expect(events).not.toHaveBeenCalled();
+ } finally {
+ vi.mocked(pdfCacheConfigured).mockReturnValue(true);
+ }
+ });
+
+ it("counts hits, misses and both refresh outcomes", async () => {
+ await read();
+ expect(events).toHaveBeenCalledWith({ event: "miss", variant: "base" });
+ getCached.mockResolvedValueOnce({ markdown: "cached", html: "<p>c</p>" });
+ await read();
+ expect(events).toHaveBeenCalledWith({ event: "hit", variant: "base" });
+ await read(makeMeta(false, [{ type: "pdf", refresh: true }]));
+ expect(events).toHaveBeenCalledWith({
+ event: "bypass_refresh",
+ variant: "base",
+ });
+ vi.mocked(consumeRefresh).mockResolvedValueOnce("limited");
+ await read(makeMeta(false, [{ type: "pdf", refresh: true }]));
+ expect(events).toHaveBeenCalledWith({
+ event: "bypass_refresh_denied",
+ variant: "base",
+ });
+ });
+});
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/refreshBudget.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/refreshBudget.test.ts
new file mode 100644
index 000000000..c0dff780e
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/refreshBudget.test.ts
@@ -0,0 +1,84 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+// The limiter classes are hoisted so every re-import of the module under
+// test (vi.resetModules below) sees the same class identities; the
+// `instanceof RateLimiterRes` check in refresh-budget depends on that.
+const { consume, RateLimiterRedis, RateLimiterRes } = vi.hoisted(() => {
+ const consume = vi.fn();
+ class RateLimiterRes {}
+ class RateLimiterRedis {
+ consume = consume;
+ }
+ return { consume, RateLimiterRedis, RateLimiterRes };
+});
+
+vi.mock("rate-limiter-flexible", () => ({ RateLimiterRedis, RateLimiterRes }));
+
+vi.mock("../../../../../services/rate-limiter", () => ({
+ redisRateLimitClient: {},
+}));
+
+type Budget = typeof import("../fire-pdf/refresh-budget");
+
+describe("refresh budget", () => {
+ let budget: Budget;
+
+ beforeEach(async () => {
+ consume.mockReset();
+ consume.mockResolvedValue(undefined);
+ // A fresh module per test: the remembered decisions are module state,
+ // so no test can read what an earlier one left behind.
+ vi.resetModules();
+ budget = await import("../fire-pdf/refresh-budget.js");
+ });
+
+ it("spends one token per request and remembers the decision", async () => {
+ await expect(budget.consumeRefresh("team-a", "scrape-1")).resolves.toBe(
+ "allowed",
+ );
+ await expect(budget.consumeRefresh("team-a", "scrape-1")).resolves.toBe(
+ "allowed",
+ );
+ expect(consume).toHaveBeenCalledTimes(1);
+ expect(consume).toHaveBeenCalledWith("team-a", 1);
+ expect(budget.refreshDecisionFor("scrape-1")).toBe("allowed");
+ });
+
+ it("starts every test without remembered decisions", () => {
+ expect(budget.refreshDecisionFor("scrape-1")).toBeUndefined();
+ });
+
+ it("decides each request separately", async () => {
+ await budget.consumeRefresh("team-a", "scrape-1");
+ await budget.consumeRefresh("team-a", "scrape-2");
+ expect(consume).toHaveBeenCalledTimes(2);
+ });
+
+ it("keeps a denied decision for the same request too", async () => {
+ consume.mockRejectedValueOnce(new RateLimiterRes());
+ await expect(budget.consumeRefresh("team-b", "scrape-1")).resolves.toBe(
+ "limited",
+ );
+ // A later engine on the same request sees the same answer without
+ // asking the limiter again.
+ await expect(budget.consumeRefresh("team-b", "scrape-1")).resolves.toBe(
+ "limited",
+ );
+ expect(consume).toHaveBeenCalledTimes(1);
+ });
+
+ it("reports the limiter store being unreachable as unavailable", async () => {
+ consume.mockRejectedValueOnce(new Error("ECONNREFUSED"));
+ await expect(budget.consumeRefresh("team-c", "scrape-1")).resolves.toBe(
+ "unavailable",
+ );
+ });
+
+ it("decides afresh without a scrape id and knows nothing about unknown ids", async () => {
+ await budget.consumeRefresh("team-d");
+ await budget.consumeRefresh("team-d");
+ expect(consume).toHaveBeenCalledTimes(2);
+ expect(budget.refreshDecisionFor("never-seen")).toBeUndefined();
+ expect(budget.refreshDecisionFor(undefined)).toBeUndefined();
+ });
+});
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/runpodMUCache.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/runpodMUCache.test.ts
new file mode 100644
index 000000000..e8499d8cc
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/runpodMUCache.test.ts
@@ -0,0 +1,130 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import { scrapePDFWithRunPodMU } from "../runpodMU";
+import {
+ getPdfResultFromCache,
+ pdfCacheConfigured,
+ resolvePdfCacheKey,
+} from "../../../../../lib/gcs-pdf-cache";
+import { consumeRefresh } from "../fire-pdf/refresh-budget";
+
+vi.mock("../../../../../lib/gcs-pdf-cache", () => ({
+ pdfCacheConfigured: vi.fn(() => true),
+ getPdfResultFromCache: vi.fn(),
+ savePdfResultToCache: vi.fn(),
+ resolvePdfCacheKey: vi.fn(() => "key-of-BASE64"),
+}));
+
+vi.mock("../fire-pdf/refresh-budget", () => ({
+ consumeRefresh: vi.fn(async () => "allowed"),
+}));
+
+const getCached = vi.mocked(getPdfResultFromCache);
+const budget = vi.mocked(consumeRefresh);
+
+const cached = { markdown: "cached", html: "<p>cached</p>" };
+
+function makeMeta(parsers?: unknown[], zeroDataRetention = false) {
+ return {
+ id: "mu-cache-test",
+ logger: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), child: vi.fn() },
+ internalOptions: { teamId: "team-1", zeroDataRetention },
+ // Already aborted: the engine stops right after the cache step, before
+ // any network call, which is all these tests exercise.
+ abort: AbortSignal.abort(),
+ ...(parsers ? { options: { parsers } } : {}),
+ } as any;
+}
+
+describe("RunPod MU cache read and refresh", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ getCached.mockResolvedValue(cached as any);
+ budget.mockResolvedValue("allowed");
+ });
+
+ it("serves the cached result and leaves the refresh budget alone", async () => {
+ const meta = makeMeta();
+ await expect(
+ scrapePDFWithRunPodMU(meta, "/tmp/doc.pdf", "BASE64"),
+ ).resolves.toEqual(cached);
+ // The key is hashed once and handed to the cache layer as is.
+ expect(getCached).toHaveBeenCalledWith({ key: "key-of-BASE64" });
+ expect(budget).not.toHaveBeenCalled();
+ });
+
+ it("skips the cache read on refresh within the team budget", async () => {
+ const meta = makeMeta([{ type: "pdf", refresh: true }]);
+ await expect(
+ scrapePDFWithRunPodMU(meta, "/tmp/doc.pdf", "BASE64"),
+ ).rejects.toThrow();
+ expect(budget).toHaveBeenCalledWith("team-1", "mu-cache-test");
+ expect(getCached).not.toHaveBeenCalled();
+ expect(meta.logger.info).toHaveBeenCalledWith(
+ "RunPod MU cache bypassed by refresh",
+ expect.objectContaining({
+ tempFilePath: "/tmp/doc.pdf",
+ cacheKey: "key-of-BASE64",
+ }),
+ );
+ });
+
+ it("serves the cached result when the refresh budget is spent", async () => {
+ budget.mockResolvedValueOnce("limited");
+ const meta = makeMeta([{ type: "pdf", refresh: true }]);
+ await expect(
+ scrapePDFWithRunPodMU(meta, "/tmp/doc.pdf", "BASE64"),
+ ).resolves.toEqual(cached);
+ expect(getCached).toHaveBeenCalledWith({ key: "key-of-BASE64" });
+ expect(meta.logger.warn).toHaveBeenCalledWith(
+ "RunPod MU cache refresh not applied",
+ expect.objectContaining({
+ decision: "limited",
+ cacheKey: "key-of-BASE64",
+ }),
+ );
+ });
+
+ it("names the entry when the cache lookup itself fails", async () => {
+ getCached.mockRejectedValueOnce(new Error("gcs down"));
+ const meta = makeMeta();
+ await expect(
+ scrapePDFWithRunPodMU(meta, "/tmp/doc.pdf", "BASE64"),
+ ).rejects.toThrow();
+ expect(meta.logger.warn).toHaveBeenCalledWith(
+ "Error checking PDF cache, proceeding with RunPod MU",
+ expect.objectContaining({
+ cacheKey: "key-of-BASE64",
+ cacheProvider: "runpod",
+ }),
+ );
+ });
+
+ it("neither hashes nor spends the budget when no cache is configured", async () => {
+ vi.mocked(pdfCacheConfigured).mockReturnValueOnce(false);
+ const meta = makeMeta([{ type: "pdf", refresh: true }]);
+ await expect(
+ scrapePDFWithRunPodMU(meta, "/tmp/doc.pdf", "BASE64"),
+ ).rejects.toThrow();
+ expect(resolvePdfCacheKey).not.toHaveBeenCalled();
+ expect(getCached).not.toHaveBeenCalled();
+ expect(budget).not.toHaveBeenCalled();
+ });
+
+ it("never reads the cache for a zero-data-retention request", async () => {
+ const meta = makeMeta([{ type: "pdf", refresh: true }], true);
+ await expect(
+ scrapePDFWithRunPodMU(meta, "/tmp/doc.pdf", "BASE64"),
+ ).rejects.toThrow();
+ expect(getCached).not.toHaveBeenCalled();
+ expect(budget).not.toHaveBeenCalled();
+ });
+
+ it("does not spend the budget when maxPages already skips the cache", async () => {
+ const meta = makeMeta([{ type: "pdf", refresh: true }]);
+ await expect(
+ scrapePDFWithRunPodMU(meta, "/tmp/doc.pdf", "BASE64", 5),
+ ).rejects.toThrow();
+ expect(budget).not.toHaveBeenCalled();
+ expect(getCached).not.toHaveBeenCalled();
+ });
+});
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts
index 10c802085..3d9240adc 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts
@@ -7,7 +7,8 @@ import type { PDFProcessorResult } from "../types";
import { safeMarkdownToHtml } from "../markdownToHtml";
import { scrapePDFWithFirePDF } from "../firePDF";
import { cancelJob } from "./cancel";
-import { tryGetCached, maybeSaveResult } from "./cache";
+import { maybeSaveResult, provenanceFromResponse, tryGetCached } from "./cache";
+import { resolvePdfCacheKey } from "../../../../../lib/gcs-pdf-cache";
import {
firePdfAsyncAbandonedTotal,
firePdfAsyncTotalDurationSeconds,
@@ -403,6 +404,12 @@ export async function scrapePDFWithFirePDFAsync(
const durationMs = now() - overallStartedAt;
firePdfAsyncTotalDurationSeconds.observe(durationMs / 1000);
+ const cacheKey = resolvePdfCacheKey(cacheInput);
+ const provenance = provenanceFromResponse(fetched.provenance, meta.logger, {
+ scrapeId: meta.id,
+ cacheKey,
+ });
+
meta.logger.info("FirePDF async completed", {
scrapeId: meta.id,
durationMs,
@@ -413,6 +420,11 @@ export async function scrapePDFWithFirePDFAsync(
failedPages: fetched.failed_pages,
partialPages: fetched.partial_pages,
pollCount: polled.pollCount,
+ // The content-cache key and the producer, so a report can be turned
+ // into keys to purge and a result can be tied to a fire-pdf build.
+ cacheKey,
+ generation: provenance?.generation ?? "unknown",
+ buildSha: provenance?.build_sha ?? "unknown",
});
const processorResult: PDFProcessorResult & { markdown: string } = {
@@ -432,6 +444,8 @@ export async function scrapePDFWithFirePDFAsync(
includeBlocks,
pageMarkers,
result: processorResult,
+ provenance,
+ failedPages: fetched.failed_pages,
});
return processorResult;
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts
index 270f46aea..968094bc2 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts
@@ -1,13 +1,30 @@
import type { Meta } from "../../..";
-import type { PDFMode } from "../../../../../controllers/v2/types";
+import { config } from "../../../../../config";
+import {
+ getPDFRefresh,
+ type PDFMode,
+} from "../../../../../controllers/v2/types";
import type { PDFProcessorResult } from "../types";
import {
+ type CachedPdfResult,
getPdfResultFromCache,
+ pdfCacheConfigured,
+ resolvePdfCacheKey,
savePdfResultToCache,
type PdfCacheKeyInput,
} from "../../../../../lib/gcs-pdf-cache";
import { sniffImageContentTypeFromBase64 } from "../../../../../lib/image-formats";
-import { firePdfBlockPagesSchema } from "./schema";
+import {
+ type CacheRefusedReason,
+ firePdfCacheEventsTotal,
+ firePdfCacheRefusedWritesTotal,
+} from "./metrics";
+import { consumeRefresh, refreshDecisionFor } from "./refresh-budget";
+import {
+ firePdfBlockPagesSchema,
+ parseProvenance,
+ type FirePdfProvenance,
+} from "./schema";
// Raster images ride the same cache as PDFs (the image engine posts their
// bytes to the same FirePDF endpoint), but an EMPTY result for an image is
@@ -173,7 +190,9 @@ export async function tryGetCached(
pageMarkers = false,
): Promise<PDFProcessorResult | null> {
if (meta.internalOptions.zeroDataRetention) return null;
- const { cacheable, lookupVariants } = cacheKeyShape(
+ // No bucket, no cache: nothing to hash, no refresh budget to spend.
+ if (!pdfCacheConfigured()) return null;
+ const { cacheable, lookupVariants, ownVariant } = cacheKeyShape(
mode,
maxPages,
includePageMarkdown,
@@ -181,6 +200,39 @@ export async function tryGetCached(
pageMarkers,
);
if (!cacheable) return null;
+ const cacheKey = resolvePdfCacheKey(base64Content);
+
+ // `parsers: [{ type: "pdf", refresh: true }]`: the caller wants this
+ // document parsed again with the current pipeline. Within the team's
+ // budget the read is skipped; the fresh result is still written, so the
+ // entry is corrected for everyone. Over budget (or with the limiter
+ // unavailable) the request is served normally and the decision logged.
+ if (getPDFRefresh(meta.options?.parsers)) {
+ const decision = await consumeRefresh(meta.internalOptions.teamId, meta.id);
+ if (decision === "allowed") {
+ firePdfCacheEventsTotal.inc({
+ event: "bypass_refresh",
+ variant: ownVariant ?? "base",
+ });
+ meta.logger.info("FirePDF cache bypassed by refresh", {
+ scrapeId: meta.id,
+ requestedMode: mode,
+ cacheKey,
+ });
+ return null;
+ }
+ firePdfCacheEventsTotal.inc({
+ event: "bypass_refresh_denied",
+ variant: ownVariant ?? "base",
+ });
+ meta.logger.warn("FirePDF cache refresh not applied", {
+ scrapeId: meta.id,
+ requestedMode: mode,
+ cacheKey,
+ decision,
+ perMinute: config.FIRE_PDF_CACHE_REFRESH_PER_MINUTE,
+ });
+ }
for (const variant of lookupVariants) {
try {
@@ -216,14 +268,33 @@ export async function tryGetCached(
);
continue;
}
+ firePdfCacheEventsTotal.inc({
+ event: "hit",
+ variant: variant ?? "base",
+ });
meta.logger.info("Using cached FirePDF result", {
scrapeId: meta.id,
requestedMode: mode,
cacheVariant: variant ?? "base",
+ cacheKey,
+ // Provenance of the entry, for cache-policy work and team reports.
+ // Entries written before the stamp existed read as "unknown".
+ generation: cached.provenance?.generation ?? "unknown",
+ buildSha: cached.provenance?.build_sha ?? "unknown",
+ cachedAt: cached.cachedAt ?? null,
});
// Strip payloads the request didn't ask for so a richer sidecar
- // serves a poorer request without leaking extra capabilities.
- const { pageMarkdown, blocks, ...compactCached } = cached;
+ // serves a poorer request without leaking extra capabilities, and
+ // the entry-only fields (provenance, cachedAt, variant) that are
+ // cache bookkeeping rather than document content.
+ const {
+ pageMarkdown,
+ blocks,
+ provenance: _provenance,
+ cachedAt: _cachedAt,
+ variant: _variant,
+ ...compactCached
+ } = cached;
return {
...compactCached,
...(includePageMarkdown ? { pageMarkdown } : {}),
@@ -235,9 +306,60 @@ export async function tryGetCached(
meta.logger.warn("Error checking FirePDF cache, proceeding", {
error,
cacheVariant: variant ?? "base",
+ cacheKey,
});
}
}
+ firePdfCacheEventsTotal.inc({ event: "miss", variant: ownVariant ?? "base" });
+ return null;
+}
+
+/**
+ * fire-pdf's stamp off a response. `undefined`: none was sent (a build
+ * from before the stamp existed); the result is cached unstamped. `null`:
+ * a stamp was sent but this build cannot read it; the result is served but
+ * not cached, since an entry whose stamp cannot be judged later is worse
+ * than a miss. Never throws: the document does not depend on the stamp.
+ */
+export function provenanceFromResponse(
+ raw: unknown,
+ logger: Meta["logger"],
+ context: { scrapeId: string; cacheKey: string },
+): FirePdfProvenance | null | undefined {
+ const stamp = parseProvenance(raw);
+ if (stamp.status === "ok") return stamp.provenance;
+ if (stamp.status === "absent") return undefined;
+ logger.warn("FirePDF provenance stamp not understood", {
+ ...context,
+ issue: stamp.issue,
+ });
+ return null;
+}
+
+/**
+ * Why a result must not be remembered. Failed pages (fire-pdf's
+ * `failed_pages` list, or the stamp's `quality.failed_pages` when the list
+ * is absent) and pages that lost layout (`degraded_pages`) are usually
+ * transient β€” a fleet incident, a deadline β€” and a cached copy would serve
+ * that outcome to every later request for the document. A stamp this build
+ * cannot read (`provenance === null`) is refused, and so is a stamp without
+ * quality counts: "no counts" is not "no failures", and a later policy
+ * reads the same counts. Only an unstamped result (a build from before the
+ * stamp) is judged on the list alone. Partial pages are content-caused and
+ * expensive to redo, so those results are still cached; the stamp's counts
+ * let a later policy refresh them first.
+ */
+export function cacheRefusalReason(
+ failedPages: readonly number[] | null | undefined,
+ provenance: FirePdfProvenance | null | undefined,
+): CacheRefusedReason | null {
+ if (provenance === null) return "malformed_provenance";
+ if ((failedPages?.length ?? 0) > 0) return "failed_pages";
+ if (provenance === undefined) return null;
+ const quality = provenance.quality;
+ if (quality === undefined) return "missing_quality";
+ if (quality.failed_pages > 0) return "failed_pages";
+ if (quality.degraded_pages > 0) return "degraded_pages";
return null;
}
@@ -250,6 +372,14 @@ export async function maybeSaveResult(args: {
includeBlocks: boolean;
pageMarkers?: boolean;
result: PDFProcessorResult & { markdown: string };
+ /**
+ * fire-pdf's stamp for this result, stored verbatim with the entry.
+ * `null` means a stamp was sent but could not be read (see
+ * provenanceFromResponse); such a result is not cached.
+ */
+ provenance?: FirePdfProvenance | null;
+ /** fire-pdf's `failed_pages` for this result; a non-empty list is not cached. */
+ failedPages?: readonly number[] | null;
}): Promise<void> {
const {
meta,
@@ -260,8 +390,11 @@ export async function maybeSaveResult(args: {
includeBlocks,
pageMarkers = false,
result,
+ provenance,
+ failedPages,
} = args;
if (meta.internalOptions.zeroDataRetention) return;
+ if (!pdfCacheConfigured()) return;
const { cacheable, ownVariant, baseVariant } = cacheKeyShape(
mode,
maxPages,
@@ -273,32 +406,117 @@ export async function maybeSaveResult(args: {
// See isRasterImagePayload: never remember an empty result for an image.
if (isEmptyMarkdown(result.markdown) && isRasterImagePayload(base64Content))
return;
+ const cacheKey = resolvePdfCacheKey(base64Content);
+
+ const refusal = cacheRefusalReason(failedPages, provenance);
+ if (refusal !== null) {
+ firePdfCacheRefusedWritesTotal.inc({ reason: refusal });
+ firePdfCacheEventsTotal.inc({
+ event: "refused_write",
+ variant: ownVariant ?? "base",
+ });
+ meta.logger.info("FirePDF result not cached", {
+ scrapeId: meta.id,
+ requestedMode: mode,
+ cacheVariant: ownVariant ?? "base",
+ cacheKey,
+ reason: refusal,
+ failedPages: Math.max(
+ failedPages?.length ?? 0,
+ provenance?.quality?.failed_pages ?? 0,
+ ),
+ degradedPages: provenance?.quality?.degraded_pages ?? 0,
+ });
+ return;
+ }
+
+ const cachedAt = new Date().toISOString();
+ const entry: CachedPdfResult = {
+ ...result,
+ ...(provenance ? { provenance } : {}),
+ cachedAt,
+ variant: ownVariant ?? "base",
+ };
+
+ // savePdfResultToCache returns null (without throwing) when GCS is not
+ // configured or its retries are exhausted; only a persisted write is a
+ // write. Every outcome is counted and logged with its key and variant so
+ // a stale or missing entry can be traced to the write that produced it.
+ const recordWrite = (
+ savedKey: string | null,
+ variant: string | undefined,
+ alias: boolean,
+ ): boolean => {
+ const cacheVariant = variant ?? "base";
+ if (savedKey === null) {
+ firePdfCacheEventsTotal.inc({
+ event: "write_failed",
+ variant: cacheVariant,
+ });
+ meta.logger.warn("FirePDF result not persisted to cache", {
+ scrapeId: meta.id,
+ requestedMode: mode,
+ cacheVariant,
+ cacheKey,
+ alias,
+ });
+ return false;
+ }
+ firePdfCacheEventsTotal.inc({ event: "write", variant: cacheVariant });
+ meta.logger.info("Saved FirePDF result to cache", {
+ scrapeId: meta.id,
+ requestedMode: mode,
+ cacheVariant,
+ cacheKey,
+ alias,
+ generation: provenance?.generation ?? "unknown",
+ buildSha: provenance?.build_sha ?? "unknown",
+ });
+ return true;
+ };
try {
- await savePdfResultToCache(base64Content, result, "firepdf", ownVariant);
+ const savedKey = await savePdfResultToCache(
+ base64Content,
+ entry,
+ "firepdf",
+ ownVariant,
+ );
+ if (!recordWrite(savedKey, ownVariant, false)) return;
// An enriched (page/block-capable) parse is also a valid legacy result.
// Populate the compact base key when it is missing so a later legacy
// request never repeats the conversion. A sidecar miss can coexist with
- // a warm legacy key during rollout, so avoid rewriting that object.
- // Strip the enriched payloads to keep the hot-path cache object small.
+ // a warm legacy key during rollout, so avoid rewriting that object β€”
+ // except on an allowed refresh, where the old alias is exactly what the
+ // caller asked to correct: plain requests would keep being served the
+ // stale content otherwise. Strip the enriched payloads to keep the
+ // hot-path cache object small.
if ((includePageMarkdown || includeBlocks) && ownVariant !== baseVariant) {
- const existingBase = await getPdfResultFromCache(
- base64Content,
- "firepdf",
- baseVariant,
- );
+ const refreshed =
+ getPDFRefresh(meta.options?.parsers) &&
+ refreshDecisionFor(meta.id) === "allowed";
+ const existingBase = refreshed
+ ? null
+ : await getPdfResultFromCache(base64Content, "firepdf", baseVariant);
if (!existingBase || !isValidCachedDocument(existingBase)) {
const {
pageMarkdown: _pageMarkdown,
blocks: _blocks,
...baseResult
} = result;
- await savePdfResultToCache(
+ const baseEntry: CachedPdfResult = {
+ ...baseResult,
+ ...(provenance ? { provenance } : {}),
+ cachedAt,
+ variant: baseVariant ?? "base",
+ };
+ const savedBase = await savePdfResultToCache(
base64Content,
- baseResult,
+ baseEntry,
"firepdf",
baseVariant,
);
+ recordWrite(savedBase, baseVariant, true);
}
}
} catch (error) {
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.ts
index ecdb5acaf..1d8bec15f 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.ts
@@ -1,5 +1,23 @@
import { Counter, Histogram } from "prom-client";
+export const firePdfCacheEventsTotal = new Counter({
+ name: "firecrawl_fire_pdf_cache_events_total",
+ help: "fire-pdf content-cache events by outcome: hit, miss, bypass_refresh, bypass_refresh_denied, write, write_failed, refused_write",
+ labelNames: ["event", "variant"],
+});
+
+export const firePdfCacheRefusedWritesTotal = new Counter({
+ name: "firecrawl_fire_pdf_cache_refused_writes_total",
+ help: "fire-pdf results not written to the content cache, by reason; reasons: failed_pages, degraded_pages, missing_quality, malformed_provenance",
+ labelNames: ["reason"],
+});
+
+export type CacheRefusedReason =
+ | "failed_pages"
+ | "degraded_pages"
+ | "missing_quality"
+ | "malformed_provenance";
+
export const firePdfAsyncSubmittedTotal = new Counter({
name: "firecrawl_fire_pdf_async_submitted_total",
help: "Count of POST /jobs requests successfully submitted to fire-pdf async",
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/refresh-budget.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/refresh-budget.ts
new file mode 100644
index 000000000..e2c9262ea
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/refresh-budget.ts
@@ -0,0 +1,103 @@
+import { RateLimiterRedis, RateLimiterRes } from "rate-limiter-flexible";
+import { config } from "../../../../../config";
+
+/**
+ * Per-team budget for `parsers: [{ type: "pdf", refresh: true }]`.
+ *
+ * A refresh skips the content cache and forces a billed parse, so an
+ * unbounded stream of them could push every request of a team back onto
+ * fire-pdf. The budget is small (FIRE_PDF_CACHE_REFRESH_PER_MINUTE, default
+ * 10 a minute) and fails closed: when it is exhausted, or the limiter store
+ * is unreachable, the request is served from the cache like any other and
+ * the decision is logged. 0 disables the option entirely.
+ *
+ * One decision per request. Engines fall back to one another (fire-pdf,
+ * then RunPod MU) and each consults the budget, so the decision is
+ * remembered by scrape id: a fallback neither spends a second token nor
+ * flips to "serve the cached entry" after the first engine was allowed to
+ * bypass it. The write path reads the same decision to know whether it
+ * may overwrite existing aliases.
+ *
+ * The rate-limit Redis client is imported lazily so importing the cache
+ * module never opens a connection (tests, tooling).
+ */
+export type RefreshDecision =
+ | "allowed"
+ | "limited"
+ | "disabled"
+ | "unavailable";
+
+let limiter: RateLimiterRedis | null = null;
+
+async function budget(): Promise<RateLimiterRedis> {
+ if (limiter) return limiter;
+ const { redisRateLimitClient } = await import(
+ "../../../../../services/rate-limiter.js"
+ );
+ limiter = new RateLimiterRedis({
+ storeClient: redisRateLimitClient,
+ keyPrefix: "fire-pdf-cache-refresh",
+ points: Math.max(1, config.FIRE_PDF_CACHE_REFRESH_PER_MINUTE),
+ duration: 60,
+ });
+ return limiter;
+}
+
+// A request is over within minutes; the map is bounded so a burst of
+// requests cannot grow it without limit (insertion order = age order).
+const DECISION_TTL_MS = 15 * 60 * 1000;
+const DECISION_CAP = 20_000;
+const decisions = new Map<string, { decision: RefreshDecision; at: number }>();
+
+function prune(now: number): void {
+ for (const [scrapeId, entry] of decisions) {
+ if (now - entry.at < DECISION_TTL_MS && decisions.size < DECISION_CAP) {
+ break;
+ }
+ decisions.delete(scrapeId);
+ }
+}
+
+/** The decision already taken for this request, if any. */
+export function refreshDecisionFor(
+ scrapeId: string | undefined,
+): RefreshDecision | undefined {
+ if (!scrapeId) return undefined;
+ const entry = decisions.get(scrapeId);
+ if (!entry) return undefined;
+ if (Date.now() - entry.at >= DECISION_TTL_MS) {
+ decisions.delete(scrapeId);
+ return undefined;
+ }
+ return entry.decision;
+}
+
+async function decide(teamId: string | undefined): Promise<RefreshDecision> {
+ if (config.FIRE_PDF_CACHE_REFRESH_PER_MINUTE <= 0) return "disabled";
+ try {
+ await (await budget()).consume(teamId ?? "anonymous", 1);
+ return "allowed";
+ } catch (err) {
+ return err instanceof RateLimiterRes ? "limited" : "unavailable";
+ }
+}
+
+/**
+ * Spend one refresh of the team's budget for this request, or return the
+ * decision this request already got. Without a scrape id every call is a
+ * fresh decision.
+ */
+export async function consumeRefresh(
+ teamId: string | undefined,
+ scrapeId?: string,
+): Promise<RefreshDecision> {
+ const prior = refreshDecisionFor(scrapeId);
+ if (prior) return prior;
+ const decision = await decide(teamId);
+ if (scrapeId) {
+ const now = Date.now();
+ prune(now);
+ decisions.set(scrapeId, { decision, at: now });
+ }
+ return decision;
+}
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts
index b622b41a0..a6cfb78fd 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts
@@ -195,6 +195,77 @@ export const firePdfBlockPagesSchema = z.array(
export const firePdfBlocksSchema = firePdfBlockPagesSchema.optional();
+/**
+ * fire-pdf's provenance stamp: who produced a result and how complete it is.
+ * Stored verbatim with every cache entry so a later cache policy can judge
+ * the entry without reading its content (fire-pdf docs/cache-policy.md).
+ * `passthrough` keeps fields a newer fire-pdf adds.
+ */
+export const firePdfProvenanceSchema = z
+ .object({
+ generation: z.string(),
+ build_sha: z.string(),
+ built_at: z.string().nullable(),
+ produced_at: z.string(),
+ stages: z.array(z.string()).optional(),
+ // Page counts: the write rule reads them, so a malformed stamp must
+ // fail validation rather than pass as a healthy result. parseProvenance
+ // turns that failure into a refused cache write, never a failed scrape.
+ quality: z
+ .object({
+ total_pages: z.int().nonnegative(),
+ failed_pages: z.int().nonnegative(),
+ partial_pages: z.int().nonnegative(),
+ degraded_pages: z.int().nonnegative(),
+ ocr_pages: z.int().nonnegative(),
+ })
+ .passthrough()
+ .optional(),
+ // `passthrough` on each item too: a newer fire-pdf may add per-build
+ // fields, and the entry stores the stamp verbatim.
+ contributing_builds: z
+ .array(
+ z
+ .object({
+ generation: z.string(),
+ build_sha: z.string(),
+ built_at: z.string().nullable(),
+ })
+ .passthrough(),
+ )
+ .optional(),
+ })
+ .passthrough();
+
+export type FirePdfProvenance = z.infer<typeof firePdfProvenanceSchema>;
+
+/**
+ * The stamp is parsed apart from the document. The document is what the
+ * caller asked for; the stamp only decides what the cache may remember, so
+ * a stamp this build cannot read degrades to `malformed` (the result is
+ * served, not cached) instead of failing the response.
+ */
+export type ProvenanceParse =
+ | { status: "absent" }
+ | { status: "ok"; provenance: FirePdfProvenance }
+ | { status: "malformed"; issue: string };
+
+export function parseProvenance(raw: unknown): ProvenanceParse {
+ // Only a missing field is "no stamp" (a build from before the stamp
+ // existed). fire-pdf never sends an explicit null; one is unreadable.
+ if (raw === undefined) return { status: "absent" };
+ if (raw === null) return { status: "malformed", issue: "provenance: null" };
+ const parsed = firePdfProvenanceSchema.safeParse(raw);
+ if (parsed.success) return { status: "ok", provenance: parsed.data };
+ return {
+ status: "malformed",
+ issue: parsed.error.issues
+ .slice(0, 3)
+ .map(i => `${i.path.join(".") || "<root>"}: ${i.message}`)
+ .join("; "),
+ };
+}
+
export const resultResponseSchema = z.object({
schema_version: z
.union([z.literal(1), z.literal(2), z.literal(3)])
@@ -210,6 +281,9 @@ export const resultResponseSchema = z.object({
// is the only proof the fire-pdf worker build understood the option β€”
// older workers ignore unknown option keys and omit it.
page_markers: z.literal(true).optional(),
+ // Raw on purpose: parsed separately by parseProvenance so a stamp this
+ // build does not understand never fails the scrape.
+ provenance: z.unknown().optional(),
});
export type PollResponse = z.infer<typeof pollResponseSchema>;
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts
index cb3a6b83b..dd2ff5aee 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts
@@ -6,7 +6,11 @@ import type { PDFProcessorResult } from "./types";
import type { PDFMode } from "../../../../controllers/v2/types";
import { safeMarkdownToHtml } from "./markdownToHtml";
import { createPdfCacheKey } from "../../../../lib/gcs-pdf-cache";
-import { maybeSaveResult, tryGetCached } from "./fire-pdf/cache";
+import {
+ maybeSaveResult,
+ provenanceFromResponse,
+ tryGetCached,
+} from "./fire-pdf/cache";
import { firePdfBlocksSchema, firePdfPagesSchema } from "./fire-pdf/schema";
import {
buildFirePdfRequestMetadata,
@@ -153,9 +157,15 @@ export async function scrapePDFWithFirePDF(
schema: z.object({
markdown: z.string(),
failed_pages: z.array(z.number()).nullable(),
+ partial_pages: z.array(z.number()).nullable().optional(),
pages_processed: z.number().optional(),
pages: firePdfPagesSchema,
blocks: firePdfBlocksSchema,
+ // fire-pdf's stamp (generation, build, stages, quality). Taken raw
+ // and parsed separately (provenanceFromResponse) so a stamp this
+ // build cannot read never fails the scrape; a missing stamp means an
+ // older fire-pdf build.
+ provenance: z.unknown().optional(),
// Echo of an honored page_markers request. Markers are baked into
// `markdown` and their absence is not reliably detectable there (a
// single-page or fully-stitched document legitimately has none), so
@@ -184,6 +194,10 @@ export async function scrapePDFWithFirePDF(
);
}
const pages = resp.pages_processed ?? pagesProcessed;
+ const provenance = provenanceFromResponse(resp.provenance, logger, {
+ scrapeId: meta.id,
+ cacheKey: pdfSha256,
+ });
logger.info("FirePDF completed", {
scrapeId: meta.id,
@@ -191,8 +205,14 @@ export async function scrapePDFWithFirePDF(
durationMs,
markdownLength: resp.markdown.length,
failedPages: resp.failed_pages,
+ partialPages: resp.partial_pages ?? null,
pagesProcessed: pages,
perPageMs: pages ? Math.round(durationMs / pages) : undefined,
+ // The content-cache key and the producer, so a report can be turned
+ // into keys to purge and a result can be tied to a fire-pdf build.
+ cacheKey: pdfSha256,
+ generation: provenance?.generation ?? "unknown",
+ buildSha: provenance?.build_sha ?? "unknown",
});
const processorResult: PDFProcessorResult & { markdown: string } = {
@@ -213,6 +233,8 @@ export async function scrapePDFWithFirePDF(
includeBlocks,
pageMarkers,
result: processorResult,
+ provenance,
+ failedPages: resp.failed_pages,
});
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/runpodMU.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/runpodMU.ts
index 554a5460c..dafbb0eb3 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/runpodMU.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/runpodMU.ts
@@ -6,9 +6,13 @@ import { z } from "zod";
import path from "node:path";
import {
getPdfResultFromCache,
+ pdfCacheConfigured,
+ resolvePdfCacheKey,
savePdfResultToCache,
} from "../../../../lib/gcs-pdf-cache";
import type { PDFProcessorResult } from "./types";
+import { getPDFRefresh } from "../../../../controllers/v2/types";
+import { consumeRefresh } from "./fire-pdf/refresh-budget";
export async function scrapePDFWithRunPodMU(
meta: Meta,
@@ -21,20 +25,66 @@ export async function scrapePDFWithRunPodMU(
tempFilePath,
});
- if (!maxPages) {
- try {
- const cachedResult = await getPdfResultFromCache(base64Content);
- if (cachedResult) {
- meta.logger.info("Using cached RunPod MU result for PDF", {
+ // The cache key (sha256 of the payload, a full pass over the document) is
+ // computed at most once per request and reused for the read, the write
+ // and the log lines. This cache lives under the legacy `pdf-cache-v2/`
+ // prefix; the key is logged so a report can name the entry.
+ let cacheKeyMemo: string | undefined;
+ const cacheKeyOf = (): string =>
+ (cacheKeyMemo ??= resolvePdfCacheKey(base64Content));
+
+ // Without a bucket there is no cache: nothing is hashed and no refresh
+ // budget is spent. Zero-data-retention requests never read the shared
+ // cache, as on the fire-pdf path. `parsers: [{ type: "pdf", refresh: true }]`
+ // skips it too, under the same per-team budget: a MinerU-diverted request
+ // never reaches the fire-pdf path, so the budget has to be applied here as
+ // well, and the decision is taken once per request, so a fire-pdf β†’ MU
+ // fallback keeps the decision the first engine got instead of spending a
+ // second token. Over budget, or with the limiter unavailable, the entry is
+ // served.
+ const cacheOn = pdfCacheConfigured();
+ if (cacheOn && !maxPages && !meta.internalOptions.zeroDataRetention) {
+ const cacheKey = cacheKeyOf();
+ let bypass = false;
+ if (getPDFRefresh(meta.options?.parsers)) {
+ const decision = await consumeRefresh(
+ meta.internalOptions.teamId,
+ meta.id,
+ );
+ bypass = decision === "allowed";
+ if (bypass) {
+ meta.logger.info("RunPod MU cache bypassed by refresh", {
+ tempFilePath,
+ cacheKey,
+ cacheProvider: "runpod",
+ });
+ } else {
+ meta.logger.warn("RunPod MU cache refresh not applied", {
tempFilePath,
+ cacheKey,
+ cacheProvider: "runpod",
+ decision,
+ perMinute: config.FIRE_PDF_CACHE_REFRESH_PER_MINUTE,
});
- return cachedResult;
}
- } catch (error) {
- meta.logger.warn("Error checking PDF cache, proceeding with RunPod MU", {
- error,
- tempFilePath,
- });
+ }
+ if (!bypass) {
+ try {
+ const cachedResult = await getPdfResultFromCache({ key: cacheKey });
+ if (cachedResult) {
+ meta.logger.info("Using cached RunPod MU result for PDF", {
+ tempFilePath,
+ cacheKey,
+ cacheProvider: "runpod",
+ });
+ return cachedResult;
+ }
+ } catch (error) {
+ meta.logger.warn(
+ "Error checking PDF cache, proceeding with RunPod MU",
+ { error, tempFilePath, cacheKey, cacheProvider: "runpod" },
+ );
+ }
}
}
@@ -191,9 +241,9 @@ export async function scrapePDFWithRunPodMU(
html: await safeMarkdownToHtml(result.markdown, meta.logger, meta.id),
};
- if (!meta.internalOptions.zeroDataRetention) {
+ if (cacheOn && !meta.internalOptions.zeroDataRetention) {
try {
- await savePdfResultToCache(base64Content, processorResult);
+ await savePdfResultToCache({ key: cacheKeyOf() }, processorResult);
} catch (error) {
meta.logger.warn("Error saving PDF to cache", {
error,