Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(pdf): store fire-pdf provenance with cache entries, count cache events, refuse degraded writes, add refresh option (#4687)
Abimael Martell791bebb4c915
diff --git a/apps/api/src/__tests__/pdf-parser-options.test.ts b/apps/api/src/__tests__/pdf-parser-options.test.tsindex 118317407..dce6926de 100644--- a/apps/api/src/__tests__/pdf-parser-options.test.ts+++ b/apps/api/src/__tests__/pdf-parser-options.test.ts@@ -3,6 +3,7 @@ import {getPDFMode,getPDFPageMarkdown,getPDFPageMarkers,+ getPDFRefresh,scrapeOptions,shouldParseImages,shouldParsePDF,@@ -111,3 +112,20 @@ describe("image parser", () => {).toBe(false);});});++describe("pdf parser refresh option", () => {+ it("is off unless the pdf parser object sets it", () => {+ expect(getPDFRefresh(undefined)).toBe(false);+ expect(getPDFRefresh(["pdf"])).toBe(false);+ expect(getPDFRefresh([{ type: "pdf" }])).toBe(false);+ expect(getPDFRefresh([{ type: "pdf", refresh: false }])).toBe(false);+ expect(getPDFRefresh([{ type: "pdf", refresh: true }])).toBe(true);+ });++ it("parses through the strict schema", () => {+ expect(parsePdfParser({ type: "pdf", refresh: true })).toMatchObject({+ type: "pdf",+ refresh: true,+ });+ });+});diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex 8d3d9436a..a00b526b6 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -388,6 +388,10 @@ const configSchema = z.object({FIRE_PDF_PERCENT: z.coerce.number().min(0).max(100).default(10),FIRE_PDF_BASE_URL: z.string().optional(),FIRE_PDF_API_KEY: z.string().optional(),+ // `parsers: [{ type: "pdf", refresh: true }]` skips the content cache and+ // forces a fresh parse. Per team, per minute; beyond the budget the+ // request is served normally. 0 disables the option.+ FIRE_PDF_CACHE_REFRESH_PER_MINUTE: z.coerce.number().int().min(0).default(10),// Raster image OCR of image URLs and parse uploads through FirePDF (see// lib/image-ocr-gate.ts). Needs FIRE_PDF_BASE_URL.IMAGE_OCR_ENABLED: z.stringbool().default(false),diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 05cf16dc5..1ff4ef484 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -508,6 +508,11 @@ const pdfParserWithOptions = z* cross-page stitching β callers that need every physical page should* use `pages: true` instead. No new response field. */pageMarkers: z.boolean().optional(),+ /** Skip the cached conversion for this document and parse it again with+ * the current pipeline; the fresh result overwrites the cache entry for+ * everyone. Billed like a fresh parse. Use it when a cached result is+ * wrong or outdated. */+ refresh: z.boolean().optional(),// Experimental: route this request through the fire-pdf async pipeline// (POST /jobs + poll) instead of the sync POST /ocr endpoint. Falls back// to sync on any async-path failure, so user-visible behavior is unchanged@@ -639,6 +644,17 @@ export function getPDFPageMarkers(parsers?: Parsers): boolean {return false;}+/** `parsers: [{ type: "pdf", refresh: true }]`: bypass the content cache for this request. */+export function getPDFRefresh(parsers?: Parsers): boolean {+ if (!parsers) return false;+ for (const parser of parsers) {+ if (typeof parser === "object" && parser.type === "pdf") {+ return parser.refresh === true;+ }+ }+ return false;+}+export function getFirePdfAsync(parsers?: Parsers): boolean {if (!parsers) return false;for (const parser of parsers) {diff --git a/apps/api/src/lib/gcs-pdf-cache.ts b/apps/api/src/lib/gcs-pdf-cache.tsindex 4f2465af5..6fb0a9684 100644--- a/apps/api/src/lib/gcs-pdf-cache.ts+++ b/apps/api/src/lib/gcs-pdf-cache.ts@@ -1,5 +1,6 @@import { ApiError } from "@google-cloud/storage";import type { FirePdfPageBlocks } from "../scraper/scrapeURL/engines/pdf/types";+import type { FirePdfProvenance } from "../scraper/scrapeURL/engines/pdf/fire-pdf/schema";import { logger } from "./logger";import { config } from "../config";import crypto from "crypto";@@ -10,7 +11,7 @@ type PdfCacheProvider = "runpod" | "firepdf";// Cache shape β markdown/html are required; pagesProcessed is optional so// pre-existing entries (written before the field existed) round-trip cleanly// and the caller can fall back to its own page-count signal on a stale hit.-type CachedPdfResult = {+export type CachedPdfResult = {markdown: string;html: string;pagesProcessed?: number;@@ -19,6 +20,13 @@ type CachedPdfResult = {/** Typed layout blocks (fire-pdf wire shape); present only in* block-capable cache variants. */blocks?: FirePdfPageBlocks[];+ /** fire-pdf's stamp for the result this entry holds. Absent on entries+ * written before it existed, which a cache policy reads as "unknown". */+ provenance?: FirePdfProvenance;+ /** When this entry was written (ISO-8601). */+ cachedAt?: string;+ /** The variant this entry was written under ("base" for the bare key). */+ variant?: string;};const PROVIDER_PREFIXES: Record<PdfCacheProvider, string> = {@@ -37,10 +45,20 @@ export function createPdfCacheKey(pdfContent: string | Buffer): string {* distinct. */export type PdfCacheKeyInput = string | { key: string };-function resolvePdfCacheKey(input: PdfCacheKeyInput): string {+/** The cache key for an input: sha256 of the inline base64 payload, or the+ * caller's precomputed `raw-<sha256>` key. Logged on every cache event so a+ * team report can be turned into the keys to purge. */+export function resolvePdfCacheKey(input: PdfCacheKeyInput): string {return typeof input === "string" ? createPdfCacheKey(input) : input.key;}+/** Whether the content cache exists at all (self-hosted deployments may run+ * without a bucket). Callers check this before hashing a payload or spending+ * a refresh token for a cache that would neither read nor write. */+export function pdfCacheConfigured(): boolean {+ return !!config.GCS_BUCKET_NAME;+}+export async function savePdfResultToCache(pdfContent: PdfCacheKeyInput,result: CachedPdfResult,diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.tsindex 008e14084..fffb7bcd5 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts@@ -2,7 +2,10 @@// client calls these on the happy path; without the stub the first cache lookup// blows up trying to download from GCS using the credentials in .env.vi.mock("../../../../../lib/gcs-pdf-cache", () => ({+ pdfCacheConfigured: vi.fn(() => true),createPdfCacheKey: (s: string) => `sha-${s.length}`,+ resolvePdfCacheKey: (input: string | { key: string }) =>+ typeof input === "string" ? `sha-${input.length}` : input.key,getPdfResultFromCache: vi.fn(async () => null),savePdfResultToCache: vi.fn(async () => null),}));diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncDeadline.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncDeadline.test.tsindex be11af646..ceef41988 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncDeadline.test.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncDeadline.test.ts@@ -1,7 +1,10 @@// Stub the GCS cache so unit tests never reach real cloud storage β same// setup as firePDFAsync.test.ts, which covers the orchestration end to end.vi.mock("../../../../../lib/gcs-pdf-cache", () => ({+ pdfCacheConfigured: vi.fn(() => true),createPdfCacheKey: (s: string) => `sha-${s.length}`,+ resolvePdfCacheKey: (input: string | { key: string }) =>+ typeof input === "string" ? `sha-${input.length}` : input.key,getPdfResultFromCache: vi.fn(async () => null),savePdfResultToCache: vi.fn(async () => null),}));diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.tsindex e812bab65..c34da25c5 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts@@ -1,22 +1,39 @@import {getPdfResultFromCache,+ pdfCacheConfigured,savePdfResultToCache,} from "../../../../../lib/gcs-pdf-cache";import {cacheKeyShape,maybeSaveResult,+ provenanceFromResponse,tryGetCached,} from "../fire-pdf/cache";+import { consumeRefresh, refreshDecisionFor } from "../fire-pdf/refresh-budget";+import { firePdfProvenanceSchema } from "../fire-pdf/schema";+import {+ firePdfCacheEventsTotal,+ firePdfCacheRefusedWritesTotal,+} from "../fire-pdf/metrics";++vi.mock("../fire-pdf/refresh-budget", () => ({+ consumeRefresh: vi.fn(async () => "allowed"),+ refreshDecisionFor: vi.fn(() => undefined),+}));vi.mock("../../../../../lib/gcs-pdf-cache", () => ({+ pdfCacheConfigured: vi.fn(() => true),getPdfResultFromCache: vi.fn(),savePdfResultToCache: vi.fn(),+ resolvePdfCacheKey: vi.fn((input: string | { key: string }) =>+ typeof input === "string" ? `key-of-${input}` : input.key,+ ),}));const getCached = vi.mocked(getPdfResultFromCache);const saveCached = vi.mocked(savePdfResultToCache);-function makeMeta(zeroDataRetention = false) {+function makeMeta(zeroDataRetention = false, parsers?: unknown[]) {return {id: "page-cache-test",logger: {@@ -24,14 +41,32 @@ function makeMeta(zeroDataRetention = false) {warn: vi.fn(),},internalOptions: { zeroDataRetention },+ ...(parsers ? { options: { parsers } } : {}),} as any;}+const provenance = {+ generation: "2026-09-16.1",+ build_sha: "25c376ac15489dc6d5cab9af8728de41357ffc69",+ built_at: "2026-09-17T20:11:39Z",+ produced_at: "2026-09-17T20:12:00.000Z",+ stages: ["native_text", "layout", "ocr"],+ quality: {+ total_pages: 3,+ failed_pages: 0,+ partial_pages: 0,+ degraded_pages: 0,+ ocr_pages: 1,+ },+};+describe("FirePDF page-markdown cache capabilities", () => {beforeEach(() => {vi.clearAllMocks();getCached.mockResolvedValue(null);- saveCached.mockResolvedValue(null);+ // The real save returns the cache key on success and null when it could+ // not persist; a null default would read as "not written".+ saveCached.mockResolvedValue("saved-key");});it("uses versioned page-capable variants", () => {@@ -626,7 +661,9 @@ describe("FirePDF cache and empty raster-image results", () => {beforeEach(() => {vi.clearAllMocks();getCached.mockResolvedValue(null);- saveCached.mockResolvedValue(null);+ // The real save returns the cache key on success and null when it could+ // not persist; a null default would read as "not written".+ saveCached.mockResolvedValue("saved-key");});it("treats a cached empty result for an image as a miss", async () => {@@ -720,3 +757,594 @@ describe("FirePDF cache and empty raster-image results", () => {);});});++describe("FirePDF cache provenance and write rules", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ getCached.mockResolvedValue(null);+ // The real save returns the cache key on success and null when it could+ // not persist; a null default would read as "not written".+ saveCached.mockResolvedValue("saved-key");+ });++ it("stores the stamp, the write time and the variant with the entry", async () => {+ await maybeSaveResult({+ meta: makeMeta(),+ base64Content: "BASE64",+ mode: "auto",+ maxPages: undefined,+ includePageMarkdown: false,+ includeBlocks: false,+ result: { markdown: "whole", html: "<p>whole</p>", pagesProcessed: 3 },+ provenance,+ failedPages: null,+ });++ expect(saveCached).toHaveBeenCalledTimes(1);+ const entry = saveCached.mock.calls[0][1];+ expect(entry).toMatchObject({+ markdown: "whole",+ provenance,+ variant: "base",+ });+ expect(typeof entry.cachedAt).toBe("string");+ expect(Number.isNaN(Date.parse(entry.cachedAt!))).toBe(false);+ });++ it("stamps the compact legacy entry it back-fills beside a sidecar", async () => {+ await maybeSaveResult({+ meta: makeMeta(),+ base64Content: "BASE64",+ mode: "auto",+ maxPages: undefined,+ includePageMarkdown: true,+ includeBlocks: false,+ result: {+ markdown: "whole",+ html: "<p>whole</p>",+ pagesProcessed: 2,+ pageMarkdown: [{ page: 1, markdown: "one" }],+ },+ provenance,+ });++ expect(saveCached).toHaveBeenCalledTimes(2);+ expect(saveCached.mock.calls[0][1]).toMatchObject({+ provenance,+ variant: "page-markdown-v1",+ });+ expect(saveCached.mock.calls[1][1]).toMatchObject({+ provenance,+ variant: "base",+ });+ });++ it("writes entries without a stamp when fire-pdf sent none", async () => {+ await maybeSaveResult({+ meta: makeMeta(),+ base64Content: "BASE64",+ mode: "auto",+ maxPages: undefined,+ includePageMarkdown: false,+ includeBlocks: false,+ result: { markdown: "whole", html: "<p>whole</p>" },+ });+ expect(saveCached).toHaveBeenCalledTimes(1);+ expect(saveCached.mock.calls[0][1]).not.toHaveProperty("provenance");+ expect(saveCached.mock.calls[0][1]).toMatchObject({ variant: "base" });+ });++ it("does not cache a result with failed pages, and says why", async () => {+ const meta = makeMeta();+ await maybeSaveResult({+ meta,+ base64Content: "BASE64",+ mode: "auto",+ maxPages: undefined,+ includePageMarkdown: false,+ includeBlocks: false,+ result: { markdown: "most of it", html: "<p>most of it</p>" },+ provenance,+ failedPages: [4],+ });+ expect(saveCached).not.toHaveBeenCalled();+ expect(meta.logger.info).toHaveBeenCalledWith(+ "FirePDF result not cached",+ expect.objectContaining({+ reason: "failed_pages",+ cacheKey: "key-of-BASE64",+ }),+ );+ });++ it("does not cache a result whose pages lost layout (degraded), but does cache partial pages", async () => {+ await maybeSaveResult({+ meta: makeMeta(),+ base64Content: "BASE64",+ mode: "auto",+ maxPages: undefined,+ includePageMarkdown: false,+ includeBlocks: false,+ result: { markdown: "text only", html: "<p>text only</p>" },+ provenance: {+ ...provenance,+ quality: { ...provenance.quality, degraded_pages: 2 },+ },+ failedPages: null,+ });+ expect(saveCached).not.toHaveBeenCalled();++ await maybeSaveResult({+ meta: makeMeta(),+ base64Content: "BASE64",+ mode: "auto",+ maxPages: undefined,+ includePageMarkdown: false,+ includeBlocks: false,+ result: { markdown: "nearly all", html: "<p>nearly all</p>" },+ provenance: {+ ...provenance,+ quality: { ...provenance.quality, partial_pages: 1 },+ },+ failedPages: null,+ });+ expect(saveCached).toHaveBeenCalledTimes(1);+ expect(saveCached.mock.calls[0][1]).toMatchObject({+ provenance: expect.objectContaining({+ quality: expect.objectContaining({ partial_pages: 1 }),+ }),+ });+ });++ it("serves a stamped entry without leaking the bookkeeping fields, and logs the key and generation", async () => {+ getCached.mockResolvedValueOnce({+ markdown: "whole",+ html: "<p>whole</p>",+ pagesProcessed: 3,+ provenance,+ cachedAt: "2026-09-17T20:12:01.000Z",+ variant: "base",+ });+ const meta = makeMeta();+ const result = await tryGetCached(+ meta,+ "BASE64",+ "auto",+ undefined,+ undefined,+ false,+ false,+ );+ expect(result).toEqual({+ markdown: "whole",+ html: "<p>whole</p>",+ pagesProcessed: 3,+ });+ expect(meta.logger.info).toHaveBeenCalledWith(+ "Using cached FirePDF result",+ expect.objectContaining({+ cacheKey: "key-of-BASE64",+ generation: "2026-09-16.1",+ buildSha: provenance.build_sha,+ cachedAt: "2026-09-17T20:12:01.000Z",+ }),+ );+ });++ it("reads an entry written before the stamp existed as unknown provenance", async () => {+ getCached.mockResolvedValueOnce({ markdown: "old", html: "<p>old</p>" });+ const meta = makeMeta();+ await tryGetCached(meta, "BASE64", "auto", undefined, 1, false, false);+ expect(meta.logger.info).toHaveBeenCalledWith(+ "Using cached FirePDF result",+ expect.objectContaining({+ generation: "unknown",+ buildSha: "unknown",+ cachedAt: null,+ }),+ );+ });++ it("bypasses the read, but not the write, when the pdf parser asks for a refresh", async () => {+ getCached.mockResolvedValue({ markdown: "stale", html: "<p>stale</p>" });+ const meta = makeMeta(false, [{ type: "pdf", refresh: true }]);+ const result = await tryGetCached(+ meta,+ "BASE64",+ "auto",+ undefined,+ 1,+ false,+ false,+ );+ expect(result).toBeNull();+ expect(getCached).not.toHaveBeenCalled();+ expect(meta.logger.info).toHaveBeenCalledWith(+ "FirePDF cache bypassed by refresh",+ expect.objectContaining({ cacheKey: "key-of-BASE64" }),+ );++ await maybeSaveResult({+ meta,+ base64Content: "BASE64",+ mode: "auto",+ maxPages: undefined,+ includePageMarkdown: false,+ includeBlocks: false,+ result: { markdown: "fresh", html: "<p>fresh</p>" },+ provenance,+ });+ expect(saveCached).toHaveBeenCalledTimes(1);+ });++ it("serves the cache normally when the team's refresh budget is spent, and says so", async () => {+ vi.mocked(consumeRefresh).mockResolvedValueOnce("limited");+ getCached.mockResolvedValue({ markdown: "cached", html: "<p>cached</p>" });+ const meta = makeMeta(false, [{ type: "pdf", refresh: true }]);+ const result = await tryGetCached(+ meta,+ "BASE64",+ "auto",+ undefined,+ 1,+ false,+ false,+ );+ expect(result?.markdown).toBe("cached");+ expect(getCached).toHaveBeenCalled();+ expect(meta.logger.warn).toHaveBeenCalledWith(+ "FirePDF cache refresh not applied",+ expect.objectContaining({+ decision: "limited",+ cacheKey: "key-of-BASE64",+ }),+ );+ });+ it("does not report a write the cache layer could not persist", async () => {+ saveCached.mockResolvedValueOnce(null);+ const meta = makeMeta();+ await maybeSaveResult({+ meta,+ base64Content: "BASE64",+ mode: "auto",+ maxPages: undefined,+ includePageMarkdown: false,+ includeBlocks: false,+ result: { markdown: "whole", html: "<p>whole</p>" },+ provenance,+ });+ expect(saveCached).toHaveBeenCalledTimes(1);+ expect(meta.logger.warn).toHaveBeenCalledWith(+ "FirePDF result not persisted to cache",+ expect.objectContaining({ cacheKey: "key-of-BASE64" }),+ );+ expect(meta.logger.info).not.toHaveBeenCalledWith(+ "Saved FirePDF result to cache",+ expect.anything(),+ );+ });++ it("refuses a result whose stamp reports failed pages when the list is absent", async () => {+ const meta = makeMeta();+ await maybeSaveResult({+ meta,+ base64Content: "BASE64",+ mode: "auto",+ maxPages: undefined,+ includePageMarkdown: false,+ includeBlocks: false,+ result: { markdown: "whole", html: "<p>whole</p>" },+ provenance: {+ ...provenance,+ quality: { ...provenance.quality, failed_pages: 2 },+ },+ failedPages: undefined,+ });+ expect(saveCached).not.toHaveBeenCalled();+ expect(meta.logger.info).toHaveBeenCalledWith(+ "FirePDF result not cached",+ expect.objectContaining({ reason: "failed_pages", failedPages: 2 }),+ );+ });++ it("refuses a result whose stamp could not be read", async () => {+ const meta = makeMeta();+ await maybeSaveResult({+ meta,+ base64Content: "BASE64",+ mode: "auto",+ maxPages: undefined,+ includePageMarkdown: false,+ includeBlocks: false,+ result: { markdown: "whole", html: "<p>whole</p>" },+ provenance: null,+ failedPages: [],+ });+ expect(saveCached).not.toHaveBeenCalled();+ expect(meta.logger.info).toHaveBeenCalledWith(+ "FirePDF result not cached",+ expect.objectContaining({ reason: "malformed_provenance" }),+ );+ });++ it("reads the stamp apart from the document and never throws", () => {+ const logger = { warn: vi.fn(), info: vi.fn() } as any;+ const context = { scrapeId: "s1", cacheKey: "key-of-BASE64" };+ expect(provenanceFromResponse(provenance, logger, context)).toEqual(+ provenance,+ );+ expect(provenanceFromResponse(undefined, logger, context)).toBeUndefined();+ expect(logger.warn).not.toHaveBeenCalled();+ expect(+ provenanceFromResponse(+ {+ ...provenance,+ quality: { ...provenance.quality, failed_pages: "2" },+ },+ logger,+ context,+ ),+ ).toBeNull();+ expect(provenanceFromResponse("garbage", logger, context)).toBeNull();+ expect(logger.warn).toHaveBeenCalledWith(+ "FirePDF provenance stamp not understood",+ expect.objectContaining({+ cacheKey: "key-of-BASE64",+ issue: expect.stringContaining("quality.failed_pages"),+ }),+ );+ });++ it("rewrites the base alias on an allowed refresh even when one exists", async () => {+ vi.mocked(refreshDecisionFor).mockReturnValueOnce("allowed");+ const meta = makeMeta(false, [{ type: "pdf", refresh: true }]);+ await maybeSaveResult({+ meta,+ base64Content: "BASE64",+ mode: "auto",+ maxPages: undefined,+ includePageMarkdown: true,+ includeBlocks: false,+ result: {+ markdown: "fresh",+ html: "<p>fresh</p>",+ pageMarkdown: [{ page: 1, markdown: "fresh" }],+ },+ provenance,+ });+ // No read of the existing alias: it is overwritten regardless.+ expect(getCached).not.toHaveBeenCalled();+ expect(saveCached).toHaveBeenCalledTimes(2);+ expect(saveCached).toHaveBeenLastCalledWith(+ "BASE64",+ expect.objectContaining({ markdown: "fresh", variant: "base" }),+ "firepdf",+ undefined,+ );+ });++ it("refuses a stamped result that carries no quality counts", async () => {+ const meta = makeMeta();+ const { quality: _quality, ...withoutQuality } = provenance;+ await maybeSaveResult({+ meta,+ base64Content: "BASE64",+ mode: "auto",+ maxPages: undefined,+ includePageMarkdown: false,+ includeBlocks: false,+ result: { markdown: "whole", html: "<p>whole</p>" },+ provenance: withoutQuality,+ failedPages: [],+ });+ expect(saveCached).not.toHaveBeenCalled();+ expect(meta.logger.info).toHaveBeenCalledWith(+ "FirePDF result not cached",+ expect.objectContaining({ reason: "missing_quality" }),+ );+ });++ it("treats an explicit null stamp as unreadable, not as absent", () => {+ const logger = { warn: vi.fn(), info: vi.fn() } as any;+ expect(+ provenanceFromResponse(null, logger, {+ scrapeId: "s1",+ cacheKey: "key-of-BASE64",+ }),+ ).toBeNull();+ expect(logger.warn).toHaveBeenCalledWith(+ "FirePDF provenance stamp not understood",+ expect.objectContaining({ issue: expect.stringContaining("null") }),+ );+ });++ it("keeps fields a newer fire-pdf adds to a contributing build", () => {+ const parsed = firePdfProvenanceSchema.parse({+ ...provenance,+ contributing_builds: [+ {+ generation: "2026-09-16.1",+ build_sha: "abc",+ built_at: null,+ lane: "heavy",+ },+ ],+ });+ expect(parsed.contributing_builds?.[0]).toMatchObject({ lane: "heavy" });+ });++ it("logs an alias write the cache layer could not persist", async () => {+ saveCached.mockResolvedValueOnce("saved-key").mockResolvedValueOnce(null);+ const meta = makeMeta();+ await maybeSaveResult({+ meta,+ base64Content: "BASE64",+ mode: "auto",+ maxPages: undefined,+ includePageMarkdown: true,+ includeBlocks: false,+ result: {+ markdown: "whole",+ html: "<p>whole</p>",+ pageMarkdown: [{ page: 1, markdown: "whole" }],+ },+ provenance,+ });+ expect(saveCached).toHaveBeenCalledTimes(2);+ expect(meta.logger.info).toHaveBeenCalledWith(+ "Saved FirePDF result to cache",+ expect.objectContaining({+ cacheVariant: "page-markdown-v1",+ alias: false,+ cacheKey: "key-of-BASE64",+ }),+ );+ expect(meta.logger.warn).toHaveBeenCalledWith(+ "FirePDF result not persisted to cache",+ expect.objectContaining({+ cacheVariant: "base",+ alias: true,+ cacheKey: "key-of-BASE64",+ }),+ );+ });++ it("rejects a stamp whose page counts are negative or fractional", () => {+ expect(firePdfProvenanceSchema.safeParse(provenance).success).toBe(true);+ for (const bad of [+ { ...provenance.quality, degraded_pages: -1 },+ { ...provenance.quality, failed_pages: 0.5 },+ ]) {+ expect(+ firePdfProvenanceSchema.safeParse({ ...provenance, quality: bad })+ .success,+ ).toBe(false);+ }+ });+});++// The counters are the headline deliverable: spy on the real prom-client+// counters so a change to an event name, a label or a refusal reason fails+// here instead of silently on a dashboard.+describe("FirePDF cache counters", () => {+ const events = vi.spyOn(firePdfCacheEventsTotal, "inc");+ const refused = vi.spyOn(firePdfCacheRefusedWritesTotal, "inc");++ beforeEach(() => {+ vi.clearAllMocks();+ getCached.mockResolvedValue(null);+ saveCached.mockResolvedValue("saved-key");+ vi.mocked(consumeRefresh).mockResolvedValue("allowed");+ });++ const save = (+ overrides: Partial<Parameters<typeof maybeSaveResult>[0]> = {},+ ) =>+ maybeSaveResult({+ meta: makeMeta(),+ base64Content: "BASE64",+ mode: "auto",+ maxPages: undefined,+ includePageMarkdown: false,+ includeBlocks: false,+ result: { markdown: "whole", html: "<p>whole</p>" },+ provenance,+ failedPages: [],+ ...overrides,+ });++ const read = (meta = makeMeta()) =>+ tryGetCached(meta, "BASE64", "auto", undefined, 1, false, false);++ it("counts a persisted write, and a write that was not persisted", async () => {+ await save();+ expect(events).toHaveBeenCalledWith({ event: "write", variant: "base" });+ saveCached.mockResolvedValueOnce(null);+ await save();+ expect(events).toHaveBeenCalledWith({+ event: "write_failed",+ variant: "base",+ });+ });++ it("counts the alias write under its own variant", async () => {+ saveCached.mockResolvedValueOnce("saved-key").mockResolvedValueOnce(null);+ await save({+ includePageMarkdown: true,+ result: {+ markdown: "whole",+ html: "<p>whole</p>",+ pageMarkdown: [{ page: 1, markdown: "whole" }],+ },+ });+ expect(events).toHaveBeenCalledWith({+ event: "write",+ variant: "page-markdown-v1",+ });+ expect(events).toHaveBeenCalledWith({+ event: "write_failed",+ variant: "base",+ });+ });++ it("counts every refusal reason", async () => {+ await save({ failedPages: [2] });+ await save({+ provenance: {+ ...provenance,+ quality: { ...provenance.quality, degraded_pages: 1 },+ },+ });+ const { quality: _quality, ...withoutQuality } = provenance;+ await save({ provenance: withoutQuality });+ await save({ provenance: null });+ for (const reason of [+ "failed_pages",+ "degraded_pages",+ "missing_quality",+ "malformed_provenance",+ ]) {+ expect(refused).toHaveBeenCalledWith({ reason });+ }+ expect(+ events.mock.calls.filter(+ ([labels]) => (labels as { event?: string }).event === "refused_write",+ ),+ ).toHaveLength(4);+ expect(saveCached).not.toHaveBeenCalled();+ });++ it("does nothing, and counts nothing, when no cache is configured", async () => {+ vi.mocked(pdfCacheConfigured).mockReturnValue(false);+ try {+ const meta = makeMeta(false, [{ type: "pdf", refresh: true }]);+ await expect(read(meta)).resolves.toBeNull();+ await save({ meta });+ expect(getCached).not.toHaveBeenCalled();+ expect(saveCached).not.toHaveBeenCalled();+ expect(consumeRefresh).not.toHaveBeenCalled();+ expect(events).not.toHaveBeenCalled();+ } finally {+ vi.mocked(pdfCacheConfigured).mockReturnValue(true);+ }+ });++ it("counts hits, misses and both refresh outcomes", async () => {+ await read();+ expect(events).toHaveBeenCalledWith({ event: "miss", variant: "base" });+ getCached.mockResolvedValueOnce({ markdown: "cached", html: "<p>c</p>" });+ await read();+ expect(events).toHaveBeenCalledWith({ event: "hit", variant: "base" });+ await read(makeMeta(false, [{ type: "pdf", refresh: true }]));+ expect(events).toHaveBeenCalledWith({+ event: "bypass_refresh",+ variant: "base",+ });+ vi.mocked(consumeRefresh).mockResolvedValueOnce("limited");+ await read(makeMeta(false, [{ type: "pdf", refresh: true }]));+ expect(events).toHaveBeenCalledWith({+ event: "bypass_refresh_denied",+ variant: "base",+ });+ });+});diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/refreshBudget.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/refreshBudget.test.tsnew file mode 100644index 000000000..c0dff780e--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/refreshBudget.test.ts@@ -0,0 +1,84 @@+import { beforeEach, describe, expect, it, vi } from "vitest";++// The limiter classes are hoisted so every re-import of the module under+// test (vi.resetModules below) sees the same class identities; the+// `instanceof RateLimiterRes` check in refresh-budget depends on that.+const { consume, RateLimiterRedis, RateLimiterRes } = vi.hoisted(() => {+ const consume = vi.fn();+ class RateLimiterRes {}+ class RateLimiterRedis {+ consume = consume;+ }+ return { consume, RateLimiterRedis, RateLimiterRes };+});++vi.mock("rate-limiter-flexible", () => ({ RateLimiterRedis, RateLimiterRes }));++vi.mock("../../../../../services/rate-limiter", () => ({+ redisRateLimitClient: {},+}));++type Budget = typeof import("../fire-pdf/refresh-budget");++describe("refresh budget", () => {+ let budget: Budget;++ beforeEach(async () => {+ consume.mockReset();+ consume.mockResolvedValue(undefined);+ // A fresh module per test: the remembered decisions are module state,+ // so no test can read what an earlier one left behind.+ vi.resetModules();+ budget = await import("../fire-pdf/refresh-budget.js");+ });++ it("spends one token per request and remembers the decision", async () => {+ await expect(budget.consumeRefresh("team-a", "scrape-1")).resolves.toBe(+ "allowed",+ );+ await expect(budget.consumeRefresh("team-a", "scrape-1")).resolves.toBe(+ "allowed",+ );+ expect(consume).toHaveBeenCalledTimes(1);+ expect(consume).toHaveBeenCalledWith("team-a", 1);+ expect(budget.refreshDecisionFor("scrape-1")).toBe("allowed");+ });++ it("starts every test without remembered decisions", () => {+ expect(budget.refreshDecisionFor("scrape-1")).toBeUndefined();+ });++ it("decides each request separately", async () => {+ await budget.consumeRefresh("team-a", "scrape-1");+ await budget.consumeRefresh("team-a", "scrape-2");+ expect(consume).toHaveBeenCalledTimes(2);+ });++ it("keeps a denied decision for the same request too", async () => {+ consume.mockRejectedValueOnce(new RateLimiterRes());+ await expect(budget.consumeRefresh("team-b", "scrape-1")).resolves.toBe(+ "limited",+ );+ // A later engine on the same request sees the same answer without+ // asking the limiter again.+ await expect(budget.consumeRefresh("team-b", "scrape-1")).resolves.toBe(+ "limited",+ );+ expect(consume).toHaveBeenCalledTimes(1);+ });++ it("reports the limiter store being unreachable as unavailable", async () => {+ consume.mockRejectedValueOnce(new Error("ECONNREFUSED"));+ await expect(budget.consumeRefresh("team-c", "scrape-1")).resolves.toBe(+ "unavailable",+ );+ });++ it("decides afresh without a scrape id and knows nothing about unknown ids", async () => {+ await budget.consumeRefresh("team-d");+ await budget.consumeRefresh("team-d");+ expect(consume).toHaveBeenCalledTimes(2);+ expect(budget.refreshDecisionFor("never-seen")).toBeUndefined();+ expect(budget.refreshDecisionFor(undefined)).toBeUndefined();+ });+});diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/runpodMUCache.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/runpodMUCache.test.tsnew file mode 100644index 000000000..e8499d8cc--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/runpodMUCache.test.ts@@ -0,0 +1,130 @@+import { beforeEach, describe, expect, it, vi } from "vitest";+import { scrapePDFWithRunPodMU } from "../runpodMU";+import {+ getPdfResultFromCache,+ pdfCacheConfigured,+ resolvePdfCacheKey,+} from "../../../../../lib/gcs-pdf-cache";+import { consumeRefresh } from "../fire-pdf/refresh-budget";++vi.mock("../../../../../lib/gcs-pdf-cache", () => ({+ pdfCacheConfigured: vi.fn(() => true),+ getPdfResultFromCache: vi.fn(),+ savePdfResultToCache: vi.fn(),+ resolvePdfCacheKey: vi.fn(() => "key-of-BASE64"),+}));++vi.mock("../fire-pdf/refresh-budget", () => ({+ consumeRefresh: vi.fn(async () => "allowed"),+}));++const getCached = vi.mocked(getPdfResultFromCache);+const budget = vi.mocked(consumeRefresh);++const cached = { markdown: "cached", html: "<p>cached</p>" };++function makeMeta(parsers?: unknown[], zeroDataRetention = false) {+ return {+ id: "mu-cache-test",+ logger: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), child: vi.fn() },+ internalOptions: { teamId: "team-1", zeroDataRetention },+ // Already aborted: the engine stops right after the cache step, before+ // any network call, which is all these tests exercise.+ abort: AbortSignal.abort(),+ ...(parsers ? { options: { parsers } } : {}),+ } as any;+}++describe("RunPod MU cache read and refresh", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ getCached.mockResolvedValue(cached as any);+ budget.mockResolvedValue("allowed");+ });++ it("serves the cached result and leaves the refresh budget alone", async () => {+ const meta = makeMeta();+ await expect(+ scrapePDFWithRunPodMU(meta, "/tmp/doc.pdf", "BASE64"),+ ).resolves.toEqual(cached);+ // The key is hashed once and handed to the cache layer as is.+ expect(getCached).toHaveBeenCalledWith({ key: "key-of-BASE64" });+ expect(budget).not.toHaveBeenCalled();+ });++ it("skips the cache read on refresh within the team budget", async () => {+ const meta = makeMeta([{ type: "pdf", refresh: true }]);+ await expect(+ scrapePDFWithRunPodMU(meta, "/tmp/doc.pdf", "BASE64"),+ ).rejects.toThrow();+ expect(budget).toHaveBeenCalledWith("team-1", "mu-cache-test");+ expect(getCached).not.toHaveBeenCalled();+ expect(meta.logger.info).toHaveBeenCalledWith(+ "RunPod MU cache bypassed by refresh",+ expect.objectContaining({+ tempFilePath: "/tmp/doc.pdf",+ cacheKey: "key-of-BASE64",+ }),+ );+ });++ it("serves the cached result when the refresh budget is spent", async () => {+ budget.mockResolvedValueOnce("limited");+ const meta = makeMeta([{ type: "pdf", refresh: true }]);+ await expect(+ scrapePDFWithRunPodMU(meta, "/tmp/doc.pdf", "BASE64"),+ ).resolves.toEqual(cached);+ expect(getCached).toHaveBeenCalledWith({ key: "key-of-BASE64" });+ expect(meta.logger.warn).toHaveBeenCalledWith(+ "RunPod MU cache refresh not applied",+ expect.objectContaining({+ decision: "limited",+ cacheKey: "key-of-BASE64",+ }),+ );+ });++ it("names the entry when the cache lookup itself fails", async () => {+ getCached.mockRejectedValueOnce(new Error("gcs down"));+ const meta = makeMeta();+ await expect(+ scrapePDFWithRunPodMU(meta, "/tmp/doc.pdf", "BASE64"),+ ).rejects.toThrow();+ expect(meta.logger.warn).toHaveBeenCalledWith(+ "Error checking PDF cache, proceeding with RunPod MU",+ expect.objectContaining({+ cacheKey: "key-of-BASE64",+ cacheProvider: "runpod",+ }),+ );+ });++ it("neither hashes nor spends the budget when no cache is configured", async () => {+ vi.mocked(pdfCacheConfigured).mockReturnValueOnce(false);+ const meta = makeMeta([{ type: "pdf", refresh: true }]);+ await expect(+ scrapePDFWithRunPodMU(meta, "/tmp/doc.pdf", "BASE64"),+ ).rejects.toThrow();+ expect(resolvePdfCacheKey).not.toHaveBeenCalled();+ expect(getCached).not.toHaveBeenCalled();+ expect(budget).not.toHaveBeenCalled();+ });++ it("never reads the cache for a zero-data-retention request", async () => {+ const meta = makeMeta([{ type: "pdf", refresh: true }], true);+ await expect(+ scrapePDFWithRunPodMU(meta, "/tmp/doc.pdf", "BASE64"),+ ).rejects.toThrow();+ expect(getCached).not.toHaveBeenCalled();+ expect(budget).not.toHaveBeenCalled();+ });++ it("does not spend the budget when maxPages already skips the cache", async () => {+ const meta = makeMeta([{ type: "pdf", refresh: true }]);+ await expect(+ scrapePDFWithRunPodMU(meta, "/tmp/doc.pdf", "BASE64", 5),+ ).rejects.toThrow();+ expect(budget).not.toHaveBeenCalled();+ expect(getCached).not.toHaveBeenCalled();+ });+});diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.tsindex 10c802085..3d9240adc 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts@@ -7,7 +7,8 @@ import type { PDFProcessorResult } from "../types";import { safeMarkdownToHtml } from "../markdownToHtml";import { scrapePDFWithFirePDF } from "../firePDF";import { cancelJob } from "./cancel";-import { tryGetCached, maybeSaveResult } from "./cache";+import { maybeSaveResult, provenanceFromResponse, tryGetCached } from "./cache";+import { resolvePdfCacheKey } from "../../../../../lib/gcs-pdf-cache";import {firePdfAsyncAbandonedTotal,firePdfAsyncTotalDurationSeconds,@@ -403,6 +404,12 @@ export async function scrapePDFWithFirePDFAsync(const durationMs = now() - overallStartedAt;firePdfAsyncTotalDurationSeconds.observe(durationMs / 1000);+ const cacheKey = resolvePdfCacheKey(cacheInput);+ const provenance = provenanceFromResponse(fetched.provenance, meta.logger, {+ scrapeId: meta.id,+ cacheKey,+ });+meta.logger.info("FirePDF async completed", {scrapeId: meta.id,durationMs,@@ -413,6 +420,11 @@ export async function scrapePDFWithFirePDFAsync(failedPages: fetched.failed_pages,partialPages: fetched.partial_pages,pollCount: polled.pollCount,+ // The content-cache key and the producer, so a report can be turned+ // into keys to purge and a result can be tied to a fire-pdf build.+ cacheKey,+ generation: provenance?.generation ?? "unknown",+ buildSha: provenance?.build_sha ?? "unknown",});const processorResult: PDFProcessorResult & { markdown: string } = {@@ -432,6 +444,8 @@ export async function scrapePDFWithFirePDFAsync(includeBlocks,pageMarkers,result: processorResult,+ provenance,+ failedPages: fetched.failed_pages,});return processorResult;diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.tsindex 270f46aea..968094bc2 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts@@ -1,13 +1,30 @@import type { Meta } from "../../..";-import type { PDFMode } from "../../../../../controllers/v2/types";+import { config } from "../../../../../config";+import {+ getPDFRefresh,+ type PDFMode,+} from "../../../../../controllers/v2/types";import type { PDFProcessorResult } from "../types";import {+ type CachedPdfResult,getPdfResultFromCache,+ pdfCacheConfigured,+ resolvePdfCacheKey,savePdfResultToCache,type PdfCacheKeyInput,} from "../../../../../lib/gcs-pdf-cache";import { sniffImageContentTypeFromBase64 } from "../../../../../lib/image-formats";-import { firePdfBlockPagesSchema } from "./schema";+import {+ type CacheRefusedReason,+ firePdfCacheEventsTotal,+ firePdfCacheRefusedWritesTotal,+} from "./metrics";+import { consumeRefresh, refreshDecisionFor } from "./refresh-budget";+import {+ firePdfBlockPagesSchema,+ parseProvenance,+ type FirePdfProvenance,+} from "./schema";// Raster images ride the same cache as PDFs (the image engine posts their// bytes to the same FirePDF endpoint), but an EMPTY result for an image is@@ -173,7 +190,9 @@ export async function tryGetCached(pageMarkers = false,): Promise<PDFProcessorResult | null> {if (meta.internalOptions.zeroDataRetention) return null;- const { cacheable, lookupVariants } = cacheKeyShape(+ // No bucket, no cache: nothing to hash, no refresh budget to spend.+ if (!pdfCacheConfigured()) return null;+ const { cacheable, lookupVariants, ownVariant } = cacheKeyShape(mode,maxPages,includePageMarkdown,@@ -181,6 +200,39 @@ export async function tryGetCached(pageMarkers,);if (!cacheable) return null;+ const cacheKey = resolvePdfCacheKey(base64Content);++ // `parsers: [{ type: "pdf", refresh: true }]`: the caller wants this+ // document parsed again with the current pipeline. Within the team's+ // budget the read is skipped; the fresh result is still written, so the+ // entry is corrected for everyone. Over budget (or with the limiter+ // unavailable) the request is served normally and the decision logged.+ if (getPDFRefresh(meta.options?.parsers)) {+ const decision = await consumeRefresh(meta.internalOptions.teamId, meta.id);+ if (decision === "allowed") {+ firePdfCacheEventsTotal.inc({+ event: "bypass_refresh",+ variant: ownVariant ?? "base",+ });+ meta.logger.info("FirePDF cache bypassed by refresh", {+ scrapeId: meta.id,+ requestedMode: mode,+ cacheKey,+ });+ return null;+ }+ firePdfCacheEventsTotal.inc({+ event: "bypass_refresh_denied",+ variant: ownVariant ?? "base",+ });+ meta.logger.warn("FirePDF cache refresh not applied", {+ scrapeId: meta.id,+ requestedMode: mode,+ cacheKey,+ decision,+ perMinute: config.FIRE_PDF_CACHE_REFRESH_PER_MINUTE,+ });+ }for (const variant of lookupVariants) {try {@@ -216,14 +268,33 @@ export async function tryGetCached();continue;}+ firePdfCacheEventsTotal.inc({+ event: "hit",+ variant: variant ?? "base",+ });meta.logger.info("Using cached FirePDF result", {scrapeId: meta.id,requestedMode: mode,cacheVariant: variant ?? "base",+ cacheKey,+ // Provenance of the entry, for cache-policy work and team reports.+ // Entries written before the stamp existed read as "unknown".+ generation: cached.provenance?.generation ?? "unknown",+ buildSha: cached.provenance?.build_sha ?? "unknown",+ cachedAt: cached.cachedAt ?? null,});// Strip payloads the request didn't ask for so a richer sidecar- // serves a poorer request without leaking extra capabilities.- const { pageMarkdown, blocks, ...compactCached } = cached;+ // serves a poorer request without leaking extra capabilities, and+ // the entry-only fields (provenance, cachedAt, variant) that are+ // cache bookkeeping rather than document content.+ const {+ pageMarkdown,+ blocks,+ provenance: _provenance,+ cachedAt: _cachedAt,+ variant: _variant,+ ...compactCached+ } = cached;return {...compactCached,...(includePageMarkdown ? { pageMarkdown } : {}),@@ -235,9 +306,60 @@ export async function tryGetCached(meta.logger.warn("Error checking FirePDF cache, proceeding", {error,cacheVariant: variant ?? "base",+ cacheKey,});}}+ firePdfCacheEventsTotal.inc({ event: "miss", variant: ownVariant ?? "base" });+ return null;+}++/**+ * fire-pdf's stamp off a response. `undefined`: none was sent (a build+ * from before the stamp existed); the result is cached unstamped. `null`:+ * a stamp was sent but this build cannot read it; the result is served but+ * not cached, since an entry whose stamp cannot be judged later is worse+ * than a miss. Never throws: the document does not depend on the stamp.+ */+export function provenanceFromResponse(+ raw: unknown,+ logger: Meta["logger"],+ context: { scrapeId: string; cacheKey: string },+): FirePdfProvenance | null | undefined {+ const stamp = parseProvenance(raw);+ if (stamp.status === "ok") return stamp.provenance;+ if (stamp.status === "absent") return undefined;+ logger.warn("FirePDF provenance stamp not understood", {+ ...context,+ issue: stamp.issue,+ });+ return null;+}++/**+ * Why a result must not be remembered. Failed pages (fire-pdf's+ * `failed_pages` list, or the stamp's `quality.failed_pages` when the list+ * is absent) and pages that lost layout (`degraded_pages`) are usually+ * transient β a fleet incident, a deadline β and a cached copy would serve+ * that outcome to every later request for the document. A stamp this build+ * cannot read (`provenance === null`) is refused, and so is a stamp without+ * quality counts: "no counts" is not "no failures", and a later policy+ * reads the same counts. Only an unstamped result (a build from before the+ * stamp) is judged on the list alone. Partial pages are content-caused and+ * expensive to redo, so those results are still cached; the stamp's counts+ * let a later policy refresh them first.+ */+export function cacheRefusalReason(+ failedPages: readonly number[] | null | undefined,+ provenance: FirePdfProvenance | null | undefined,+): CacheRefusedReason | null {+ if (provenance === null) return "malformed_provenance";+ if ((failedPages?.length ?? 0) > 0) return "failed_pages";+ if (provenance === undefined) return null;+ const quality = provenance.quality;+ if (quality === undefined) return "missing_quality";+ if (quality.failed_pages > 0) return "failed_pages";+ if (quality.degraded_pages > 0) return "degraded_pages";return null;}@@ -250,6 +372,14 @@ export async function maybeSaveResult(args: {includeBlocks: boolean;pageMarkers?: boolean;result: PDFProcessorResult & { markdown: string };+ /**+ * fire-pdf's stamp for this result, stored verbatim with the entry.+ * `null` means a stamp was sent but could not be read (see+ * provenanceFromResponse); such a result is not cached.+ */+ provenance?: FirePdfProvenance | null;+ /** fire-pdf's `failed_pages` for this result; a non-empty list is not cached. */+ failedPages?: readonly number[] | null;}): Promise<void> {const {meta,@@ -260,8 +390,11 @@ export async function maybeSaveResult(args: {includeBlocks,pageMarkers = false,result,+ provenance,+ failedPages,} = args;if (meta.internalOptions.zeroDataRetention) return;+ if (!pdfCacheConfigured()) return;const { cacheable, ownVariant, baseVariant } = cacheKeyShape(mode,maxPages,@@ -273,32 +406,117 @@ export async function maybeSaveResult(args: {// See isRasterImagePayload: never remember an empty result for an image.if (isEmptyMarkdown(result.markdown) && isRasterImagePayload(base64Content))return;+ const cacheKey = resolvePdfCacheKey(base64Content);++ const refusal = cacheRefusalReason(failedPages, provenance);+ if (refusal !== null) {+ firePdfCacheRefusedWritesTotal.inc({ reason: refusal });+ firePdfCacheEventsTotal.inc({+ event: "refused_write",+ variant: ownVariant ?? "base",+ });+ meta.logger.info("FirePDF result not cached", {+ scrapeId: meta.id,+ requestedMode: mode,+ cacheVariant: ownVariant ?? "base",+ cacheKey,+ reason: refusal,+ failedPages: Math.max(+ failedPages?.length ?? 0,+ provenance?.quality?.failed_pages ?? 0,+ ),+ degradedPages: provenance?.quality?.degraded_pages ?? 0,+ });+ return;+ }++ const cachedAt = new Date().toISOString();+ const entry: CachedPdfResult = {+ ...result,+ ...(provenance ? { provenance } : {}),+ cachedAt,+ variant: ownVariant ?? "base",+ };++ // savePdfResultToCache returns null (without throwing) when GCS is not+ // configured or its retries are exhausted; only a persisted write is a+ // write. Every outcome is counted and logged with its key and variant so+ // a stale or missing entry can be traced to the write that produced it.+ const recordWrite = (+ savedKey: string | null,+ variant: string | undefined,+ alias: boolean,+ ): boolean => {+ const cacheVariant = variant ?? "base";+ if (savedKey === null) {+ firePdfCacheEventsTotal.inc({+ event: "write_failed",+ variant: cacheVariant,+ });+ meta.logger.warn("FirePDF result not persisted to cache", {+ scrapeId: meta.id,+ requestedMode: mode,+ cacheVariant,+ cacheKey,+ alias,+ });+ return false;+ }+ firePdfCacheEventsTotal.inc({ event: "write", variant: cacheVariant });+ meta.logger.info("Saved FirePDF result to cache", {+ scrapeId: meta.id,+ requestedMode: mode,+ cacheVariant,+ cacheKey,+ alias,+ generation: provenance?.generation ?? "unknown",+ buildSha: provenance?.build_sha ?? "unknown",+ });+ return true;+ };try {- await savePdfResultToCache(base64Content, result, "firepdf", ownVariant);+ const savedKey = await savePdfResultToCache(+ base64Content,+ entry,+ "firepdf",+ ownVariant,+ );+ if (!recordWrite(savedKey, ownVariant, false)) return;// An enriched (page/block-capable) parse is also a valid legacy result.// Populate the compact base key when it is missing so a later legacy// request never repeats the conversion. A sidecar miss can coexist with- // a warm legacy key during rollout, so avoid rewriting that object.- // Strip the enriched payloads to keep the hot-path cache object small.+ // a warm legacy key during rollout, so avoid rewriting that object β+ // except on an allowed refresh, where the old alias is exactly what the+ // caller asked to correct: plain requests would keep being served the+ // stale content otherwise. Strip the enriched payloads to keep the+ // hot-path cache object small.if ((includePageMarkdown || includeBlocks) && ownVariant !== baseVariant) {- const existingBase = await getPdfResultFromCache(- base64Content,- "firepdf",- baseVariant,- );+ const refreshed =+ getPDFRefresh(meta.options?.parsers) &&+ refreshDecisionFor(meta.id) === "allowed";+ const existingBase = refreshed+ ? null+ : await getPdfResultFromCache(base64Content, "firepdf", baseVariant);if (!existingBase || !isValidCachedDocument(existingBase)) {const {pageMarkdown: _pageMarkdown,blocks: _blocks,...baseResult} = result;- await savePdfResultToCache(+ const baseEntry: CachedPdfResult = {+ ...baseResult,+ ...(provenance ? { provenance } : {}),+ cachedAt,+ variant: baseVariant ?? "base",+ };+ const savedBase = await savePdfResultToCache(base64Content,- baseResult,+ baseEntry,"firepdf",baseVariant,);+ recordWrite(savedBase, baseVariant, true);}}} catch (error) {diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.tsindex ecdb5acaf..1d8bec15f 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.ts@@ -1,5 +1,23 @@import { Counter, Histogram } from "prom-client";+export const firePdfCacheEventsTotal = new Counter({+ name: "firecrawl_fire_pdf_cache_events_total",+ help: "fire-pdf content-cache events by outcome: hit, miss, bypass_refresh, bypass_refresh_denied, write, write_failed, refused_write",+ labelNames: ["event", "variant"],+});++export const firePdfCacheRefusedWritesTotal = new Counter({+ name: "firecrawl_fire_pdf_cache_refused_writes_total",+ help: "fire-pdf results not written to the content cache, by reason; reasons: failed_pages, degraded_pages, missing_quality, malformed_provenance",+ labelNames: ["reason"],+});++export type CacheRefusedReason =+ | "failed_pages"+ | "degraded_pages"+ | "missing_quality"+ | "malformed_provenance";+export const firePdfAsyncSubmittedTotal = new Counter({name: "firecrawl_fire_pdf_async_submitted_total",help: "Count of POST /jobs requests successfully submitted to fire-pdf async",diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/refresh-budget.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/refresh-budget.tsnew file mode 100644index 000000000..e2c9262ea--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/refresh-budget.ts@@ -0,0 +1,103 @@+import { RateLimiterRedis, RateLimiterRes } from "rate-limiter-flexible";+import { config } from "../../../../../config";++/**+ * Per-team budget for `parsers: [{ type: "pdf", refresh: true }]`.+ *+ * A refresh skips the content cache and forces a billed parse, so an+ * unbounded stream of them could push every request of a team back onto+ * fire-pdf. The budget is small (FIRE_PDF_CACHE_REFRESH_PER_MINUTE, default+ * 10 a minute) and fails closed: when it is exhausted, or the limiter store+ * is unreachable, the request is served from the cache like any other and+ * the decision is logged. 0 disables the option entirely.+ *+ * One decision per request. Engines fall back to one another (fire-pdf,+ * then RunPod MU) and each consults the budget, so the decision is+ * remembered by scrape id: a fallback neither spends a second token nor+ * flips to "serve the cached entry" after the first engine was allowed to+ * bypass it. The write path reads the same decision to know whether it+ * may overwrite existing aliases.+ *+ * The rate-limit Redis client is imported lazily so importing the cache+ * module never opens a connection (tests, tooling).+ */+export type RefreshDecision =+ | "allowed"+ | "limited"+ | "disabled"+ | "unavailable";++let limiter: RateLimiterRedis | null = null;++async function budget(): Promise<RateLimiterRedis> {+ if (limiter) return limiter;+ const { redisRateLimitClient } = await import(+ "../../../../../services/rate-limiter.js"+ );+ limiter = new RateLimiterRedis({+ storeClient: redisRateLimitClient,+ keyPrefix: "fire-pdf-cache-refresh",+ points: Math.max(1, config.FIRE_PDF_CACHE_REFRESH_PER_MINUTE),+ duration: 60,+ });+ return limiter;+}++// A request is over within minutes; the map is bounded so a burst of+// requests cannot grow it without limit (insertion order = age order).+const DECISION_TTL_MS = 15 * 60 * 1000;+const DECISION_CAP = 20_000;+const decisions = new Map<string, { decision: RefreshDecision; at: number }>();++function prune(now: number): void {+ for (const [scrapeId, entry] of decisions) {+ if (now - entry.at < DECISION_TTL_MS && decisions.size < DECISION_CAP) {+ break;+ }+ decisions.delete(scrapeId);+ }+}++/** The decision already taken for this request, if any. */+export function refreshDecisionFor(+ scrapeId: string | undefined,+): RefreshDecision | undefined {+ if (!scrapeId) return undefined;+ const entry = decisions.get(scrapeId);+ if (!entry) return undefined;+ if (Date.now() - entry.at >= DECISION_TTL_MS) {+ decisions.delete(scrapeId);+ return undefined;+ }+ return entry.decision;+}++async function decide(teamId: string | undefined): Promise<RefreshDecision> {+ if (config.FIRE_PDF_CACHE_REFRESH_PER_MINUTE <= 0) return "disabled";+ try {+ await (await budget()).consume(teamId ?? "anonymous", 1);+ return "allowed";+ } catch (err) {+ return err instanceof RateLimiterRes ? "limited" : "unavailable";+ }+}++/**+ * Spend one refresh of the team's budget for this request, or return the+ * decision this request already got. Without a scrape id every call is a+ * fresh decision.+ */+export async function consumeRefresh(+ teamId: string | undefined,+ scrapeId?: string,+): Promise<RefreshDecision> {+ const prior = refreshDecisionFor(scrapeId);+ if (prior) return prior;+ const decision = await decide(teamId);+ if (scrapeId) {+ const now = Date.now();+ prune(now);+ decisions.set(scrapeId, { decision, at: now });+ }+ return decision;+}diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.tsindex b622b41a0..a6cfb78fd 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts@@ -195,6 +195,77 @@ export const firePdfBlockPagesSchema = z.array(export const firePdfBlocksSchema = firePdfBlockPagesSchema.optional();+/**+ * fire-pdf's provenance stamp: who produced a result and how complete it is.+ * Stored verbatim with every cache entry so a later cache policy can judge+ * the entry without reading its content (fire-pdf docs/cache-policy.md).+ * `passthrough` keeps fields a newer fire-pdf adds.+ */+export const firePdfProvenanceSchema = z+ .object({+ generation: z.string(),+ build_sha: z.string(),+ built_at: z.string().nullable(),+ produced_at: z.string(),+ stages: z.array(z.string()).optional(),+ // Page counts: the write rule reads them, so a malformed stamp must+ // fail validation rather than pass as a healthy result. parseProvenance+ // turns that failure into a refused cache write, never a failed scrape.+ quality: z+ .object({+ total_pages: z.int().nonnegative(),+ failed_pages: z.int().nonnegative(),+ partial_pages: z.int().nonnegative(),+ degraded_pages: z.int().nonnegative(),+ ocr_pages: z.int().nonnegative(),+ })+ .passthrough()+ .optional(),+ // `passthrough` on each item too: a newer fire-pdf may add per-build+ // fields, and the entry stores the stamp verbatim.+ contributing_builds: z+ .array(+ z+ .object({+ generation: z.string(),+ build_sha: z.string(),+ built_at: z.string().nullable(),+ })+ .passthrough(),+ )+ .optional(),+ })+ .passthrough();++export type FirePdfProvenance = z.infer<typeof firePdfProvenanceSchema>;++/**+ * The stamp is parsed apart from the document. The document is what the+ * caller asked for; the stamp only decides what the cache may remember, so+ * a stamp this build cannot read degrades to `malformed` (the result is+ * served, not cached) instead of failing the response.+ */+export type ProvenanceParse =+ | { status: "absent" }+ | { status: "ok"; provenance: FirePdfProvenance }+ | { status: "malformed"; issue: string };++export function parseProvenance(raw: unknown): ProvenanceParse {+ // Only a missing field is "no stamp" (a build from before the stamp+ // existed). fire-pdf never sends an explicit null; one is unreadable.+ if (raw === undefined) return { status: "absent" };+ if (raw === null) return { status: "malformed", issue: "provenance: null" };+ const parsed = firePdfProvenanceSchema.safeParse(raw);+ if (parsed.success) return { status: "ok", provenance: parsed.data };+ return {+ status: "malformed",+ issue: parsed.error.issues+ .slice(0, 3)+ .map(i => `${i.path.join(".") || "<root>"}: ${i.message}`)+ .join("; "),+ };+}+export const resultResponseSchema = z.object({schema_version: z.union([z.literal(1), z.literal(2), z.literal(3)])@@ -210,6 +281,9 @@ export const resultResponseSchema = z.object({// is the only proof the fire-pdf worker build understood the option β// older workers ignore unknown option keys and omit it.page_markers: z.literal(true).optional(),+ // Raw on purpose: parsed separately by parseProvenance so a stamp this+ // build does not understand never fails the scrape.+ provenance: z.unknown().optional(),});export type PollResponse = z.infer<typeof pollResponseSchema>;diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.tsindex cb3a6b83b..dd2ff5aee 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts@@ -6,7 +6,11 @@ import type { PDFProcessorResult } from "./types";import type { PDFMode } from "../../../../controllers/v2/types";import { safeMarkdownToHtml } from "./markdownToHtml";import { createPdfCacheKey } from "../../../../lib/gcs-pdf-cache";-import { maybeSaveResult, tryGetCached } from "./fire-pdf/cache";+import {+ maybeSaveResult,+ provenanceFromResponse,+ tryGetCached,+} from "./fire-pdf/cache";import { firePdfBlocksSchema, firePdfPagesSchema } from "./fire-pdf/schema";import {buildFirePdfRequestMetadata,@@ -153,9 +157,15 @@ export async function scrapePDFWithFirePDF(schema: z.object({markdown: z.string(),failed_pages: z.array(z.number()).nullable(),+ partial_pages: z.array(z.number()).nullable().optional(),pages_processed: z.number().optional(),pages: firePdfPagesSchema,blocks: firePdfBlocksSchema,+ // fire-pdf's stamp (generation, build, stages, quality). Taken raw+ // and parsed separately (provenanceFromResponse) so a stamp this+ // build cannot read never fails the scrape; a missing stamp means an+ // older fire-pdf build.+ provenance: z.unknown().optional(),// Echo of an honored page_markers request. Markers are baked into// `markdown` and their absence is not reliably detectable there (a// single-page or fully-stitched document legitimately has none), so@@ -184,6 +194,10 @@ export async function scrapePDFWithFirePDF();}const pages = resp.pages_processed ?? pagesProcessed;+ const provenance = provenanceFromResponse(resp.provenance, logger, {+ scrapeId: meta.id,+ cacheKey: pdfSha256,+ });logger.info("FirePDF completed", {scrapeId: meta.id,@@ -191,8 +205,14 @@ export async function scrapePDFWithFirePDF(durationMs,markdownLength: resp.markdown.length,failedPages: resp.failed_pages,+ partialPages: resp.partial_pages ?? null,pagesProcessed: pages,perPageMs: pages ? Math.round(durationMs / pages) : undefined,+ // The content-cache key and the producer, so a report can be turned+ // into keys to purge and a result can be tied to a fire-pdf build.+ cacheKey: pdfSha256,+ generation: provenance?.generation ?? "unknown",+ buildSha: provenance?.build_sha ?? "unknown",});const processorResult: PDFProcessorResult & { markdown: string } = {@@ -213,6 +233,8 @@ export async function scrapePDFWithFirePDF(includeBlocks,pageMarkers,result: processorResult,+ provenance,+ failedPages: resp.failed_pages,});}diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/runpodMU.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/runpodMU.tsindex 554a5460c..dafbb0eb3 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/runpodMU.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/runpodMU.ts@@ -6,9 +6,13 @@ import { z } from "zod";import path from "node:path";import {getPdfResultFromCache,+ pdfCacheConfigured,+ resolvePdfCacheKey,savePdfResultToCache,} from "../../../../lib/gcs-pdf-cache";import type { PDFProcessorResult } from "./types";+import { getPDFRefresh } from "../../../../controllers/v2/types";+import { consumeRefresh } from "./fire-pdf/refresh-budget";export async function scrapePDFWithRunPodMU(meta: Meta,@@ -21,20 +25,66 @@ export async function scrapePDFWithRunPodMU(tempFilePath,});- if (!maxPages) {- try {- const cachedResult = await getPdfResultFromCache(base64Content);- if (cachedResult) {- meta.logger.info("Using cached RunPod MU result for PDF", {+ // The cache key (sha256 of the payload, a full pass over the document) is+ // computed at most once per request and reused for the read, the write+ // and the log lines. This cache lives under the legacy `pdf-cache-v2/`+ // prefix; the key is logged so a report can name the entry.+ let cacheKeyMemo: string | undefined;+ const cacheKeyOf = (): string =>+ (cacheKeyMemo ??= resolvePdfCacheKey(base64Content));++ // Without a bucket there is no cache: nothing is hashed and no refresh+ // budget is spent. Zero-data-retention requests never read the shared+ // cache, as on the fire-pdf path. `parsers: [{ type: "pdf", refresh: true }]`+ // skips it too, under the same per-team budget: a MinerU-diverted request+ // never reaches the fire-pdf path, so the budget has to be applied here as+ // well, and the decision is taken once per request, so a fire-pdf β MU+ // fallback keeps the decision the first engine got instead of spending a+ // second token. Over budget, or with the limiter unavailable, the entry is+ // served.+ const cacheOn = pdfCacheConfigured();+ if (cacheOn && !maxPages && !meta.internalOptions.zeroDataRetention) {+ const cacheKey = cacheKeyOf();+ let bypass = false;+ if (getPDFRefresh(meta.options?.parsers)) {+ const decision = await consumeRefresh(+ meta.internalOptions.teamId,+ meta.id,+ );+ bypass = decision === "allowed";+ if (bypass) {+ meta.logger.info("RunPod MU cache bypassed by refresh", {+ tempFilePath,+ cacheKey,+ cacheProvider: "runpod",+ });+ } else {+ meta.logger.warn("RunPod MU cache refresh not applied", {tempFilePath,+ cacheKey,+ cacheProvider: "runpod",+ decision,+ perMinute: config.FIRE_PDF_CACHE_REFRESH_PER_MINUTE,});- return cachedResult;}- } catch (error) {- meta.logger.warn("Error checking PDF cache, proceeding with RunPod MU", {- error,- tempFilePath,- });+ }+ if (!bypass) {+ try {+ const cachedResult = await getPdfResultFromCache({ key: cacheKey });+ if (cachedResult) {+ meta.logger.info("Using cached RunPod MU result for PDF", {+ tempFilePath,+ cacheKey,+ cacheProvider: "runpod",+ });+ return cachedResult;+ }+ } catch (error) {+ meta.logger.warn(+ "Error checking PDF cache, proceeding with RunPod MU",+ { error, tempFilePath, cacheKey, cacheProvider: "runpod" },+ );+ }}}@@ -191,9 +241,9 @@ export async function scrapePDFWithRunPodMU(html: await safeMarkdownToHtml(result.markdown, meta.logger, meta.id),};- if (!meta.internalOptions.zeroDataRetention) {+ if (cacheOn && !meta.internalOptions.zeroDataRetention) {try {- await savePdfResultToCache(base64Content, processorResult);+ await savePdfResultToCache({ key: cacheKeyOf() }, processorResult);} catch (error) {meta.logger.warn("Error saving PDF to cache", {error,