Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

feat(api): look up cached PDF results through fire-pdf's cache service (#4727)

Abimael Martell992053cfab48
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index a00b526b6..21e032cdb 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -388,9 +388,14 @@ const configSchema = z.object({
FIRE_PDF_PERCENT: z.coerce.number().min(0).max(100).default(10),
FIRE_PDF_BASE_URL: z.string().optional(),
FIRE_PDF_API_KEY: z.string().optional(),
+ // Cached fire-pdf results are looked up through this service when set
+ // (POST /cache/lookup, same key as FIRE_PDF_API_KEY), and fire-pdf writes
+ // them; without it the bucket is read and written from here.
+ FIRE_PDF_CACHE_BASE_URL: z.string().optional(),
// `parsers: [{ type: "pdf", refresh: true }]` skips the content cache and
- // forces a fresh parse. Per team, per minute; beyond the budget the
- // request is served normally. 0 disables the option.
+ // forces a fresh parse. Per team, per minute, budgeted here or by the
+ // cache service when one is configured; beyond the budget the request is
+ // served normally. 0 disables the option.
FIRE_PDF_CACHE_REFRESH_PER_MINUTE: z.coerce.number().int().min(0).default(10),
// Raster image OCR of image URLs and parse uploads through FirePDF (see
// lib/image-ocr-gate.ts). Needs FIRE_PDF_BASE_URL.
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts
index c34da25c5..def6a0072 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts
@@ -9,27 +9,45 @@ import {
provenanceFromResponse,
tryGetCached,
} from "../fire-pdf/cache";
-import { consumeRefresh, refreshDecisionFor } from "../fire-pdf/refresh-budget";
+import {
+ consumeRefresh,
+ recordRefreshDecision,
+ refreshDecisionFor,
+} from "../fire-pdf/refresh-budget";
import { firePdfProvenanceSchema } from "../fire-pdf/schema";
import {
firePdfCacheEventsTotal,
firePdfCacheRefusedWritesTotal,
} from "../fire-pdf/metrics";
+import crypto from "crypto";
+import { config } from "../../../../../config";
+import { robustFetch } from "../../../lib/fetch";
vi.mock("../fire-pdf/refresh-budget", () => ({
consumeRefresh: vi.fn(async () => "allowed"),
refreshDecisionFor: vi.fn(() => undefined),
+ recordRefreshDecision: vi.fn(),
}));
-vi.mock("../../../../../lib/gcs-pdf-cache", () => ({
- pdfCacheConfigured: vi.fn(() => true),
- getPdfResultFromCache: vi.fn(),
- savePdfResultToCache: vi.fn(),
- resolvePdfCacheKey: vi.fn((input: string | { key: string }) =>
- typeof input === "string" ? `key-of-${input}` : input.key,
- ),
+vi.mock("../../../lib/fetch", () => ({ robustFetch: vi.fn() }));
+vi.mock("../markdownToHtml", () => ({
+ safeMarkdownToHtml: vi.fn(async (markdown: string) => `<p>${markdown}</p>`),
}));
+vi.mock("../../../../../lib/gcs-pdf-cache", async () => {
+ const { createHash } = await import("crypto");
+ return {
+ pdfCacheConfigured: vi.fn(() => true),
+ getPdfResultFromCache: vi.fn(),
+ savePdfResultToCache: vi.fn(),
+ resolvePdfCacheKey: vi.fn((input: string | { key: string }) =>
+ typeof input === "string" ? `key-of-${input}` : input.key,
+ ),
+ createPdfCacheKey: (input: string) =>
+ createHash("sha256").update(input).digest("hex"),
+ };
+});
+
const getCached = vi.mocked(getPdfResultFromCache);
const saveCached = vi.mocked(savePdfResultToCache);
@@ -1348,3 +1366,325 @@ describe("FirePDF cache counters", () => {
});
});
});
+
+describe("FirePDF cache through the lookup service", () => {
+ const fetchMock = vi.mocked(robustFetch);
+ // The service's answer is parsed with the schema the client hands to the
+ // fetch, so these fixtures pin the wire contract.
+ const answerWith = (fixture: unknown) =>
+ fetchMock.mockImplementationOnce(async ({ schema }: any) =>
+ schema.parse(fixture),
+ );
+ const base64 = Buffer.from("%PDF-1.7 service test").toString("base64");
+ const rawKey = `raw-${crypto
+ .createHash("sha256")
+ .update(Buffer.from(base64, "base64"))
+ .digest("hex")}`;
+ const inlineKey = crypto.createHash("sha256").update(base64).digest("hex");
+
+ function serviceMeta(parsers?: unknown[]) {
+ return {
+ id: "svc-test",
+ logger: { info: vi.fn(), warn: vi.fn() },
+ internalOptions: { zeroDataRetention: false, teamId: "team-1" },
+ mock: null,
+ abort: {
+ asSignal: () => new AbortController().signal,
+ throwIfAborted: vi.fn(),
+ },
+ ...(parsers ? { options: { parsers } } : {}),
+ } as any;
+ }
+ // Base64 of a PNG signature followed by padding: sniffs as image/png.
+ const PNG_BASE64 = Buffer.from([
+ 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 0, 0, 0, 0, 0,
+ ]).toString("base64");
+
+ beforeEach(() => {
+ config.FIRE_PDF_CACHE_BASE_URL = "http://cache";
+ fetchMock.mockReset();
+ getCached.mockReset();
+ saveCached.mockReset();
+ vi.mocked(consumeRefresh).mockClear();
+ vi.mocked(recordRefreshDecision).mockClear();
+ });
+
+ afterEach(() => {
+ config.FIRE_PDF_CACHE_BASE_URL = undefined;
+ });
+
+ it("asks the service once with both keys and the options, and serves a hit without reading the bucket", async () => {
+ answerWith({
+ outcome: "hit",
+ key: rawKey,
+ variant: "page-markdown-v1",
+ result: {
+ markdown: "# doc",
+ pages_processed: 2,
+ pages: [{ page: 1, markdown: "p1" }],
+ },
+ });
+ const result = await tryGetCached(
+ serviceMeta(),
+ base64,
+ "auto",
+ undefined,
+ 5,
+ true,
+ false,
+ );
+ expect(result).toEqual({
+ markdown: "# doc",
+ html: "<p># doc</p>",
+ pagesProcessed: 2,
+ pageMarkdown: [{ page: 1, markdown: "p1" }],
+ });
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ const call = fetchMock.mock.calls[0][0];
+ expect(call.url).toBe("http://cache/cache/lookup");
+ expect(call.method).toBe("POST");
+ expect(call.body).toEqual({
+ keys: [rawKey, inlineKey],
+ options: { mode: "auto", include_page_markdown: true },
+ team_id: "team-1",
+ kind: "scrape",
+ refresh: false,
+ source_kind: "pdf",
+ scrape_id: "svc-test",
+ });
+ expect(getCached).not.toHaveBeenCalled();
+ });
+
+ it("uses the caller's key alone for a by-reference document", async () => {
+ answerWith({ outcome: "miss", reason: "not_found" });
+ expect(
+ await tryGetCached(
+ serviceMeta(),
+ { key: "raw-abc" },
+ undefined,
+ undefined,
+ undefined,
+ false,
+ false,
+ ),
+ ).toBeNull();
+ expect(fetchMock.mock.calls[0][0].body.keys).toEqual(["raw-abc"]);
+ });
+
+ it("serves a stale answer, and treats a failed lookup as a miss", async () => {
+ answerWith({
+ outcome: "stale",
+ key: inlineKey,
+ variant: "base",
+ campaign: "c1",
+ result: { markdown: "# old" },
+ });
+ expect(
+ await tryGetCached(
+ serviceMeta(),
+ base64,
+ undefined,
+ undefined,
+ 3,
+ false,
+ false,
+ ),
+ ).toEqual({ markdown: "# old", html: "<p># old</p>", pagesProcessed: 3 });
+ fetchMock.mockRejectedValueOnce(new Error("timed out"));
+ const meta = serviceMeta();
+ expect(
+ await tryGetCached(meta, base64, undefined, undefined, 3, false, false),
+ ).toBeNull();
+ expect(meta.logger.warn).toHaveBeenCalledWith(
+ "FirePDF cache lookup failed, proceeding",
+ expect.objectContaining({ scrapeId: "svc-test" }),
+ );
+ });
+
+ it("passes refresh through for the service to budget", async () => {
+ answerWith({ outcome: "miss", reason: "refresh" });
+ expect(
+ await tryGetCached(
+ serviceMeta([{ type: "pdf", refresh: true }]),
+ base64,
+ undefined,
+ undefined,
+ undefined,
+ false,
+ false,
+ ),
+ ).toBeNull();
+ expect(fetchMock.mock.calls[0][0].body.refresh).toBe(true);
+ expect(consumeRefresh).not.toHaveBeenCalled();
+ // The service's decision is remembered for a fallback engine.
+ expect(recordRefreshDecision).toHaveBeenCalledWith("svc-test", "allowed");
+ answerWith({
+ outcome: "hit",
+ key: inlineKey,
+ variant: "base",
+ result: { markdown: "# doc" },
+ });
+ await tryGetCached(
+ serviceMeta([{ type: "pdf", refresh: true }]),
+ base64,
+ undefined,
+ undefined,
+ undefined,
+ false,
+ false,
+ );
+ expect(recordRefreshDecision).toHaveBeenLastCalledWith(
+ "svc-test",
+ "limited",
+ );
+ });
+
+ it("sends no refresh when the option is switched off locally", async () => {
+ const perMinute = config.FIRE_PDF_CACHE_REFRESH_PER_MINUTE;
+ config.FIRE_PDF_CACHE_REFRESH_PER_MINUTE = 0;
+ try {
+ answerWith({ outcome: "miss", reason: "not_found" });
+ await tryGetCached(
+ serviceMeta([{ type: "pdf", refresh: true }]),
+ base64,
+ undefined,
+ undefined,
+ undefined,
+ false,
+ false,
+ );
+ expect(fetchMock.mock.calls[0][0].body.refresh).toBe(false);
+ expect(recordRefreshDecision).not.toHaveBeenCalled();
+ } finally {
+ config.FIRE_PDF_CACHE_REFRESH_PER_MINUTE = perMinute;
+ }
+ });
+
+ it("serves a marker request only an entry that carries markers", async () => {
+ answerWith({
+ outcome: "hit",
+ key: inlineKey,
+ variant: "markers-v1",
+ result: { markdown: "<!-- page 1 -->\n# doc" },
+ });
+ expect(
+ await tryGetCached(
+ serviceMeta(),
+ base64,
+ undefined,
+ undefined,
+ undefined,
+ false,
+ false,
+ true,
+ ),
+ ).toBeNull();
+ answerWith({
+ outcome: "hit",
+ key: inlineKey,
+ variant: "markers-v1",
+ result: { markdown: "<!-- page 1 -->\n# doc", page_markers: true },
+ });
+ expect(
+ await tryGetCached(
+ serviceMeta(),
+ base64,
+ undefined,
+ undefined,
+ undefined,
+ false,
+ false,
+ true,
+ ),
+ ).toMatchObject({ markdown: "<!-- page 1 -->\n# doc" });
+ expect(fetchMock.mock.calls[1][0].body.options).toEqual({
+ page_markers: true,
+ });
+ });
+
+ it("lets the scrape's own abort through instead of counting it as a lookup failure", async () => {
+ fetchMock.mockRejectedValueOnce(new Error("aborted mid-lookup"));
+ const meta = serviceMeta();
+ // The abort lands once the request has gone out.
+ meta.abort.throwIfAborted.mockImplementation(() => {
+ if (fetchMock.mock.calls.length > 0) throw new Error("scrape aborted");
+ });
+ await expect(
+ tryGetCached(meta, base64, undefined, undefined, undefined, false, false),
+ ).rejects.toThrow("scrape aborted");
+ expect(meta.logger.warn).not.toHaveBeenCalled();
+ });
+
+ it("never serves an empty result for a raster image, and names the image to the service", async () => {
+ answerWith({
+ outcome: "hit",
+ key: "k",
+ variant: "base",
+ result: { markdown: " " },
+ });
+ expect(
+ await tryGetCached(
+ serviceMeta(),
+ PNG_BASE64,
+ undefined,
+ undefined,
+ undefined,
+ false,
+ false,
+ ),
+ ).toBeNull();
+ expect(fetchMock.mock.calls[0][0].body.source_kind).toBe("image");
+ });
+
+ it("never lets a poorer entry satisfy a richer request", async () => {
+ answerWith({
+ outcome: "hit",
+ key: inlineKey,
+ variant: "base",
+ result: { markdown: "# doc" },
+ });
+ expect(
+ await tryGetCached(
+ serviceMeta(),
+ base64,
+ undefined,
+ undefined,
+ undefined,
+ true,
+ false,
+ ),
+ ).toBeNull();
+ answerWith({
+ outcome: "hit",
+ key: inlineKey,
+ variant: "page-markdown-v1",
+ result: { markdown: "# doc", pages: [{ page: 1, markdown: "p1" }] },
+ });
+ expect(
+ await tryGetCached(
+ serviceMeta(),
+ base64,
+ undefined,
+ undefined,
+ undefined,
+ true,
+ true,
+ ),
+ ).toBeNull();
+ });
+
+ it("does not write results itself while the service is in use", async () => {
+ await maybeSaveResult({
+ meta: serviceMeta(),
+ base64Content: base64,
+ mode: undefined,
+ maxPages: undefined,
+ includePageMarkdown: false,
+ includeBlocks: false,
+ result: { markdown: "# doc", html: "<p># doc</p>" },
+ provenance: undefined,
+ failedPages: [],
+ });
+ expect(saveCached).not.toHaveBeenCalled();
+ });
+});
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts
index 3d9240adc..87f6f88ec 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts
@@ -97,9 +97,10 @@ export async function scrapePDFWithFirePDFAsync(
// Cache addressing: inline submits keep the historical key (sha256 of
// the base64 payload); by-reference submits use a `raw-` prefixed key
// over the raw-byte sha, so repeat scrapes of the same large document
- // don't reprocess it. The two keyspaces are deliberately distinct β€” an
- // inline and a by-reference parse of the same document do not share
- // entries. The LOOKUP for by-reference happens at the call site BEFORE
+ // don't reprocess it. On the direct-bucket path the two keyspaces are
+ // distinct β€” an inline and a by-reference parse of the same document do
+ // not share entries; the cache service is asked with both keys. The
+ // LOOKUP for by-reference happens at the call site BEFORE
// the input object is uploaded (a hit must skip the 30-256MB transfer,
// which has already happened by the time this function runs); only the
// inline path looks up here. Both paths save here.
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache-service.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache-service.ts
new file mode 100644
index 000000000..25eb2ab61
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache-service.ts
@@ -0,0 +1,198 @@
+import crypto from "crypto";
+import { z } from "zod";
+import type { Meta } from "../../..";
+import { config } from "../../../../../config";
+import {
+ createPdfCacheKey,
+ type PdfCacheKeyInput,
+} from "../../../../../lib/gcs-pdf-cache";
+import type { PDFMode } from "../../../../../controllers/v2/types";
+import { robustFetch } from "../../../lib/fetch";
+import { safeMarkdownToHtml } from "../markdownToHtml";
+import type { PDFProcessorResult } from "../types";
+import { firePdfCacheEventsTotal } from "./metrics";
+import { recordRefreshDecision } from "./refresh-budget";
+import { type FirePdfSourceKind, firePdfRequestKind } from "./request-metadata";
+import { firePdfBlocksSchema, firePdfPagesSchema } from "./schema";
+
+/** fire-pdf answers cache lookups itself when this is set, and writes the entries. */
+export function cacheServiceConfigured(): boolean {
+ return !!config.FIRE_PDF_CACHE_BASE_URL;
+}
+
+/** Keys the document may be cached under: the bytes' hash first, then the historical hash of the base64 payload. */
+async function cacheLookupKeys(input: PdfCacheKeyInput): Promise<string[]> {
+ if (typeof input !== "string") return [input.key];
+ // The bytes' hash runs off the event loop; the payload hash is the one the bucket path already paid.
+ const digest = await crypto.subtle.digest(
+ "SHA-256",
+ Buffer.from(input, "base64"),
+ );
+ return [
+ `raw-${Buffer.from(digest).toString("hex")}`,
+ createPdfCacheKey(input),
+ ];
+}
+
+// The service's own read bound, so a lookup never waits longer than the
+// service would; past it the lookup is a miss and the document is parsed. One
+// retry, within the same bound, covers a connection that dropped before the
+// request was handled.
+const LOOKUP_TIMEOUT_MS = 5_000;
+const LOOKUP_TRIES = 2;
+
+const cachedResultSchema = z.object({
+ markdown: z.string(),
+ pages_processed: z.number().int().nonnegative().optional(),
+ pages: firePdfPagesSchema,
+ blocks: firePdfBlocksSchema,
+ // Present on entries written under a page-marker variant; the proof a
+ // marker request needs, as the echo is on a fresh parse.
+ page_markers: z.literal(true).optional(),
+});
+
+const lookupOutcomeSchema = z.discriminatedUnion("outcome", [
+ z.object({
+ outcome: z.literal("hit"),
+ key: z.string(),
+ variant: z.string(),
+ result: cachedResultSchema,
+ }),
+ z.object({
+ outcome: z.literal("stale"),
+ key: z.string(),
+ variant: z.string(),
+ campaign: z.string(),
+ result: cachedResultSchema,
+ }),
+ z.object({
+ outcome: z.literal("miss"),
+ reason: z.string(),
+ key: z.string().optional(),
+ campaign: z.string().optional(),
+ }),
+]);
+
+export async function lookupCachedResult(
+ meta: Meta,
+ input: PdfCacheKeyInput,
+ args: {
+ mode: PDFMode | undefined;
+ pagesProcessed: number | undefined;
+ includePageMarkdown: boolean;
+ includeBlocks: boolean;
+ pageMarkers: boolean;
+ refresh: boolean;
+ sourceKind: FirePdfSourceKind;
+ /** The variant this request would write, for the miss labels. */
+ ownVariant: string;
+ },
+): Promise<PDFProcessorResult | null> {
+ const {
+ mode,
+ pagesProcessed,
+ includePageMarkdown,
+ includeBlocks,
+ pageMarkers,
+ refresh,
+ sourceKind,
+ ownVariant,
+ } = args;
+ // An already-cancelled scrape sends nothing.
+ meta.abort.throwIfAborted();
+ let answer: z.infer<typeof lookupOutcomeSchema>;
+ try {
+ answer = await robustFetch({
+ url: `${config.FIRE_PDF_CACHE_BASE_URL}/cache/lookup`,
+ method: "POST",
+ headers: config.FIRE_PDF_API_KEY
+ ? { Authorization: `Bearer ${config.FIRE_PDF_API_KEY}` }
+ : undefined,
+ body: {
+ keys: await cacheLookupKeys(input),
+ options: {
+ ...(mode !== undefined && { mode }),
+ ...(includePageMarkdown && { include_page_markdown: true }),
+ ...(includeBlocks && { include_blocks: true }),
+ ...(pageMarkers && { page_markers: true }),
+ },
+ team_id: meta.internalOptions.teamId,
+ kind: firePdfRequestKind(meta),
+ refresh,
+ source_kind: sourceKind,
+ scrape_id: meta.id,
+ },
+ schema: lookupOutcomeSchema,
+ logger: meta.logger,
+ mock: meta.mock,
+ tryCount: LOOKUP_TRIES,
+ abort: AbortSignal.any([
+ meta.abort.asSignal(),
+ AbortSignal.timeout(LOOKUP_TIMEOUT_MS),
+ ]),
+ });
+ } catch (error) {
+ // The scrape's own abort is not a lookup failure.
+ meta.abort.throwIfAborted();
+ firePdfCacheEventsTotal.inc({ event: "lookup_error", variant: ownVariant });
+ meta.logger.warn("FirePDF cache lookup failed, proceeding", {
+ scrapeId: meta.id,
+ error,
+ });
+ return null;
+ }
+
+ // The service took the refresh decision; a fallback engine must reuse it
+ // rather than spend a second token or serve what was meant to be bypassed.
+ if (refresh) {
+ recordRefreshDecision(
+ meta.id,
+ answer.outcome === "miss" && answer.reason === "refresh"
+ ? "allowed"
+ : "limited",
+ );
+ }
+ if (answer.outcome === "miss") {
+ firePdfCacheEventsTotal.inc({
+ event: answer.reason === "refresh" ? "bypass_refresh" : "miss",
+ variant: ownVariant,
+ });
+ return null;
+ }
+ const { result } = answer;
+ // Never an unusable entry, whatever the service says: a sidecar the request
+ // needs must be present, a marker request needs a marker entry, and an
+ // empty result for a raster image is a stale verdict, not an answer (see
+ // isRasterImagePayload in cache.ts).
+ if (
+ (includePageMarkdown && result.pages === undefined) ||
+ (includeBlocks && result.blocks === undefined) ||
+ (pageMarkers && result.page_markers !== true) ||
+ (sourceKind === "image" && result.markdown.trim().length === 0)
+ ) {
+ firePdfCacheEventsTotal.inc({ event: "miss", variant: ownVariant });
+ return null;
+ }
+ firePdfCacheEventsTotal.inc({
+ event: answer.outcome,
+ variant: answer.variant,
+ });
+ meta.logger.info("Using cached FirePDF result", {
+ scrapeId: meta.id,
+ requestedMode: mode,
+ cacheVariant: answer.variant,
+ cacheKey: answer.key,
+ outcome: answer.outcome,
+ ...(answer.outcome === "stale" && { campaign: answer.campaign }),
+ });
+ // The entry carries markdown only; html is rendered here as on a fresh parse.
+ return {
+ markdown: result.markdown,
+ html: await safeMarkdownToHtml(result.markdown, meta.logger, meta.id),
+ pagesProcessed: result.pages_processed ?? pagesProcessed,
+ ...(includePageMarkdown && result.pages
+ ? { pageMarkdown: result.pages }
+ : {}),
+ ...(includeBlocks && result.blocks ? { blocks: result.blocks } : {}),
+ };
+}
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts
index aa2f1a9e7..170b2daed 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts
@@ -19,6 +19,7 @@ import {
firePdfCacheEventsTotal,
firePdfCacheRefusedWritesTotal,
} from "./metrics";
+import { cacheServiceConfigured, lookupCachedResult } from "./cache-service";
import { consumeRefresh, refreshDecisionFor } from "./refresh-budget";
import {
firePdfBlockPagesSchema,
@@ -190,8 +191,9 @@ export async function tryGetCached(
pageMarkers = false,
): Promise<PDFProcessorResult | null> {
if (meta.internalOptions.zeroDataRetention) return null;
- // No bucket, no cache: nothing to hash, no refresh budget to spend.
- if (!pdfCacheConfigured()) return null;
+ const viaService = cacheServiceConfigured();
+ // No service and no bucket, no cache: nothing to hash, no refresh budget to spend.
+ if (!viaService && !pdfCacheConfigured()) return null;
const { cacheable, lookupVariants, ownVariant } = cacheKeyShape(
mode,
maxPages,
@@ -200,6 +202,22 @@ export async function tryGetCached(
pageMarkers,
);
if (!cacheable) return null;
+ if (viaService) {
+ // The service probes the variants, applies the read-time rules and
+ // budgets `refresh`; a failed lookup is a miss.
+ return lookupCachedResult(meta, base64Content, {
+ mode,
+ pagesProcessed,
+ includePageMarkdown,
+ includeBlocks,
+ pageMarkers,
+ refresh:
+ config.FIRE_PDF_CACHE_REFRESH_PER_MINUTE > 0 &&
+ getPDFRefresh(meta.options?.parsers),
+ sourceKind: isRasterImagePayload(base64Content) ? "image" : "pdf",
+ ownVariant: ownVariant ?? "base",
+ });
+ }
const cacheKey = resolvePdfCacheKey(base64Content);
// `parsers: [{ type: "pdf", refresh: true }]`: the caller wants this
@@ -394,6 +412,8 @@ export async function maybeSaveResult(args: {
failedPages,
} = args;
if (meta.internalOptions.zeroDataRetention) return;
+ // With the lookup service in use, fire-pdf writes the result itself.
+ if (cacheServiceConfigured()) return;
if (!pdfCacheConfigured()) return;
const { cacheable, ownVariant, baseVariant } = cacheKeyShape(
mode,
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.ts
index 1d8bec15f..b4621e21c 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.ts
@@ -2,7 +2,7 @@ import { Counter, Histogram } from "prom-client";
export const firePdfCacheEventsTotal = new Counter({
name: "firecrawl_fire_pdf_cache_events_total",
- help: "fire-pdf content-cache events by outcome: hit, miss, bypass_refresh, bypass_refresh_denied, write, write_failed, refused_write",
+ help: "fire-pdf content-cache events by outcome: hit, stale, miss, bypass_refresh, bypass_refresh_denied, lookup_error, write, write_failed, refused_write",
labelNames: ["event", "variant"],
});
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/refresh-budget.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/refresh-budget.ts
index e52714be4..971de62b5 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/refresh-budget.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/refresh-budget.ts
@@ -78,6 +78,17 @@ async function decide(teamId: string | undefined): Promise<RefreshDecision> {
}
}
+/** Remember a decision taken elsewhere for this request, so a fallback engine reuses it. */
+export function recordRefreshDecision(
+ scrapeId: string | undefined,
+ decision: RefreshDecision,
+): void {
+ if (!scrapeId) return;
+ const now = Date.now();
+ prune(now);
+ decisions.set(scrapeId, { decision, at: now });
+}
+
/**
* Spend one refresh of the team's budget for this request, or return the
* decision this request already got. Without a scrape id every call is a
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/request-metadata.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/request-metadata.ts
index d67ed2d5a..07b909ee6 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/request-metadata.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/request-metadata.ts
@@ -4,6 +4,14 @@ import { hasCustomRequestContext } from "../../../lib/request-context";
/** What we hand fire-pdf: a PDF, or a raster image wrapped as a one-page document. */
export type FirePdfSourceKind = "pdf" | "image";
+/** How the document reached us: an upload to the parse endpoint, or a scrape. */
+export function firePdfRequestKind(meta: Meta): "scrape" | "parse" {
+ return meta.internalOptions.isParse === true ||
+ meta.internalOptions.uploadedFile !== undefined
+ ? "parse"
+ : "scrape";
+}
+
export function buildFirePdfRequestMetadata(
meta: Meta,
sourceKind: FirePdfSourceKind = "pdf",
@@ -13,10 +21,8 @@ export function buildFirePdfRequestMetadata(
source_kind: FirePdfSourceKind;
url?: string;
} {
- const isParse =
- meta.internalOptions.isParse === true ||
- meta.internalOptions.uploadedFile !== undefined;
- const source_endpoint = isParse ? "parse" : "scrape";
+ const source_endpoint = firePdfRequestKind(meta);
+ const isParse = source_endpoint === "parse";
// Describe supplied request options without forwarding their values.
const source_request_context = hasCustomRequestContext(meta.options)
? "custom"