Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): look up cached PDF results through fire-pdf's cache service (#4727)
Abimael Martell992053cfab48
diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex a00b526b6..21e032cdb 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -388,9 +388,14 @@ const configSchema = z.object({FIRE_PDF_PERCENT: z.coerce.number().min(0).max(100).default(10),FIRE_PDF_BASE_URL: z.string().optional(),FIRE_PDF_API_KEY: z.string().optional(),+ // Cached fire-pdf results are looked up through this service when set+ // (POST /cache/lookup, same key as FIRE_PDF_API_KEY), and fire-pdf writes+ // them; without it the bucket is read and written from here.+ FIRE_PDF_CACHE_BASE_URL: z.string().optional(),// `parsers: [{ type: "pdf", refresh: true }]` skips the content cache and- // forces a fresh parse. Per team, per minute; beyond the budget the- // request is served normally. 0 disables the option.+ // forces a fresh parse. Per team, per minute, budgeted here or by the+ // cache service when one is configured; beyond the budget the request is+ // served normally. 0 disables the option.FIRE_PDF_CACHE_REFRESH_PER_MINUTE: z.coerce.number().int().min(0).default(10),// Raster image OCR of image URLs and parse uploads through FirePDF (see// lib/image-ocr-gate.ts). Needs FIRE_PDF_BASE_URL.diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.tsindex c34da25c5..def6a0072 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts@@ -9,27 +9,45 @@ import {provenanceFromResponse,tryGetCached,} from "../fire-pdf/cache";-import { consumeRefresh, refreshDecisionFor } from "../fire-pdf/refresh-budget";+import {+ consumeRefresh,+ recordRefreshDecision,+ refreshDecisionFor,+} from "../fire-pdf/refresh-budget";import { firePdfProvenanceSchema } from "../fire-pdf/schema";import {firePdfCacheEventsTotal,firePdfCacheRefusedWritesTotal,} from "../fire-pdf/metrics";+import crypto from "crypto";+import { config } from "../../../../../config";+import { robustFetch } from "../../../lib/fetch";vi.mock("../fire-pdf/refresh-budget", () => ({consumeRefresh: vi.fn(async () => "allowed"),refreshDecisionFor: vi.fn(() => undefined),+ recordRefreshDecision: vi.fn(),}));-vi.mock("../../../../../lib/gcs-pdf-cache", () => ({- pdfCacheConfigured: vi.fn(() => true),- getPdfResultFromCache: vi.fn(),- savePdfResultToCache: vi.fn(),- resolvePdfCacheKey: vi.fn((input: string | { key: string }) =>- typeof input === "string" ? `key-of-${input}` : input.key,- ),+vi.mock("../../../lib/fetch", () => ({ robustFetch: vi.fn() }));+vi.mock("../markdownToHtml", () => ({+ safeMarkdownToHtml: vi.fn(async (markdown: string) => `<p>${markdown}</p>`),}));+vi.mock("../../../../../lib/gcs-pdf-cache", async () => {+ const { createHash } = await import("crypto");+ return {+ pdfCacheConfigured: vi.fn(() => true),+ getPdfResultFromCache: vi.fn(),+ savePdfResultToCache: vi.fn(),+ resolvePdfCacheKey: vi.fn((input: string | { key: string }) =>+ typeof input === "string" ? `key-of-${input}` : input.key,+ ),+ createPdfCacheKey: (input: string) =>+ createHash("sha256").update(input).digest("hex"),+ };+});+const getCached = vi.mocked(getPdfResultFromCache);const saveCached = vi.mocked(savePdfResultToCache);@@ -1348,3 +1366,325 @@ describe("FirePDF cache counters", () => {});});});++describe("FirePDF cache through the lookup service", () => {+ const fetchMock = vi.mocked(robustFetch);+ // The service's answer is parsed with the schema the client hands to the+ // fetch, so these fixtures pin the wire contract.+ const answerWith = (fixture: unknown) =>+ fetchMock.mockImplementationOnce(async ({ schema }: any) =>+ schema.parse(fixture),+ );+ const base64 = Buffer.from("%PDF-1.7 service test").toString("base64");+ const rawKey = `raw-${crypto+ .createHash("sha256")+ .update(Buffer.from(base64, "base64"))+ .digest("hex")}`;+ const inlineKey = crypto.createHash("sha256").update(base64).digest("hex");++ function serviceMeta(parsers?: unknown[]) {+ return {+ id: "svc-test",+ logger: { info: vi.fn(), warn: vi.fn() },+ internalOptions: { zeroDataRetention: false, teamId: "team-1" },+ mock: null,+ abort: {+ asSignal: () => new AbortController().signal,+ throwIfAborted: vi.fn(),+ },+ ...(parsers ? { options: { parsers } } : {}),+ } as any;+ }+ // Base64 of a PNG signature followed by padding: sniffs as image/png.+ const PNG_BASE64 = Buffer.from([+ 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 0, 0, 0, 0, 0,+ ]).toString("base64");++ beforeEach(() => {+ config.FIRE_PDF_CACHE_BASE_URL = "http://cache";+ fetchMock.mockReset();+ getCached.mockReset();+ saveCached.mockReset();+ vi.mocked(consumeRefresh).mockClear();+ vi.mocked(recordRefreshDecision).mockClear();+ });++ afterEach(() => {+ config.FIRE_PDF_CACHE_BASE_URL = undefined;+ });++ it("asks the service once with both keys and the options, and serves a hit without reading the bucket", async () => {+ answerWith({+ outcome: "hit",+ key: rawKey,+ variant: "page-markdown-v1",+ result: {+ markdown: "# doc",+ pages_processed: 2,+ pages: [{ page: 1, markdown: "p1" }],+ },+ });+ const result = await tryGetCached(+ serviceMeta(),+ base64,+ "auto",+ undefined,+ 5,+ true,+ false,+ );+ expect(result).toEqual({+ markdown: "# doc",+ html: "<p># doc</p>",+ pagesProcessed: 2,+ pageMarkdown: [{ page: 1, markdown: "p1" }],+ });+ expect(fetchMock).toHaveBeenCalledTimes(1);+ const call = fetchMock.mock.calls[0][0];+ expect(call.url).toBe("http://cache/cache/lookup");+ expect(call.method).toBe("POST");+ expect(call.body).toEqual({+ keys: [rawKey, inlineKey],+ options: { mode: "auto", include_page_markdown: true },+ team_id: "team-1",+ kind: "scrape",+ refresh: false,+ source_kind: "pdf",+ scrape_id: "svc-test",+ });+ expect(getCached).not.toHaveBeenCalled();+ });++ it("uses the caller's key alone for a by-reference document", async () => {+ answerWith({ outcome: "miss", reason: "not_found" });+ expect(+ await tryGetCached(+ serviceMeta(),+ { key: "raw-abc" },+ undefined,+ undefined,+ undefined,+ false,+ false,+ ),+ ).toBeNull();+ expect(fetchMock.mock.calls[0][0].body.keys).toEqual(["raw-abc"]);+ });++ it("serves a stale answer, and treats a failed lookup as a miss", async () => {+ answerWith({+ outcome: "stale",+ key: inlineKey,+ variant: "base",+ campaign: "c1",+ result: { markdown: "# old" },+ });+ expect(+ await tryGetCached(+ serviceMeta(),+ base64,+ undefined,+ undefined,+ 3,+ false,+ false,+ ),+ ).toEqual({ markdown: "# old", html: "<p># old</p>", pagesProcessed: 3 });+ fetchMock.mockRejectedValueOnce(new Error("timed out"));+ const meta = serviceMeta();+ expect(+ await tryGetCached(meta, base64, undefined, undefined, 3, false, false),+ ).toBeNull();+ expect(meta.logger.warn).toHaveBeenCalledWith(+ "FirePDF cache lookup failed, proceeding",+ expect.objectContaining({ scrapeId: "svc-test" }),+ );+ });++ it("passes refresh through for the service to budget", async () => {+ answerWith({ outcome: "miss", reason: "refresh" });+ expect(+ await tryGetCached(+ serviceMeta([{ type: "pdf", refresh: true }]),+ base64,+ undefined,+ undefined,+ undefined,+ false,+ false,+ ),+ ).toBeNull();+ expect(fetchMock.mock.calls[0][0].body.refresh).toBe(true);+ expect(consumeRefresh).not.toHaveBeenCalled();+ // The service's decision is remembered for a fallback engine.+ expect(recordRefreshDecision).toHaveBeenCalledWith("svc-test", "allowed");+ answerWith({+ outcome: "hit",+ key: inlineKey,+ variant: "base",+ result: { markdown: "# doc" },+ });+ await tryGetCached(+ serviceMeta([{ type: "pdf", refresh: true }]),+ base64,+ undefined,+ undefined,+ undefined,+ false,+ false,+ );+ expect(recordRefreshDecision).toHaveBeenLastCalledWith(+ "svc-test",+ "limited",+ );+ });++ it("sends no refresh when the option is switched off locally", async () => {+ const perMinute = config.FIRE_PDF_CACHE_REFRESH_PER_MINUTE;+ config.FIRE_PDF_CACHE_REFRESH_PER_MINUTE = 0;+ try {+ answerWith({ outcome: "miss", reason: "not_found" });+ await tryGetCached(+ serviceMeta([{ type: "pdf", refresh: true }]),+ base64,+ undefined,+ undefined,+ undefined,+ false,+ false,+ );+ expect(fetchMock.mock.calls[0][0].body.refresh).toBe(false);+ expect(recordRefreshDecision).not.toHaveBeenCalled();+ } finally {+ config.FIRE_PDF_CACHE_REFRESH_PER_MINUTE = perMinute;+ }+ });++ it("serves a marker request only an entry that carries markers", async () => {+ answerWith({+ outcome: "hit",+ key: inlineKey,+ variant: "markers-v1",+ result: { markdown: "<!-- page 1 -->\n# doc" },+ });+ expect(+ await tryGetCached(+ serviceMeta(),+ base64,+ undefined,+ undefined,+ undefined,+ false,+ false,+ true,+ ),+ ).toBeNull();+ answerWith({+ outcome: "hit",+ key: inlineKey,+ variant: "markers-v1",+ result: { markdown: "<!-- page 1 -->\n# doc", page_markers: true },+ });+ expect(+ await tryGetCached(+ serviceMeta(),+ base64,+ undefined,+ undefined,+ undefined,+ false,+ false,+ true,+ ),+ ).toMatchObject({ markdown: "<!-- page 1 -->\n# doc" });+ expect(fetchMock.mock.calls[1][0].body.options).toEqual({+ page_markers: true,+ });+ });++ it("lets the scrape's own abort through instead of counting it as a lookup failure", async () => {+ fetchMock.mockRejectedValueOnce(new Error("aborted mid-lookup"));+ const meta = serviceMeta();+ // The abort lands once the request has gone out.+ meta.abort.throwIfAborted.mockImplementation(() => {+ if (fetchMock.mock.calls.length > 0) throw new Error("scrape aborted");+ });+ await expect(+ tryGetCached(meta, base64, undefined, undefined, undefined, false, false),+ ).rejects.toThrow("scrape aborted");+ expect(meta.logger.warn).not.toHaveBeenCalled();+ });++ it("never serves an empty result for a raster image, and names the image to the service", async () => {+ answerWith({+ outcome: "hit",+ key: "k",+ variant: "base",+ result: { markdown: " " },+ });+ expect(+ await tryGetCached(+ serviceMeta(),+ PNG_BASE64,+ undefined,+ undefined,+ undefined,+ false,+ false,+ ),+ ).toBeNull();+ expect(fetchMock.mock.calls[0][0].body.source_kind).toBe("image");+ });++ it("never lets a poorer entry satisfy a richer request", async () => {+ answerWith({+ outcome: "hit",+ key: inlineKey,+ variant: "base",+ result: { markdown: "# doc" },+ });+ expect(+ await tryGetCached(+ serviceMeta(),+ base64,+ undefined,+ undefined,+ undefined,+ true,+ false,+ ),+ ).toBeNull();+ answerWith({+ outcome: "hit",+ key: inlineKey,+ variant: "page-markdown-v1",+ result: { markdown: "# doc", pages: [{ page: 1, markdown: "p1" }] },+ });+ expect(+ await tryGetCached(+ serviceMeta(),+ base64,+ undefined,+ undefined,+ undefined,+ true,+ true,+ ),+ ).toBeNull();+ });++ it("does not write results itself while the service is in use", async () => {+ await maybeSaveResult({+ meta: serviceMeta(),+ base64Content: base64,+ mode: undefined,+ maxPages: undefined,+ includePageMarkdown: false,+ includeBlocks: false,+ result: { markdown: "# doc", html: "<p># doc</p>" },+ provenance: undefined,+ failedPages: [],+ });+ expect(saveCached).not.toHaveBeenCalled();+ });+});diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.tsindex 3d9240adc..87f6f88ec 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts@@ -97,9 +97,10 @@ export async function scrapePDFWithFirePDFAsync(// Cache addressing: inline submits keep the historical key (sha256 of// the base64 payload); by-reference submits use a `raw-` prefixed key// over the raw-byte sha, so repeat scrapes of the same large document- // don't reprocess it. The two keyspaces are deliberately distinct β an- // inline and a by-reference parse of the same document do not share- // entries. The LOOKUP for by-reference happens at the call site BEFORE+ // don't reprocess it. On the direct-bucket path the two keyspaces are+ // distinct β an inline and a by-reference parse of the same document do+ // not share entries; the cache service is asked with both keys. The+ // LOOKUP for by-reference happens at the call site BEFORE// the input object is uploaded (a hit must skip the 30-256MB transfer,// which has already happened by the time this function runs); only the// inline path looks up here. Both paths save here.diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache-service.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache-service.tsnew file mode 100644index 000000000..25eb2ab61--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache-service.ts@@ -0,0 +1,198 @@+import crypto from "crypto";+import { z } from "zod";+import type { Meta } from "../../..";+import { config } from "../../../../../config";+import {+ createPdfCacheKey,+ type PdfCacheKeyInput,+} from "../../../../../lib/gcs-pdf-cache";+import type { PDFMode } from "../../../../../controllers/v2/types";+import { robustFetch } from "../../../lib/fetch";+import { safeMarkdownToHtml } from "../markdownToHtml";+import type { PDFProcessorResult } from "../types";+import { firePdfCacheEventsTotal } from "./metrics";+import { recordRefreshDecision } from "./refresh-budget";+import { type FirePdfSourceKind, firePdfRequestKind } from "./request-metadata";+import { firePdfBlocksSchema, firePdfPagesSchema } from "./schema";++/** fire-pdf answers cache lookups itself when this is set, and writes the entries. */+export function cacheServiceConfigured(): boolean {+ return !!config.FIRE_PDF_CACHE_BASE_URL;+}++/** Keys the document may be cached under: the bytes' hash first, then the historical hash of the base64 payload. */+async function cacheLookupKeys(input: PdfCacheKeyInput): Promise<string[]> {+ if (typeof input !== "string") return [input.key];+ // The bytes' hash runs off the event loop; the payload hash is the one the bucket path already paid.+ const digest = await crypto.subtle.digest(+ "SHA-256",+ Buffer.from(input, "base64"),+ );+ return [+ `raw-${Buffer.from(digest).toString("hex")}`,+ createPdfCacheKey(input),+ ];+}++// The service's own read bound, so a lookup never waits longer than the+// service would; past it the lookup is a miss and the document is parsed. One+// retry, within the same bound, covers a connection that dropped before the+// request was handled.+const LOOKUP_TIMEOUT_MS = 5_000;+const LOOKUP_TRIES = 2;++const cachedResultSchema = z.object({+ markdown: z.string(),+ pages_processed: z.number().int().nonnegative().optional(),+ pages: firePdfPagesSchema,+ blocks: firePdfBlocksSchema,+ // Present on entries written under a page-marker variant; the proof a+ // marker request needs, as the echo is on a fresh parse.+ page_markers: z.literal(true).optional(),+});++const lookupOutcomeSchema = z.discriminatedUnion("outcome", [+ z.object({+ outcome: z.literal("hit"),+ key: z.string(),+ variant: z.string(),+ result: cachedResultSchema,+ }),+ z.object({+ outcome: z.literal("stale"),+ key: z.string(),+ variant: z.string(),+ campaign: z.string(),+ result: cachedResultSchema,+ }),+ z.object({+ outcome: z.literal("miss"),+ reason: z.string(),+ key: z.string().optional(),+ campaign: z.string().optional(),+ }),+]);++export async function lookupCachedResult(+ meta: Meta,+ input: PdfCacheKeyInput,+ args: {+ mode: PDFMode | undefined;+ pagesProcessed: number | undefined;+ includePageMarkdown: boolean;+ includeBlocks: boolean;+ pageMarkers: boolean;+ refresh: boolean;+ sourceKind: FirePdfSourceKind;+ /** The variant this request would write, for the miss labels. */+ ownVariant: string;+ },+): Promise<PDFProcessorResult | null> {+ const {+ mode,+ pagesProcessed,+ includePageMarkdown,+ includeBlocks,+ pageMarkers,+ refresh,+ sourceKind,+ ownVariant,+ } = args;+ // An already-cancelled scrape sends nothing.+ meta.abort.throwIfAborted();+ let answer: z.infer<typeof lookupOutcomeSchema>;+ try {+ answer = await robustFetch({+ url: `${config.FIRE_PDF_CACHE_BASE_URL}/cache/lookup`,+ method: "POST",+ headers: config.FIRE_PDF_API_KEY+ ? { Authorization: `Bearer ${config.FIRE_PDF_API_KEY}` }+ : undefined,+ body: {+ keys: await cacheLookupKeys(input),+ options: {+ ...(mode !== undefined && { mode }),+ ...(includePageMarkdown && { include_page_markdown: true }),+ ...(includeBlocks && { include_blocks: true }),+ ...(pageMarkers && { page_markers: true }),+ },+ team_id: meta.internalOptions.teamId,+ kind: firePdfRequestKind(meta),+ refresh,+ source_kind: sourceKind,+ scrape_id: meta.id,+ },+ schema: lookupOutcomeSchema,+ logger: meta.logger,+ mock: meta.mock,+ tryCount: LOOKUP_TRIES,+ abort: AbortSignal.any([+ meta.abort.asSignal(),+ AbortSignal.timeout(LOOKUP_TIMEOUT_MS),+ ]),+ });+ } catch (error) {+ // The scrape's own abort is not a lookup failure.+ meta.abort.throwIfAborted();+ firePdfCacheEventsTotal.inc({ event: "lookup_error", variant: ownVariant });+ meta.logger.warn("FirePDF cache lookup failed, proceeding", {+ scrapeId: meta.id,+ error,+ });+ return null;+ }++ // The service took the refresh decision; a fallback engine must reuse it+ // rather than spend a second token or serve what was meant to be bypassed.+ if (refresh) {+ recordRefreshDecision(+ meta.id,+ answer.outcome === "miss" && answer.reason === "refresh"+ ? "allowed"+ : "limited",+ );+ }+ if (answer.outcome === "miss") {+ firePdfCacheEventsTotal.inc({+ event: answer.reason === "refresh" ? "bypass_refresh" : "miss",+ variant: ownVariant,+ });+ return null;+ }+ const { result } = answer;+ // Never an unusable entry, whatever the service says: a sidecar the request+ // needs must be present, a marker request needs a marker entry, and an+ // empty result for a raster image is a stale verdict, not an answer (see+ // isRasterImagePayload in cache.ts).+ if (+ (includePageMarkdown && result.pages === undefined) ||+ (includeBlocks && result.blocks === undefined) ||+ (pageMarkers && result.page_markers !== true) ||+ (sourceKind === "image" && result.markdown.trim().length === 0)+ ) {+ firePdfCacheEventsTotal.inc({ event: "miss", variant: ownVariant });+ return null;+ }+ firePdfCacheEventsTotal.inc({+ event: answer.outcome,+ variant: answer.variant,+ });+ meta.logger.info("Using cached FirePDF result", {+ scrapeId: meta.id,+ requestedMode: mode,+ cacheVariant: answer.variant,+ cacheKey: answer.key,+ outcome: answer.outcome,+ ...(answer.outcome === "stale" && { campaign: answer.campaign }),+ });+ // The entry carries markdown only; html is rendered here as on a fresh parse.+ return {+ markdown: result.markdown,+ html: await safeMarkdownToHtml(result.markdown, meta.logger, meta.id),+ pagesProcessed: result.pages_processed ?? pagesProcessed,+ ...(includePageMarkdown && result.pages+ ? { pageMarkdown: result.pages }+ : {}),+ ...(includeBlocks && result.blocks ? { blocks: result.blocks } : {}),+ };+}diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.tsindex aa2f1a9e7..170b2daed 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts@@ -19,6 +19,7 @@ import {firePdfCacheEventsTotal,firePdfCacheRefusedWritesTotal,} from "./metrics";+import { cacheServiceConfigured, lookupCachedResult } from "./cache-service";import { consumeRefresh, refreshDecisionFor } from "./refresh-budget";import {firePdfBlockPagesSchema,@@ -190,8 +191,9 @@ export async function tryGetCached(pageMarkers = false,): Promise<PDFProcessorResult | null> {if (meta.internalOptions.zeroDataRetention) return null;- // No bucket, no cache: nothing to hash, no refresh budget to spend.- if (!pdfCacheConfigured()) return null;+ const viaService = cacheServiceConfigured();+ // No service and no bucket, no cache: nothing to hash, no refresh budget to spend.+ if (!viaService && !pdfCacheConfigured()) return null;const { cacheable, lookupVariants, ownVariant } = cacheKeyShape(mode,maxPages,@@ -200,6 +202,22 @@ export async function tryGetCached(pageMarkers,);if (!cacheable) return null;+ if (viaService) {+ // The service probes the variants, applies the read-time rules and+ // budgets `refresh`; a failed lookup is a miss.+ return lookupCachedResult(meta, base64Content, {+ mode,+ pagesProcessed,+ includePageMarkdown,+ includeBlocks,+ pageMarkers,+ refresh:+ config.FIRE_PDF_CACHE_REFRESH_PER_MINUTE > 0 &&+ getPDFRefresh(meta.options?.parsers),+ sourceKind: isRasterImagePayload(base64Content) ? "image" : "pdf",+ ownVariant: ownVariant ?? "base",+ });+ }const cacheKey = resolvePdfCacheKey(base64Content);// `parsers: [{ type: "pdf", refresh: true }]`: the caller wants this@@ -394,6 +412,8 @@ export async function maybeSaveResult(args: {failedPages,} = args;if (meta.internalOptions.zeroDataRetention) return;+ // With the lookup service in use, fire-pdf writes the result itself.+ if (cacheServiceConfigured()) return;if (!pdfCacheConfigured()) return;const { cacheable, ownVariant, baseVariant } = cacheKeyShape(mode,diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.tsindex 1d8bec15f..b4621e21c 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.ts@@ -2,7 +2,7 @@ import { Counter, Histogram } from "prom-client";export const firePdfCacheEventsTotal = new Counter({name: "firecrawl_fire_pdf_cache_events_total",- help: "fire-pdf content-cache events by outcome: hit, miss, bypass_refresh, bypass_refresh_denied, write, write_failed, refused_write",+ help: "fire-pdf content-cache events by outcome: hit, stale, miss, bypass_refresh, bypass_refresh_denied, lookup_error, write, write_failed, refused_write",labelNames: ["event", "variant"],});diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/refresh-budget.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/refresh-budget.tsindex e52714be4..971de62b5 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/refresh-budget.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/refresh-budget.ts@@ -78,6 +78,17 @@ async function decide(teamId: string | undefined): Promise<RefreshDecision> {}}+/** Remember a decision taken elsewhere for this request, so a fallback engine reuses it. */+export function recordRefreshDecision(+ scrapeId: string | undefined,+ decision: RefreshDecision,+): void {+ if (!scrapeId) return;+ const now = Date.now();+ prune(now);+ decisions.set(scrapeId, { decision, at: now });+}+/*** Spend one refresh of the team's budget for this request, or return the* decision this request already got. Without a scrape id every call is adiff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/request-metadata.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/request-metadata.tsindex d67ed2d5a..07b909ee6 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/request-metadata.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/request-metadata.ts@@ -4,6 +4,14 @@ import { hasCustomRequestContext } from "../../../lib/request-context";/** What we hand fire-pdf: a PDF, or a raster image wrapped as a one-page document. */export type FirePdfSourceKind = "pdf" | "image";+/** How the document reached us: an upload to the parse endpoint, or a scrape. */+export function firePdfRequestKind(meta: Meta): "scrape" | "parse" {+ return meta.internalOptions.isParse === true ||+ meta.internalOptions.uploadedFile !== undefined+ ? "parse"+ : "scrape";+}+export function buildFirePdfRequestMetadata(meta: Meta,sourceKind: FirePdfSourceKind = "pdf",@@ -13,10 +21,8 @@ export function buildFirePdfRequestMetadata(source_kind: FirePdfSourceKind;url?: string;} {- const isParse =- meta.internalOptions.isParse === true ||- meta.internalOptions.uploadedFile !== undefined;- const source_endpoint = isParse ? "parse" : "scrape";+ const source_endpoint = firePdfRequestKind(meta);+ const isParse = source_endpoint === "parse";// Describe supplied request options without forwarding their values.const source_request_context = hasCustomRequestContext(meta.options)? "custom"