Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): submit a background fire-pdf refresh when the cache tier grants one (FIRE_PDF_CACHE_BACKGROUND_REFRESH, default off) (#5294)
Abimael Martell3fdaeb5b016f
diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex d214e52eb..e42e46c14 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -529,6 +529,13 @@ const configSchema = z.object({// cache service when one is configured; beyond the budget the request is// served normally. 0 disables the option.FIRE_PDF_CACHE_REFRESH_PER_MINUTE: z.coerce.number().int().min(0).default(10),+ // Opt in to background refresh: lookups carry `background_refresh: true`,+ // so the cache service can answer a refresh-campaign match with the stale+ // entry instead of a miss. When it grants a refresh (`refresh_requested`),+ // the stale result is served and the bytes are submitted to fire-pdf's bulk+ // queue for a fresh parse, which fire-pdf writes back to the cache. Needs+ // FIRE_PDF_CACHE_BASE_URL and FIRE_PDF_BASE_URL.+ FIRE_PDF_CACHE_BACKGROUND_REFRESH: z.stringbool().default(false),// /v2/parse sends every PDF (auto/ocr mode) to FirePDF: no direct// text-layer extraction, no FIRE_PDF_PERCENT sampling, no MinerU// diversion. Explicit `mode: "fast"` keeps direct extraction. FirePDFdiff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFBackgroundRefresh.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFBackgroundRefresh.test.tsnew file mode 100644index 000000000..5f31e4fb7--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFBackgroundRefresh.test.ts@@ -0,0 +1,299 @@+import { createHash } from "crypto";+import { fetch as undiciFetch } from "undici";+import { config } from "../../../../../config";+import { robustFetch } from "../../../lib/fetch";+import {+ BACKGROUND_REFRESH_CRAWL_ID,+ refreshOptionsForVariant,+ submitBackgroundRefresh,+} from "../fire-pdf/background-refresh";+import { lookupCachedResult } from "../fire-pdf/cache-service";+import { firePdfCacheBackgroundRefreshTotal } from "../fire-pdf/metrics";++vi.mock("undici", async importOriginal => ({+ ...(await importOriginal<typeof import("undici")>()),+ fetch: vi.fn(),+}));+vi.mock("../../../lib/fetch", () => ({ robustFetch: vi.fn() }));+vi.mock("../markdownToHtml", () => ({+ safeMarkdownToHtml: vi.fn(async (markdown: string) => `<p>${markdown}</p>`),+}));+vi.mock("../fire-pdf/refresh-budget", () => ({+ recordRefreshDecision: vi.fn(),+}));+vi.mock("../../../../../lib/gcs-pdf-cache", async () => {+ const { createHash } = await import("crypto");+ return {+ createPdfCacheKey: (input: string) =>+ createHash("sha256").update(input).digest("hex"),+ };+});++const lookupFetch = vi.mocked(robustFetch);+const jobsFetch = vi.mocked(undiciFetch);++const base64 = Buffer.from("%PDF-1.7 background refresh").toString("base64");++function makeMeta(zeroDataRetention = false) {+ return {+ id: "user-scrape",+ url: "https://example.com/doc.pdf",+ logger: { info: vi.fn(), warn: vi.fn() },+ internalOptions: {+ zeroDataRetention,+ teamId: "team-1",+ crawlId: "user-crawl",+ },+ options: {},+ mock: null,+ abort: {+ asSignal: () => new AbortController().signal,+ throwIfAborted: vi.fn(),+ },+ } as any;+}++const lookupArgs = {+ mode: "auto" as const,+ pagesProcessed: 2,+ includePageMarkdown: true,+ includeBlocks: false,+ pageMarkers: false,+ refresh: false,+ sourceKind: "pdf" as const,+ ownVariant: "page-markdown-v1",+};++const staleAnswer = (refreshRequested?: boolean) => ({+ outcome: "stale",+ key: "raw-abc",+ variant: "page-markdown-v1",+ campaign: "c1",+ ...(refreshRequested !== undefined && {+ refresh_requested: refreshRequested,+ }),+ result: { markdown: "# old", pages: [{ page: 1, markdown: "p1" }] },+});++const staleResult = {+ markdown: "# old",+ html: "<p># old</p>",+ pagesProcessed: 2,+ pageMarkdown: [{ page: 1, markdown: "p1" }],+};++function answerWith(fixture: unknown) {+ lookupFetch.mockImplementationOnce(async ({ schema }: any) =>+ schema.parse(fixture),+ );+}++function jobsAnswer(status: number, body: unknown = {}) {+ jobsFetch.mockResolvedValueOnce({+ status,+ json: async () => body,+ } as any);+}++async function refreshCount(result: string): Promise<number> {+ const metric = await firePdfCacheBackgroundRefreshTotal.get();+ return metric.values.find(v => v.labels.result === result)?.value ?? 0;+}++// Lets the fire-and-forget submit run to completion.+const settle = () => new Promise(resolve => setTimeout(resolve, 10));++describe("FirePDF cache background refresh", () => {+ beforeEach(() => {+ config.FIRE_PDF_CACHE_BASE_URL = "http://cache";+ config.FIRE_PDF_BASE_URL = "http://fire-pdf";+ config.FIRE_PDF_CACHE_BACKGROUND_REFRESH = true;+ lookupFetch.mockReset();+ jobsFetch.mockReset();+ firePdfCacheBackgroundRefreshTotal.reset();+ });++ afterEach(() => {+ config.FIRE_PDF_CACHE_BASE_URL = undefined;+ config.FIRE_PDF_BASE_URL = undefined;+ config.FIRE_PDF_CACHE_BACKGROUND_REFRESH = false;+ });++ it("leaves the lookup body unchanged and submits nothing when the flag is off", async () => {+ config.FIRE_PDF_CACHE_BACKGROUND_REFRESH = false;+ answerWith(staleAnswer(true));+ expect(await lookupCachedResult(makeMeta(), base64, lookupArgs)).toEqual(+ staleResult,+ );+ await settle();+ expect(lookupFetch.mock.calls[0][0].body).toStrictEqual({+ keys: [+ `raw-${createHash("sha256").update(Buffer.from(base64, "base64")).digest("hex")}`,+ createHash("sha256").update(base64).digest("hex"),+ ],+ options: { mode: "auto", include_page_markdown: true },+ team_id: "team-1",+ kind: "scrape",+ refresh: false,+ source_kind: "pdf",+ scrape_id: "user-scrape",+ });+ expect(jobsFetch).not.toHaveBeenCalled();+ });++ it("asks for background refresh when the flag is on", async () => {+ answerWith({ outcome: "miss", reason: "not_found" });+ expect(await lookupCachedResult(makeMeta(), base64, lookupArgs)).toBeNull();+ expect(lookupFetch.mock.calls[0][0].body).toMatchObject({+ background_refresh: true,+ });+ await settle();+ expect(jobsFetch).not.toHaveBeenCalled();+ });++ it("serves the stale entry and submits one refresh to the bulk queue", async () => {+ jobsAnswer(202, { status: "queued", lane: "fast" });+ answerWith(staleAnswer(true));+ expect(await lookupCachedResult(makeMeta(), base64, lookupArgs)).toEqual(+ staleResult,+ );+ await settle();+ expect(jobsFetch).toHaveBeenCalledTimes(1);+ const [url, init] = jobsFetch.mock.calls[0] as any;+ expect(url).toBe("http://fire-pdf/jobs");+ expect(init.method).toBe("POST");+ const body = JSON.parse(init.body);+ expect(body).toMatchObject({+ pdf_b64: base64,+ source: "firecrawl",+ zdr: false,+ crawl_id: BACKGROUND_REFRESH_CRAWL_ID,+ source_endpoint: "scrape",+ source_kind: "pdf",+ options: { mode: "auto", include_page_markdown: true, refresh: true },+ });+ expect(body.options).not.toHaveProperty("pages_estimate");+ expect(body).not.toHaveProperty("team_id");+ expect(body).not.toHaveProperty("team_concurrency");+ expect(body.scrape_id).toMatch(/^[0-9a-f-]{36}$/);+ expect(body.scrape_id).not.toBe("user-scrape");+ const deadlineMs = Date.parse(body.deadline_at) - Date.now();+ expect(deadlineMs).toBeGreaterThan(29 * 60 * 1_000);+ expect(deadlineMs).toBeLessThanOrEqual(30 * 60 * 1_000);+ expect(await refreshCount("submitted")).toBe(1);+ });++ it("does not submit when the service did not grant a refresh", async () => {+ answerWith(staleAnswer(false));+ expect(await lookupCachedResult(makeMeta(), base64, lookupArgs)).toEqual(+ staleResult,+ );+ answerWith(staleAnswer());+ expect(await lookupCachedResult(makeMeta(), base64, lookupArgs)).toEqual(+ staleResult,+ );+ await settle();+ expect(jobsFetch).not.toHaveBeenCalled();+ });++ it.each([+ ["a by-reference document", { key: "raw-abc" }, "pdf"],+ ["a raster image", base64, "image"],+ ] as const)(+ "does not offer background refresh for %s and ignores a grant",+ async (_name, input, sourceKind) => {+ answerWith(staleAnswer(true));+ await lookupCachedResult(makeMeta(), input as any, {+ ...lookupArgs,+ sourceKind,+ });+ await settle();+ expect(lookupFetch.mock.calls[0][0].body).not.toHaveProperty(+ "background_refresh",+ );+ expect(jobsFetch).not.toHaveBeenCalled();+ },+ );++ it("refreshes the variant that was served, not the one requested", async () => {+ jobsAnswer(202, { status: "queued" });+ answerWith({+ ...staleAnswer(true),+ variant: "ocr-page-markdown-blocks-v1",+ result: {+ markdown: "# old",+ pages: [{ page: 1, markdown: "p1" }],+ blocks: [],+ },+ });+ await lookupCachedResult(makeMeta(), base64, lookupArgs);+ await settle();+ const body = JSON.parse((jobsFetch.mock.calls[0] as any)[1].body);+ expect(body.options).toMatchObject({+ mode: "ocr",+ include_page_markdown: true,+ include_blocks: true,+ refresh: true,+ });+ });++ it.each([+ ["base", undefined, [undefined, false, false, false]],+ ["ocr", "auto", ["ocr", false, false, false]],+ ["page-markdown-v1", "auto", ["auto", true, false, false]],+ ["blocks-v1", undefined, [undefined, false, true, false]],+ ["ocr-page-markdown-blocks-v1", "auto", ["ocr", true, true, false]],+ ["markers-v1", "auto", ["auto", false, false, true]],+ ["ocr-page-markdown-markers-v1", "auto", ["ocr", true, false, true]],+ ] as const)(+ "maps variant %s to its job options",+ (variant, mode, [m, pages, blocks, markers]) => {+ expect(refreshOptionsForVariant(variant, mode)).toEqual({+ mode: m,+ includePageMarkdown: pages,+ includeBlocks: blocks,+ pageMarkers: markers,+ });+ },+ );++ it("serves the stale entry unaffected when the submit fails", async () => {+ jobsFetch.mockRejectedValueOnce(new Error("connection refused"));+ answerWith(staleAnswer(true));+ expect(await lookupCachedResult(makeMeta(), base64, lookupArgs)).toEqual(+ staleResult,+ );+ jobsAnswer(503, { error: "admission_rejected" });+ answerWith(staleAnswer(true));+ expect(await lookupCachedResult(makeMeta(), base64, lookupArgs)).toEqual(+ staleResult,+ );+ await settle();+ expect(jobsFetch).toHaveBeenCalledTimes(2);+ expect(await refreshCount("submit_failed")).toBe(2);+ });++ it("skips ZDR requests and raster images", async () => {+ const args = {+ mode: undefined,+ includePageMarkdown: false,+ includeBlocks: false,+ pageMarkers: false,+ };+ expect(+ await submitBackgroundRefresh(makeMeta(true), base64, {+ ...args,+ sourceKind: "pdf",+ }),+ ).toBe("skipped_zdr");+ expect(+ await submitBackgroundRefresh(makeMeta(), base64, {+ ...args,+ sourceKind: "image",+ }),+ ).toBe("skipped_image");+ expect(jobsFetch).not.toHaveBeenCalled();+ expect(await refreshCount("skipped_zdr")).toBe(1);+ expect(await refreshCount("skipped_image")).toBe(1);+ });+});diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/background-refresh.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/background-refresh.tsnew file mode 100644index 000000000..c4de43c97--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/background-refresh.ts@@ -0,0 +1,163 @@+import crypto from "crypto";+import { fetch as undiciFetch } from "undici";+import type { Meta } from "../../..";+import type { PDFMode } from "../../../../../controllers/v2/types";+import { config } from "../../../../../config";+import type { PdfCacheKeyInput } from "../../../../../lib/gcs-pdf-cache";+import {+ type BackgroundRefreshResult,+ firePdfCacheBackgroundRefreshTotal,+} from "./metrics";+import {+ buildFirePdfRequestMetadata,+ type FirePdfSourceKind,+} from "./request-metadata";+import { MAX_DEADLINE_MS } from "./schema";+import { buildFirePdfJobOptions, firePdfHeaders } from "./utils";++/**+ * The crawl id every background refresh is submitted under. fire-pdf queues+ * any job with a crawl id on its bulk (lowest priority) queues, and paces+ * bulk publishing round-robin across crawl ids, so one shared id keeps all+ * refreshes together as a single crawl's worth of work behind real crawls.+ * Firecrawl crawl ids are UUIDs, so this can never name a real crawl.+ */+export const BACKGROUND_REFRESH_CRAWL_ID = "fire-pdf-cache-refresh";++// Nobody waits on a refresh: advertise the longest deadline fire-pdf accepts+// (POST /jobs rejects anything further out as deadline_too_far).+const REFRESH_DEADLINE_MS = MAX_DEADLINE_MS;+// Bounds only the submit request itself, never the job.+const SUBMIT_TIMEOUT_MS = 30_000;++/**+ * The job options that re-create the variant the cache service served. A+ * lookup may be answered from a richer or OCR variant than the request would+ * write, and that served variant is the one to refresh. Variant names follow+ * fire-pdf's lattice (cache.ts cacheKeyShape): an `ocr` prefix, and+ * `page-markdown`, `blocks` and `markers` capabilities.+ */+export function refreshOptionsForVariant(+ variant: string,+ requestMode: PDFMode | undefined,+): {+ mode: PDFMode | undefined;+ includePageMarkdown: boolean;+ includeBlocks: boolean;+ pageMarkers: boolean;+} {+ return {+ mode: variant.startsWith("ocr") ? "ocr" : requestMode,+ includePageMarkdown: variant.includes("page-markdown"),+ includeBlocks: variant.includes("blocks"),+ pageMarkers: variant.includes("markers"),+ };+}++/**+ * Submits a fresh parse of a document the cache service answered stale with+ * `refresh_requested: true`. Fire-and-forget: the scrape has already been+ * served the stale entry, this is never awaited by it, never polls, and never+ * throws. fire-pdf writes the fresh entry itself when the job completes.+ *+ * Built from the same pieces as the async submit (submit.ts) but sent once+ * without its retry and fallback handling, which counts into the async+ * fallback metrics and is bound to the scrape's abort signal. The job:+ * - uses a fresh scrape_id, so it can never collide with the caller's job;+ * - carries `options.refresh`, so fire-pdf parses rather than answering+ * from the cache, with the options of the variant that was served stale+ * (refreshOptionsForVariant), so it rewrites that entry;+ * - carries BACKGROUND_REFRESH_CRAWL_ID, so it lands on the bulk queue;+ * - carries no team_id (nor team_concurrency): it is internal work the+ * cache tier budgeted, so it must not count against the requesting+ * team's admission counters or allowance. The requesting team and scrape+ * are logged here for tracing instead.+ * Nothing in Firecrawl bills it: its result never comes back to a scrape.+ */+export function submitBackgroundRefresh(+ meta: Meta,+ input: PdfCacheKeyInput,+ args: {+ mode: PDFMode | undefined;+ includePageMarkdown: boolean;+ includeBlocks: boolean;+ pageMarkers: boolean;+ sourceKind: FirePdfSourceKind;+ },+ fetchImpl: typeof undiciFetch = undiciFetch,+): Promise<BackgroundRefreshResult> {+ return submit(meta, input, args, fetchImpl)+ .catch(error => {+ meta.logger.warn("FirePDF background refresh submit failed", {+ scrapeId: meta.id,+ error,+ });+ return "submit_failed" as const;+ })+ .then(result => {+ firePdfCacheBackgroundRefreshTotal.inc({ result });+ return result;+ });+}++async function submit(+ meta: Meta,+ input: PdfCacheKeyInput,+ args: Parameters<typeof submitBackgroundRefresh>[2],+ fetchImpl: typeof undiciFetch,+): Promise<BackgroundRefreshResult> {+ if (meta.internalOptions.zeroDataRetention === true) return "skipped_zdr";+ // A by-reference document's bytes are in fire-pdf's input bucket, not here.+ if (typeof input !== "string") return "skipped_by_reference";+ // Raster images are only ever parsed through the sync endpoint.+ if (args.sourceKind === "image") return "skipped_image";+ if (!config.FIRE_PDF_BASE_URL) {+ throw new Error("FIRE_PDF_BASE_URL is not set");+ }++ const refreshScrapeId = crypto.randomUUID();+ const resp = await fetchImpl(`${config.FIRE_PDF_BASE_URL}/jobs`, {+ method: "POST",+ headers: firePdfHeaders(true),+ body: JSON.stringify({+ pdf_b64: input,+ scrape_id: refreshScrapeId,+ source: "firecrawl",+ ...buildFirePdfRequestMetadata(meta, args.sourceKind),+ zdr: false,+ deadline_at: new Date(Date.now() + REFRESH_DEADLINE_MS).toISOString(),+ crawl_id: BACKGROUND_REFRESH_CRAWL_ID,+ options: {+ ...buildFirePdfJobOptions({+ maxPages: undefined,+ pagesProcessed: undefined,+ mode: args.mode,+ includePageMarkdown: args.includePageMarkdown,+ includeBlocks: args.includeBlocks,+ pageMarkers: args.pageMarkers,+ // The cache is only consulted without an explicit table format.+ tableFormat: undefined,+ }),+ refresh: true,+ },+ }),+ signal: AbortSignal.timeout(SUBMIT_TIMEOUT_MS),+ });+ const body = await resp.json().catch(() => ({}));+ if (resp.status !== 200 && resp.status !== 202) {+ meta.logger.warn("FirePDF background refresh rejected", {+ scrapeId: meta.id,+ refreshScrapeId,+ status: resp.status,+ body,+ });+ return "submit_failed";+ }+ meta.logger.info("FirePDF background refresh submitted", {+ scrapeId: meta.id,+ refreshScrapeId,+ teamId: meta.internalOptions.teamId,+ httpStatus: resp.status,+ });+ return "submitted";+}diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache-service.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache-service.tsindex 9d5076305..b8579a190 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache-service.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache-service.ts@@ -16,6 +16,10 @@ import {} from "../../../../../lib/otel-tracer";import { safeMarkdownToHtml } from "../markdownToHtml";import type { PDFProcessorResult } from "../types";+import {+ refreshOptionsForVariant,+ submitBackgroundRefresh,+} from "./background-refresh";import { firePdfCacheEventsTotal } from "./metrics";import { recordRefreshDecision } from "./refresh-budget";import { type FirePdfSourceKind, firePdfRequestKind } from "./request-metadata";@@ -72,6 +76,9 @@ const lookupOutcomeSchema = z.discriminatedUnion("outcome", [generation: z.string().optional(),campaign: z.string(),capped: z.boolean().optional(),+ // Only with `background_refresh` on the lookup: the service granted a+ // refresh and expects this client, which holds the bytes, to submit it.+ refresh_requested: z.boolean().optional(),result: cachedResultSchema,}),z.object({@@ -184,6 +191,14 @@ async function lookupWithinSpan(} = args;// An already-cancelled scrape sends nothing.meta.abort.throwIfAborted();+ // Background refresh is only offered when this client could act on a+ // grant: it needs the bytes (inline, not by reference) of a PDF (images+ // are only parsed through the sync endpoint), outside zero data retention.+ const canRefreshInBackground =+ config.FIRE_PDF_CACHE_BACKGROUND_REFRESH === true &&+ typeof input === "string" &&+ sourceKind === "pdf" &&+ meta.internalOptions.zeroDataRetention !== true;const keys = await cacheLookupKeys(input);if (sourceKind === "pdf" &&@@ -213,6 +228,7 @@ async function lookupWithinSpan(refresh,source_kind: sourceKind,scrape_id: meta.id,+ ...(canRefreshInBackground && { background_refresh: true }),},schema: lookupOutcomeSchema,logger: meta.logger,@@ -302,6 +318,17 @@ async function lookupWithinSpan(outcome: answer.outcome,...(answer.outcome === "stale" && { campaign: answer.campaign }),});+ // Never awaited: the stale entry is served now, the refresh lands later.+ if (+ canRefreshInBackground &&+ answer.outcome === "stale" &&+ answer.refresh_requested === true+ ) {+ void submitBackgroundRefresh(meta, input, {+ ...refreshOptionsForVariant(answer.variant, mode),+ sourceKind,+ });+ }// The entry carries markdown only; html is rendered here as on a fresh parse.return {markdown: result.markdown,diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.tsindex c413d4e9c..3befd414b 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.ts@@ -12,6 +12,19 @@ export const firePdfCacheRefusedWritesTotal = new Counter({labelNames: ["reason"],});+export const firePdfCacheBackgroundRefreshTotal = new Counter({+ name: "firecrawl_fire_pdf_cache_background_refresh_total",+ help: "Background refreshes the fire-pdf cache service granted on a stale answer, by result: submitted, submit_failed, skipped_by_reference, skipped_zdr, skipped_image",+ labelNames: ["result"],+});++export type BackgroundRefreshResult =+ | "submitted"+ | "submit_failed"+ | "skipped_by_reference"+ | "skipped_zdr"+ | "skipped_image";+export type CacheRefusedReason =| "failed_pages"| "degraded_pages"