Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): skip URL index PDF copies that fire-pdf has re-parsed (INDEX_PDF_SUPERSEDE_CHECK, default off) (#5293)
Abimael Martell423c4ade665a
diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex 23cdf0ba0..0925b1348 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -264,6 +264,11 @@ const configSchema = z.object({// (e.g. 600000 = 10min) also short-circuits repeat lookups for URLs with no// index entry. Kept short so any missed cache-clear self-heals quickly.INDEX_CACHE_NEGATIVE_TTL_MS: z.coerce.number().default(0),+ // On a URL index hit for a PDF whose entry recorded its fire-pdf content+ // key, ask the fire-pdf cache service whether it holds a newer parse; if it+ // does, the index copy is treated as a miss so the request is answered from+ // the newer entry (no parse) and re-indexed. Needs FIRE_PDF_CACHE_BASE_URL.+ INDEX_PDF_SUPERSEDE_CHECK: z.stringbool().default(false),REDIS_URL: z.string().optional(),REDIS_EVICT_URL: z.string().optional(),REDIS_RATE_LIMIT_URL: z.string().optional(),diff --git a/apps/api/src/lib/index-cache-metrics.ts b/apps/api/src/lib/index-cache-metrics.tsindex 4de79a2b9..412ce643f 100644--- a/apps/api/src/lib/index-cache-metrics.ts+++ b/apps/api/src/lib/index-cache-metrics.ts@@ -12,6 +12,12 @@ export const indexCacheReadDuration = new Histogram({buckets: [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.15, 0.25, 0.5],});+export const indexPdfSupersedeCounter = new Counter({+ name: "firecrawl_index_pdf_supersede_total",+ help: "URL index PDF hits checked against the fire-pdf cache service: superseded (a newer entry exists, index copy skipped), current, or error (check failed, index copy served)",+ labelNames: ["result"],+});+export const indexCacheErrorCounter = new Counter({name: "firecrawl_index_cache_errors_total",help: "Index cache (Dragonfly) operation errors",diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.tsindex 15c065291..053a76eae 100644--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts@@ -29,7 +29,14 @@ import {useIndexNegativeCache,type IndexCacheEntry,} from "../../../../services/index-cache";-import { indexLookupCounter } from "../../../../lib/index-cache-metrics";+import {+ indexLookupCounter,+ indexPdfSupersedeCounter,+} from "../../../../lib/index-cache-metrics";+import {+ cacheServiceConfigured,+ checkFirePdfSupersedes,+} from "../pdf/fire-pdf/cache-service";import {AgentIndexOnlyError,EngineError,@@ -135,6 +142,10 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {contentType: document.metadata.contentType,postprocessorsUsed: document.metadata.postprocessorsUsed,proxyUsed: document.metadata.proxyUsed,+ ...(meta.firePdfCacheKey?.current !== undefined &&+ document.metadata.contentType === "application/pdf"+ ? { firePdfCacheKey: meta.firePdfCacheKey.current }+ : {}),});} catch (error) {meta.logger.error("Failed to save document to index", {@@ -577,6 +588,33 @@ export async function scrapeURLWithIndex(throw new IndexMissError();}+ // fire-pdf may have re-parsed this PDF since it was indexed (a refresh or a+ // campaign). Only once a fresh entry exists is the index copy skipped, so+ // the request lands on that entry without a parse and is re-indexed.+ if (+ config.INDEX_PDF_SUPERSEDE_CHECK &&+ typeof doc.firePdfCacheKey === "string" &&+ doc.contentType === "application/pdf" &&+ !isCachedPdfBase64 &&+ cacheServiceConfigured()+ ) {+ const result = await checkFirePdfSupersedes(+ meta,+ doc.firePdfCacheKey,+ new Date(selectedRow.created_at),+ );+ indexPdfSupersedeCounter.inc({ result });+ if (result === "superseded") {+ // The DB path may have just written this id through to Dragonfly.+ deleteCachedIndexEntry(variantKey, id, meta.logger).catch(() => {});+ logLookup("debug", "hit", {+ pdfSuperseded: true,+ indexDocumentId: id,+ });+ throw new IndexMissError();+ }+ }+logLookup("debug", "hit", {age: Date.now() - new Date(selectedRow.created_at).getTime(),status: selectedRow.status,diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFSupersede.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFSupersede.test.tsnew file mode 100644index 000000000..15d09d567--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFSupersede.test.ts@@ -0,0 +1,173 @@+import crypto from "crypto";+import { robustFetch } from "../../../lib/fetch";+import {+ checkFirePdfSupersedes,+ lookupCachedResult,+} from "../fire-pdf/cache-service";+import { config } from "../../../../../config";++vi.mock("../../../lib/fetch", () => ({ robustFetch: vi.fn() }));+vi.mock("../fire-pdf/refresh-budget", () => ({+ recordRefreshDecision: vi.fn(),+}));++const fetchMock = vi.mocked(robustFetch);++function meta() {+ return {+ id: "supersede-test",+ logger: { info: vi.fn(), warn: vi.fn() },+ internalOptions: { zeroDataRetention: false, teamId: "team-1" },+ mock: null,+ firePdfCacheKey: {},+ abort: {+ asSignal: () => new AbortController().signal,+ throwIfAborted: vi.fn(),+ },+ } as any;+}++const answerWith = (fixture: unknown) =>+ fetchMock.mockImplementationOnce(async ({ schema }: any) =>+ schema.parse(fixture),+ );++const rawKey = `raw-${"a".repeat(64)}`;+const indexedAt = new Date("2026-10-01T00:00:00Z");++describe("checkFirePdfSupersedes", () => {+ beforeEach(() => {+ config.FIRE_PDF_CACHE_BASE_URL = "http://cache";+ fetchMock.mockReset();+ });++ afterEach(() => {+ config.FIRE_PDF_CACHE_BASE_URL = undefined;+ });++ it("asks by key alone, without bytes", async () => {+ answerWith({ outcome: "miss", reason: "not_found" });+ await checkFirePdfSupersedes(meta(), rawKey, indexedAt);+ expect(fetchMock).toHaveBeenCalledTimes(1);+ const call = fetchMock.mock.calls[0][0];+ expect(call.url).toBe("http://cache/cache/lookup");+ expect(call.body).toEqual({+ keys: [rawKey],+ options: {},+ team_id: "team-1",+ kind: "scrape",+ source_kind: "pdf",+ scrape_id: "supersede-test",+ });+ expect(call.tryCount).toBe(1);+ });++ it("bounds the check at one second", async () => {+ const timeout = vi.spyOn(AbortSignal, "timeout");+ answerWith({ outcome: "miss", reason: "not_found" });+ await checkFirePdfSupersedes(meta(), rawKey, indexedAt);+ expect(timeout).toHaveBeenCalledWith(1_000);+ timeout.mockRestore();+ });++ it("is superseded by a fresh entry written well after the index row", async () => {+ answerWith({ outcome: "hit", cached_at: "2026-10-05T00:00:00Z" });+ expect(await checkFirePdfSupersedes(meta(), rawKey, indexedAt)).toBe(+ "superseded",+ );+ });++ it("treats an entry written with the index row as the same parse", async () => {+ answerWith({ outcome: "hit", cached_at: "2026-10-01T00:05:00Z" });+ expect(await checkFirePdfSupersedes(meta(), rawKey, indexedAt)).toBe(+ "current",+ );+ });++ it("treats an older entry as current", async () => {+ answerWith({ outcome: "hit", cached_at: "2026-09-01T00:00:00Z" });+ expect(await checkFirePdfSupersedes(meta(), rawKey, indexedAt)).toBe(+ "current",+ );+ });++ it.each([+ ["a miss", { outcome: "miss", reason: "campaign" }],+ [+ "a stale entry",+ { outcome: "stale", cached_at: "2026-10-05T00:00:00Z", campaign: "c" },+ ],+ ["a hit without cached_at", { outcome: "hit" }],+ ])("keeps the index copy on %s", async (_name, fixture) => {+ answerWith(fixture);+ expect(await checkFirePdfSupersedes(meta(), rawKey, indexedAt)).toBe(+ "current",+ );+ });++ it("reports a failed check as error", async () => {+ fetchMock.mockRejectedValueOnce(new Error("timeout"));+ expect(await checkFirePdfSupersedes(meta(), rawKey, indexedAt)).toBe(+ "error",+ );+ });+});++describe("lookupCachedResult records the content key", () => {+ beforeEach(() => {+ config.FIRE_PDF_CACHE_BASE_URL = "http://cache";+ fetchMock.mockReset();+ });++ afterEach(() => {+ config.FIRE_PDF_CACHE_BASE_URL = undefined;+ });++ const args = {+ mode: undefined,+ pagesProcessed: undefined,+ includePageMarkdown: false,+ includeBlocks: false,+ pageMarkers: false,+ refresh: false,+ sourceKind: "pdf" as const,+ ownVariant: "base",+ };++ it("stores the raw key of inline bytes on meta", async () => {+ answerWith({ outcome: "miss", reason: "not_found" });+ const base64 = Buffer.from("%PDF-1.7 key test").toString("base64");+ const m = meta();+ await lookupCachedResult(m, base64, args);+ const expected = `raw-${crypto+ .createHash("sha256")+ .update(Buffer.from(base64, "base64"))+ .digest("hex")}`;+ expect(m.firePdfCacheKey.current).toBe(expected);+ });++ it("stores a by-reference raw key on meta", async () => {+ answerWith({ outcome: "miss", reason: "not_found" });+ const m = meta();+ await lookupCachedResult(m, { key: rawKey }, args);+ expect(m.firePdfCacheKey.current).toBe(rawKey);+ });++ it("does not record a key for a raster image", async () => {+ answerWith({ outcome: "miss", reason: "not_found" });+ const m = meta();+ await lookupCachedResult(m, Buffer.from("png").toString("base64"), {+ ...args,+ sourceKind: "image",+ });+ expect(m.firePdfCacheKey.current).toBeUndefined();+ });++ it("records the key on the holder shared by engine copies of meta", async () => {+ answerWith({ outcome: "miss", reason: "not_found" });+ const outer = meta();+ const engineCopy = { ...outer };+ await lookupCachedResult(engineCopy, { key: rawKey }, args);+ expect(outer.firePdfCacheKey.current).toBe(rawKey);+ });+});diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPdfSupersede.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPdfSupersede.test.tsnew file mode 100644index 000000000..c102ec137--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPdfSupersede.test.ts@@ -0,0 +1,191 @@+vi.mock("../../../../../services", async importOriginal => ({+ ...(await importOriginal<typeof import("../../../../../services")>()),+ getIndexFromGCS: vi.fn(),+}));+vi.mock("../../../../../db/rpc", () => ({+ queryMaxAge: vi.fn(),+ indexGetRecent5: vi.fn(),+}));+vi.mock("../../../../../services/index-cache", async importOriginal => ({+ ...(await importOriginal<+ typeof import("../../../../../services/index-cache")+ >()),+ useIndexCache: true,+ useIndexNegativeCache: false,+ getCachedIndexEntries: vi.fn(async () => ({ status: "miss" })),+ upsertCachedIndexEntries: vi.fn(async () => {}),+ deleteCachedIndexEntry: vi.fn(async () => {}),+ getCachedMaxAge: vi.fn(async () => null),+ setCachedMaxAge: vi.fn(async () => {}),+}));+vi.mock("../fire-pdf/cache-service", () => ({+ cacheServiceConfigured: vi.fn(() => true),+ checkFirePdfSupersedes: vi.fn(),+}));++import { getIndexFromGCS } from "../../../../../services";+import {+ deleteCachedIndexEntry,+ getCachedIndexEntries,+} from "../../../../../services/index-cache";+import { indexGetRecent5 } from "../../../../../db/rpc";+import { checkFirePdfSupersedes } from "../fire-pdf/cache-service";+import { scrapeURLWithIndex } from "../../index/index";+import { IndexMissError } from "../../../error";+import { config } from "../../../../../config";++type MutableConfig = { INDEX_PDF_SUPERSEDE_CHECK: boolean };+const mutable = config as unknown as MutableConfig;++const createdAt = "2026-10-01T00:00:00.000Z";+const rawKey = `raw-${"b".repeat(64)}`;++function meta() {+ return {+ id: "index-supersede",+ url: "https://example.com/doc.pdf",+ logger: {+ debug: vi.fn(),+ info: vi.fn(),+ warn: vi.fn(),+ error: vi.fn(),+ child: () => ({ info: vi.fn(), warn: vi.fn(), error: vi.fn() }),+ },+ internalOptions: { teamId: "team-1" },+ featureFlags: new Set(),+ options: {+ maxAge: 7 * 24 * 60 * 60 * 1000,+ mobile: false,+ blockAds: true,+ waitFor: 0,+ formats: ["markdown"],+ },+ imageOcrEnabled: async () => false,+ } as any;+}++function indexedPdf(extra: Record<string, unknown> = {}) {+ return {+ url: "https://example.com/doc.pdf",+ html: "<p>old parse</p>",+ statusCode: 200,+ contentType: "application/pdf",+ pdfMetadata: { numPages: 2 },+ ...extra,+ };+}++describe("URL index PDF supersede check", () => {+ let original: boolean;++ beforeEach(() => {+ original = mutable.INDEX_PDF_SUPERSEDE_CHECK;+ vi.mocked(indexGetRecent5).mockResolvedValue([+ {+ id: "row-1",+ created_at: createdAt,+ status: 200,+ has_screenshot: false,+ has_screenshot_fullscreen: false,+ wait_time_ms: null,+ },+ ] as any);+ vi.mocked(checkFirePdfSupersedes).mockReset();+ vi.mocked(deleteCachedIndexEntry).mockClear();+ vi.mocked(getCachedIndexEntries).mockResolvedValue({ status: "miss" });+ });++ afterEach(() => {+ mutable.INDEX_PDF_SUPERSEDE_CHECK = original;+ });++ it("serves the index copy without asking while the flag is off", async () => {+ mutable.INDEX_PDF_SUPERSEDE_CHECK = false;+ vi.mocked(getIndexFromGCS).mockResolvedValue(+ indexedPdf({ firePdfCacheKey: rawKey }),+ );+ const result = await scrapeURLWithIndex(meta());+ expect(result.html).toBe("<p>old parse</p>");+ expect(checkFirePdfSupersedes).not.toHaveBeenCalled();+ });++ it("misses when fire-pdf holds a newer entry", async () => {+ mutable.INDEX_PDF_SUPERSEDE_CHECK = true;+ vi.mocked(getIndexFromGCS).mockResolvedValue(+ indexedPdf({ firePdfCacheKey: rawKey }),+ );+ vi.mocked(checkFirePdfSupersedes).mockResolvedValue("superseded");+ await expect(scrapeURLWithIndex(meta())).rejects.toBeInstanceOf(+ IndexMissError,+ );+ expect(checkFirePdfSupersedes).toHaveBeenCalledWith(+ expect.anything(),+ rawKey,+ new Date(createdAt),+ );+ expect(deleteCachedIndexEntry).toHaveBeenCalledWith(+ expect.any(String),+ "row-1",+ expect.anything(),+ );+ });++ it("drops the Dragonfly entry after a superseded cache hit", async () => {+ mutable.INDEX_PDF_SUPERSEDE_CHECK = true;+ vi.mocked(getCachedIndexEntries).mockResolvedValue({+ status: "hit",+ entries: [+ {+ id: "row-cached",+ created_at: new Date().toISOString(),+ status: 200,+ has_screenshot: false,+ has_screenshot_fullscreen: false,+ wait_time_ms: null,+ },+ ],+ });+ vi.mocked(getIndexFromGCS).mockResolvedValue(+ indexedPdf({ firePdfCacheKey: rawKey }),+ );+ vi.mocked(checkFirePdfSupersedes).mockResolvedValue("superseded");+ await expect(scrapeURLWithIndex(meta())).rejects.toBeInstanceOf(+ IndexMissError,+ );+ expect(deleteCachedIndexEntry).toHaveBeenCalledWith(+ expect.any(String),+ "row-cached",+ expect.anything(),+ );+ });++ it("does not ask for a non-PDF index entry", async () => {+ mutable.INDEX_PDF_SUPERSEDE_CHECK = true;+ vi.mocked(getIndexFromGCS).mockResolvedValue(+ indexedPdf({ firePdfCacheKey: rawKey, contentType: "text/html" }),+ );+ await scrapeURLWithIndex(meta());+ expect(checkFirePdfSupersedes).not.toHaveBeenCalled();+ });++ it.each(["current", "error"] as const)(+ "serves the index copy when the check says %s",+ async result => {+ mutable.INDEX_PDF_SUPERSEDE_CHECK = true;+ vi.mocked(getIndexFromGCS).mockResolvedValue(+ indexedPdf({ firePdfCacheKey: rawKey }),+ );+ vi.mocked(checkFirePdfSupersedes).mockResolvedValue(result);+ const served = await scrapeURLWithIndex(meta());+ expect(served.html).toBe("<p>old parse</p>");+ },+ );++ it("does not ask for entries indexed without a content key", async () => {+ mutable.INDEX_PDF_SUPERSEDE_CHECK = true;+ vi.mocked(getIndexFromGCS).mockResolvedValue(indexedPdf());+ const served = await scrapeURLWithIndex(meta());+ expect(served.html).toBe("<p>old parse</p>");+ expect(checkFirePdfSupersedes).not.toHaveBeenCalled();+ });+});diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache-service.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache-service.tsindex 01a3b9bea..9d5076305 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache-service.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache-service.ts@@ -82,6 +82,70 @@ const lookupOutcomeSchema = z.discriminatedUnion("outcome", [}),]);+// An index entry and the content-cache entry it was built from are written+// moments apart; only an entry written well after the index row is a newer+// parse rather than the same one.+const SUPERSEDE_MARGIN_MS = 15 * 60 * 1000;+const SUPERSEDE_TIMEOUT_MS = 1_000;++const supersedeAnswerSchema = z.object({+ outcome: z.string(),+ cached_at: z.string().optional(),+});++/**+ * Whether the cache service holds a fresh (no campaign match) entry for+ * `key` written after `indexCreatedAt`. Asks by key alone, so no bytes are+ * fetched and nothing is parsed. Any failure is "error": the caller keeps+ * serving the index copy.+ */+export async function checkFirePdfSupersedes(+ meta: Meta,+ key: string,+ indexCreatedAt: Date,+): Promise<"superseded" | "current" | "error"> {+ try {+ const answer = await robustFetch({+ url: `${config.FIRE_PDF_CACHE_BASE_URL}/cache/lookup`,+ method: "POST",+ headers: config.FIRE_PDF_API_KEY+ ? { Authorization: `Bearer ${config.FIRE_PDF_API_KEY}` }+ : undefined,+ body: {+ keys: [key],+ options: {},+ team_id: meta.internalOptions.teamId,+ kind: firePdfRequestKind(meta),+ source_kind: "pdf",+ scrape_id: meta.id,+ },+ schema: supersedeAnswerSchema,+ logger: meta.logger,+ mock: meta.mock,+ tryCount: 1,+ abort: AbortSignal.any([+ meta.abort.asSignal(),+ AbortSignal.timeout(SUPERSEDE_TIMEOUT_MS),+ ]),+ });+ if (answer.outcome !== "hit" || answer.cached_at === undefined) {+ return "current";+ }+ const cachedAt = new Date(answer.cached_at).getTime();+ if (isNaN(cachedAt)) return "current";+ return cachedAt - indexCreatedAt.getTime() > SUPERSEDE_MARGIN_MS+ ? "superseded"+ : "current";+ } catch (error) {+ meta.abort.throwIfAborted();+ meta.logger.warn("FirePDF supersede check failed, serving index copy", {+ scrapeId: meta.id,+ error,+ });+ return "error";+ }+}+export async function lookupCachedResult(meta: Meta,input: PdfCacheKeyInput,@@ -120,6 +184,14 @@ async function lookupWithinSpan(} = args;// An already-cancelled scrape sends nothing.meta.abort.throwIfAborted();+ const keys = await cacheLookupKeys(input);+ if (+ sourceKind === "pdf" &&+ keys[0].startsWith("raw-") &&+ meta.firePdfCacheKey !== undefined+ ) {+ meta.firePdfCacheKey.current = keys[0];+ }let answer: z.infer<typeof lookupOutcomeSchema>;try {answer = await robustFetch({@@ -129,7 +201,7 @@ async function lookupWithinSpan(? { Authorization: `Bearer ${config.FIRE_PDF_API_KEY}` }: undefined,body: {- keys: await cacheLookupKeys(input),+ keys,options: {...(mode !== undefined && { mode }),...(includePageMarkdown && { include_page_markdown: true }),diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 257d20071..231bc0ed9 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -267,6 +267,11 @@ export type Meta = {| undefined; // undefined: no prefetch yet, null: prefetch came back emptycostTracking: CostTracking;winnerEngine?: Engine;+ /** The `raw-` content-cache key of the PDF this scrape looked up through+ * the fire-pdf cache service, stored with the URL index entry so a later+ * index hit can ask whether fire-pdf holds a newer parse. A holder shared+ * by reference, because engines run on a copy of meta. */+ firePdfCacheKey?: { current?: string };abortHandle?: NodeJS.Timeout;audioCookies?: BrowserCookie[];/** Threat protection decisions made during this scrape, in order (mutable, like logs). */@@ -582,6 +587,7 @@ async function buildMetaObject(imageOcrEnabled,costTracking,threatDecisions: [],+ firePdfCacheKey: {},largePdfProcessing: {},};}diff --git a/apps/api/src/services/index.ts b/apps/api/src/services/index.tsindex d36606da6..6050a13ab 100644--- a/apps/api/src/services/index.ts+++ b/apps/api/src/services/index.ts@@ -124,6 +124,7 @@ export async function saveIndexToGCS(contentType?: string;postprocessorsUsed?: string[];proxyUsed?: "basic" | "stealth";+ firePdfCacheKey?: string;},): Promise<void> {return await withSpan("firecrawl-index-save-to-gcs", async span => {