Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): let PDF refresh skip URL index reads (INDEX_SKIP_ON_PDF_REFRESH, default off) (#5292)
Abimael Martell85a40f9deb5e
diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex 0925b1348..4d555094a 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -264,6 +264,10 @@ const configSchema = z.object({// (e.g. 600000 = 10min) also short-circuits repeat lookups for URLs with no// index entry. Kept short so any missed cache-clear self-heals quickly.INDEX_CACHE_NEGATIVE_TTL_MS: z.coerce.number().default(0),+ // `parsers: [{ type: "pdf", refresh: true }]` skips URL index reads, so the+ // request reaches the PDF engine and its content-cache refresh budget+ // instead of being answered with the indexed copy of the old parse.+ INDEX_SKIP_ON_PDF_REFRESH: z.stringbool().default(false),// On a URL index hit for a PDF whose entry recorded its fire-pdf content// key, ask the fire-pdf cache service whether it holds a newer parse; if it// does, the index copy is treated as a miss so the request is answered fromdiff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex a2481ccb0..9f2d7cd3f 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -33,6 +33,7 @@ import {getPDFBlocks,getPDFPageMarkdown,getPDFPageMarkers,+ getPDFRefresh,} from "../../../controllers/v2/types";import type { PdfMetadata, PdfPageBlocks } from "./pdf/types";import { BrandingProfile } from "../../../types/branding";@@ -631,6 +632,11 @@ export function shouldUseIndex(meta: Meta) {!getPDFPageMarkers(meta.options.parsers) &&!hasCustomScreenshotSettings &&meta.options.maxAge !== 0 &&+ // A PDF refresh asks for a fresh parse; a URL index copy is the parse+ // being refreshed, so serving it would silently ignore the request.+ !(+ config.INDEX_SKIP_ON_PDF_REFRESH && getPDFRefresh(meta.options.parsers)+ ) &&!hasCustomRequestContext(meta.options));}@@ -778,10 +784,11 @@ export async function buildFallbackList(meta: Meta): Promise<if (getPDFPageMarkdown(meta.options.parsers) ||getPDFBlocks(meta.options.parsers) ||- getPDFPageMarkers(meta.options.parsers)+ getPDFPageMarkers(meta.options.parsers) ||+ (config.INDEX_SKIP_ON_PDF_REFRESH && getPDFRefresh(meta.options.parsers))) {meta.logger.warn(- "agentIndexOnly request demands pageMarkdown/blocks/pageMarkers, which the URL index cannot serve",+ "agentIndexOnly request demands pageMarkdown/blocks/pageMarkers or a PDF refresh, which the URL index cannot serve",{ parsers: meta.options.parsers },);throw new AgentIndexOnlyError();diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPdfRefresh.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPdfRefresh.test.tsnew file mode 100644index 000000000..0d54e118f--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPdfRefresh.test.ts@@ -0,0 +1,113 @@+vi.mock("../../../../../services", async importOriginal => ({+ ...(await importOriginal<typeof import("../../../../../services")>()),+ useIndex: true,+}));++import { config } from "../../../../../config";+import { buildFallbackList, shouldUseIndex } from "../../index";+import { AgentIndexOnlyError } from "../../../error";++type MutableConfig = {+ FIRECRAWL_INDEX_WRITE_ONLY?: boolean;+ INDEX_SKIP_ON_PDF_REFRESH: boolean;+};++function metaWith(parsers: unknown) {+ return {+ internalOptions: { isParse: false },+ featureFlags: new Set(),+ options: {+ formats: ["markdown"],+ maxAge: 60_000,+ storeInCache: true,+ parsers,+ },+ } as any;+}++describe("URL index and PDF refresh", () => {+ const mutable = config as unknown as MutableConfig;+ let originalWriteOnly: boolean | undefined;+ let originalSkip: boolean;++ beforeEach(() => {+ originalWriteOnly = mutable.FIRECRAWL_INDEX_WRITE_ONLY;+ originalSkip = mutable.INDEX_SKIP_ON_PDF_REFRESH;+ mutable.FIRECRAWL_INDEX_WRITE_ONLY = false;+ });++ afterEach(() => {+ mutable.FIRECRAWL_INDEX_WRITE_ONLY = originalWriteOnly;+ mutable.INDEX_SKIP_ON_PDF_REFRESH = originalSkip;+ });++ it("keeps reading the index for a refresh while the flag is off", () => {+ mutable.INDEX_SKIP_ON_PDF_REFRESH = false;+ expect(shouldUseIndex(metaWith([{ type: "pdf", refresh: true }]))).toBe(+ true,+ );+ });++ it("skips the index for a refresh when the flag is on", () => {+ mutable.INDEX_SKIP_ON_PDF_REFRESH = true;+ expect(shouldUseIndex(metaWith([{ type: "pdf", refresh: true }]))).toBe(+ false,+ );+ });++ it.each([+ ["no parsers", undefined],+ ["pdf parser without refresh", [{ type: "pdf" }]],+ ["refresh false", [{ type: "pdf", refresh: false }]],+ ])("keeps reading the index with the flag on for %s", (_name, parsers) => {+ mutable.INDEX_SKIP_ON_PDF_REFRESH = true;+ expect(shouldUseIndex(metaWith(parsers))).toBe(true);+ });+});++describe("agentIndexOnly and PDF refresh", () => {+ const mutable = config as unknown as MutableConfig;+ let originalSkip: boolean;++ beforeEach(() => {+ originalSkip = mutable.INDEX_SKIP_ON_PDF_REFRESH;+ });++ afterEach(() => {+ mutable.INDEX_SKIP_ON_PDF_REFRESH = originalSkip;+ });++ function agentMeta(parsers: unknown) {+ return {+ id: "agent-refresh",+ url: "https://example.com/doc.pdf",+ options: { formats: ["markdown"], maxAge: 60_000, parsers },+ internalOptions: { teamId: "team-1", agentIndexOnly: true },+ featureFlags: new Set(),+ mock: null,+ logger: {+ info: vi.fn(),+ warn: vi.fn(),+ debug: vi.fn(),+ error: vi.fn(),+ child: vi.fn().mockReturnThis(),+ },+ } as any;+ }++ it("refuses an index-only refresh when the flag is on", async () => {+ mutable.INDEX_SKIP_ON_PDF_REFRESH = true;+ await expect(+ buildFallbackList(agentMeta([{ type: "pdf", refresh: true }])),+ ).rejects.toBeInstanceOf(AgentIndexOnlyError);+ });++ it("keeps index-only refresh requests on the index while the flag is off", async () => {+ mutable.INDEX_SKIP_ON_PDF_REFRESH = false;+ const list = await buildFallbackList(+ agentMeta([{ type: "pdf", refresh: true }]),+ );+ expect(list.length).toBeGreaterThan(0);+ expect(list.every(x => x.engine.startsWith("index"))).toBe(true);+ });+});