Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): let PDF refresh skip URL index reads (INDEX_SKIP_ON_PDF_REFRESH, default off) (#5292)

Abimael Martell85a40f9deb5e
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index 0925b1348..4d555094a 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -264,6 +264,10 @@ const configSchema = z.object({
// (e.g. 600000 = 10min) also short-circuits repeat lookups for URLs with no
// index entry. Kept short so any missed cache-clear self-heals quickly.
INDEX_CACHE_NEGATIVE_TTL_MS: z.coerce.number().default(0),
+ // `parsers: [{ type: "pdf", refresh: true }]` skips URL index reads, so the
+ // request reaches the PDF engine and its content-cache refresh budget
+ // instead of being answered with the indexed copy of the old parse.
+ INDEX_SKIP_ON_PDF_REFRESH: z.stringbool().default(false),
// On a URL index hit for a PDF whose entry recorded its fire-pdf content
// key, ask the fire-pdf cache service whether it holds a newer parse; if it
// does, the index copy is treated as a miss so the request is answered from
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.ts
index a2481ccb0..9f2d7cd3f 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts
@@ -33,6 +33,7 @@ import {
getPDFBlocks,
getPDFPageMarkdown,
getPDFPageMarkers,
+ getPDFRefresh,
} from "../../../controllers/v2/types";
import type { PdfMetadata, PdfPageBlocks } from "./pdf/types";
import { BrandingProfile } from "../../../types/branding";
@@ -631,6 +632,11 @@ export function shouldUseIndex(meta: Meta) {
!getPDFPageMarkers(meta.options.parsers) &&
!hasCustomScreenshotSettings &&
meta.options.maxAge !== 0 &&
+ // A PDF refresh asks for a fresh parse; a URL index copy is the parse
+ // being refreshed, so serving it would silently ignore the request.
+ !(
+ config.INDEX_SKIP_ON_PDF_REFRESH && getPDFRefresh(meta.options.parsers)
+ ) &&
!hasCustomRequestContext(meta.options)
);
}
@@ -778,10 +784,11 @@ export async function buildFallbackList(meta: Meta): Promise<
if (
getPDFPageMarkdown(meta.options.parsers) ||
getPDFBlocks(meta.options.parsers) ||
- getPDFPageMarkers(meta.options.parsers)
+ getPDFPageMarkers(meta.options.parsers) ||
+ (config.INDEX_SKIP_ON_PDF_REFRESH && getPDFRefresh(meta.options.parsers))
) {
meta.logger.warn(
- "agentIndexOnly request demands pageMarkdown/blocks/pageMarkers, which the URL index cannot serve",
+ "agentIndexOnly request demands pageMarkdown/blocks/pageMarkers or a PDF refresh, which the URL index cannot serve",
{ parsers: meta.options.parsers },
);
throw new AgentIndexOnlyError();
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPdfRefresh.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPdfRefresh.test.ts
new file mode 100644
index 000000000..0d54e118f
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPdfRefresh.test.ts
@@ -0,0 +1,113 @@
+vi.mock("../../../../../services", async importOriginal => ({
+ ...(await importOriginal<typeof import("../../../../../services")>()),
+ useIndex: true,
+}));
+
+import { config } from "../../../../../config";
+import { buildFallbackList, shouldUseIndex } from "../../index";
+import { AgentIndexOnlyError } from "../../../error";
+
+type MutableConfig = {
+ FIRECRAWL_INDEX_WRITE_ONLY?: boolean;
+ INDEX_SKIP_ON_PDF_REFRESH: boolean;
+};
+
+function metaWith(parsers: unknown) {
+ return {
+ internalOptions: { isParse: false },
+ featureFlags: new Set(),
+ options: {
+ formats: ["markdown"],
+ maxAge: 60_000,
+ storeInCache: true,
+ parsers,
+ },
+ } as any;
+}
+
+describe("URL index and PDF refresh", () => {
+ const mutable = config as unknown as MutableConfig;
+ let originalWriteOnly: boolean | undefined;
+ let originalSkip: boolean;
+
+ beforeEach(() => {
+ originalWriteOnly = mutable.FIRECRAWL_INDEX_WRITE_ONLY;
+ originalSkip = mutable.INDEX_SKIP_ON_PDF_REFRESH;
+ mutable.FIRECRAWL_INDEX_WRITE_ONLY = false;
+ });
+
+ afterEach(() => {
+ mutable.FIRECRAWL_INDEX_WRITE_ONLY = originalWriteOnly;
+ mutable.INDEX_SKIP_ON_PDF_REFRESH = originalSkip;
+ });
+
+ it("keeps reading the index for a refresh while the flag is off", () => {
+ mutable.INDEX_SKIP_ON_PDF_REFRESH = false;
+ expect(shouldUseIndex(metaWith([{ type: "pdf", refresh: true }]))).toBe(
+ true,
+ );
+ });
+
+ it("skips the index for a refresh when the flag is on", () => {
+ mutable.INDEX_SKIP_ON_PDF_REFRESH = true;
+ expect(shouldUseIndex(metaWith([{ type: "pdf", refresh: true }]))).toBe(
+ false,
+ );
+ });
+
+ it.each([
+ ["no parsers", undefined],
+ ["pdf parser without refresh", [{ type: "pdf" }]],
+ ["refresh false", [{ type: "pdf", refresh: false }]],
+ ])("keeps reading the index with the flag on for %s", (_name, parsers) => {
+ mutable.INDEX_SKIP_ON_PDF_REFRESH = true;
+ expect(shouldUseIndex(metaWith(parsers))).toBe(true);
+ });
+});
+
+describe("agentIndexOnly and PDF refresh", () => {
+ const mutable = config as unknown as MutableConfig;
+ let originalSkip: boolean;
+
+ beforeEach(() => {
+ originalSkip = mutable.INDEX_SKIP_ON_PDF_REFRESH;
+ });
+
+ afterEach(() => {
+ mutable.INDEX_SKIP_ON_PDF_REFRESH = originalSkip;
+ });
+
+ function agentMeta(parsers: unknown) {
+ return {
+ id: "agent-refresh",
+ url: "https://example.com/doc.pdf",
+ options: { formats: ["markdown"], maxAge: 60_000, parsers },
+ internalOptions: { teamId: "team-1", agentIndexOnly: true },
+ featureFlags: new Set(),
+ mock: null,
+ logger: {
+ info: vi.fn(),
+ warn: vi.fn(),
+ debug: vi.fn(),
+ error: vi.fn(),
+ child: vi.fn().mockReturnThis(),
+ },
+ } as any;
+ }
+
+ it("refuses an index-only refresh when the flag is on", async () => {
+ mutable.INDEX_SKIP_ON_PDF_REFRESH = true;
+ await expect(
+ buildFallbackList(agentMeta([{ type: "pdf", refresh: true }])),
+ ).rejects.toBeInstanceOf(AgentIndexOnlyError);
+ });
+
+ it("keeps index-only refresh requests on the index while the flag is off", async () => {
+ mutable.INDEX_SKIP_ON_PDF_REFRESH = false;
+ const list = await buildFallbackList(
+ agentMeta([{ type: "pdf", refresh: true }]),
+ );
+ expect(list.length).toBeGreaterThan(0);
+ expect(list.every(x => x.engine.startsWith("index"))).toBe(true);
+ });
+});