Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): Dragonfly cache for index URL->id lookups + canonical lookup log (#3770)
Gergő Móricz0f91869af58e
diff --git a/apps/api/src/__tests__/snips/index-cache.test.ts b/apps/api/src/__tests__/snips/index-cache.test.tsnew file mode 100644index 000000000..12e0496e8--- /dev/null+++ b/apps/api/src/__tests__/snips/index-cache.test.ts@@ -0,0 +1,230 @@+import IORedis from "ioredis";+import {+ deriveIndexVariantKey,+ filterIndexEntries,+ getCachedIndexEntries,+ getCachedMaxAge,+ upsertCachedIndexEntries,+ type IndexCacheEntry,+} from "../../services/index-cache";+import { hashURL } from "../../services";++const urlHash = hashURL("https://example.com/page");++const baseKeyParams = {+ urlHash,+ isMobile: false,+ blockAds: true,+ isStealth: false,+ locationCountry: null,+ locationLanguages: null,+};++function entry(overrides: Partial<IndexCacheEntry> = {}): IndexCacheEntry {+ return {+ id: crypto.randomUUID(),+ created_at: new Date().toISOString(),+ status: 200,+ has_screenshot: false,+ has_screenshot_fullscreen: false,+ wait_time_ms: null,+ ...overrides,+ };+}++describe("deriveIndexVariantKey", () => {+ it("is stable for identical params", () => {+ expect(deriveIndexVariantKey(baseKeyParams)).toBe(+ deriveIndexVariantKey({ ...baseKeyParams }),+ );+ });++ it("changes when any exact-match dimension changes", () => {+ const base = deriveIndexVariantKey(baseKeyParams);+ expect(deriveIndexVariantKey({ ...baseKeyParams, isMobile: true })).not.toBe(+ base,+ );+ expect(deriveIndexVariantKey({ ...baseKeyParams, blockAds: false })).not.toBe(+ base,+ );+ expect(+ deriveIndexVariantKey({ ...baseKeyParams, isStealth: true }),+ ).not.toBe(base);+ expect(+ deriveIndexVariantKey({ ...baseKeyParams, locationCountry: "DE" }),+ ).not.toBe(base);+ expect(+ deriveIndexVariantKey({+ ...baseKeyParams,+ urlHash: hashURL("https://example.com/other"),+ }),+ ).not.toBe(base);+ });++ it("treats languages as a set (order and duplicates ignored), matching the SQL @>/<@ comparison", () => {+ const a = deriveIndexVariantKey({+ ...baseKeyParams,+ locationLanguages: ["en", "de"],+ });+ const b = deriveIndexVariantKey({+ ...baseKeyParams,+ locationLanguages: ["de", "en", "de"],+ });+ expect(a).toBe(b);+ });++ it("keeps empty language array distinct from null, matching the SQL IS NULL check", () => {+ const empty = deriveIndexVariantKey({+ ...baseKeyParams,+ locationLanguages: [],+ });+ const nul = deriveIndexVariantKey({+ ...baseKeyParams,+ locationLanguages: null,+ });+ expect(empty).not.toBe(nul);+ });+});++describe("filterIndexEntries", () => {+ const now = Date.now();+ const hour = 60 * 60 * 1000;++ it("applies the maxAge window", () => {+ const fresh = entry({ created_at: new Date(now - hour).toISOString() });+ const stale = entry({ created_at: new Date(now - 5 * hour).toISOString() });+ const result = filterIndexEntries([fresh, stale], {+ maxAgeMs: 2 * hour,+ minAgeMs: null,+ needsScreenshot: false,+ needsScreenshotFullscreen: false,+ waitTimeMs: 0,+ now,+ });+ expect(result.map(x => x.id)).toEqual([fresh.id]);+ });++ it("applies the minAge window", () => {+ const tooFresh = entry({ created_at: new Date(now - hour).toISOString() });+ const oldEnough = entry({+ created_at: new Date(now - 5 * hour).toISOString(),+ });+ const result = filterIndexEntries([tooFresh, oldEnough], {+ maxAgeMs: 24 * hour,+ minAgeMs: 2 * hour,+ needsScreenshot: false,+ needsScreenshotFullscreen: false,+ waitTimeMs: 0,+ now,+ });+ expect(result.map(x => x.id)).toEqual([oldEnough.id]);+ });++ it("matches screenshot capability like the SQL (request-false matches anything, request-true requires it)", () => {+ const withShot = entry({ has_screenshot: true });+ const withoutShot = entry({ has_screenshot: false });+ const opts = {+ maxAgeMs: hour,+ minAgeMs: null,+ needsScreenshotFullscreen: false,+ waitTimeMs: 0,+ now,+ };+ expect(+ filterIndexEntries([withShot, withoutShot], {+ ...opts,+ needsScreenshot: false,+ }),+ ).toHaveLength(2);+ expect(+ filterIndexEntries([withShot, withoutShot], {+ ...opts,+ needsScreenshot: true,+ }).map(x => x.id),+ ).toEqual([withShot.id]);+ });++ it("compares waitFor like the SQL (COALESCE(entry, 0) >= requested)", () => {+ const noWait = entry({ wait_time_ms: null });+ const shortWait = entry({ wait_time_ms: 500 });+ const longWait = entry({ wait_time_ms: 5000 });+ const opts = {+ maxAgeMs: hour,+ minAgeMs: null,+ needsScreenshot: false,+ needsScreenshotFullscreen: false,+ now,+ };+ expect(+ filterIndexEntries([noWait, shortWait, longWait], {+ ...opts,+ waitTimeMs: 0,+ }),+ ).toHaveLength(3);+ expect(+ filterIndexEntries([noWait, shortWait, longWait], {+ ...opts,+ waitTimeMs: 1000,+ }).map(x => x.id),+ ).toEqual([longWait.id]);+ });++ it("sorts newest-first and caps at 5 like the SQL LIMIT", () => {+ const entries = Array.from({ length: 8 }, (_, i) =>+ entry({ created_at: new Date(now - i * 1000).toISOString() }),+ );+ const shuffled = [...entries].reverse();+ const result = filterIndexEntries(shuffled, {+ maxAgeMs: hour,+ minAgeMs: null,+ needsScreenshot: false,+ needsScreenshotFullscreen: false,+ waitTimeMs: 0,+ now,+ });+ expect(result).toHaveLength(5);+ expect(result.map(x => x.id)).toEqual(entries.slice(0, 5).map(x => x.id));+ });+});++describe("index cache fail-open", () => {+ // Points at a port nothing listens on: every operation must resolve (not+ // throw, not hang) so the read path can fall back to Postgres.+ let deadClient: IORedis;++ beforeAll(() => {+ deadClient = new IORedis("redis://127.0.0.1:1", {+ enableOfflineQueue: false,+ maxRetriesPerRequest: 1,+ retryStrategy: () => null,+ lazyConnect: true,+ });+ deadClient.on("error", () => {});+ deadClient.connect().catch(() => {});+ });++ afterAll(() => {+ deadClient.disconnect();+ });++ it("getCachedIndexEntries resolves to error status", async () => {+ const result = await getCachedIndexEntries(+ "idxc:test",+ undefined,+ deadClient,+ );+ expect(result.status).toBe("error");+ }, 5000);++ it("upsertCachedIndexEntries resolves without throwing", async () => {+ await expect(+ upsertCachedIndexEntries("idxc:test", [entry()], undefined, deadClient),+ ).resolves.toBeUndefined();+ }, 5000);++ it("getCachedMaxAge resolves to null", async () => {+ await expect(+ getCachedMaxAge(urlHash, undefined, deadClient),+ ).resolves.toBeNull();+ }, 5000);+});diff --git a/apps/api/src/__tests__/snips/v2/index-cache-variants.test.ts b/apps/api/src/__tests__/snips/v2/index-cache-variants.test.tsnew file mode 100644index 000000000..8a779baea--- /dev/null+++ b/apps/api/src/__tests__/snips/v2/index-cache-variants.test.ts@@ -0,0 +1,79 @@+import { describeIf, TEST_PRODUCTION, indexCooldown } from "../lib";+import { Identity, idmux, scrapeTimeout, scrape } from "./lib";+import crypto from "crypto";++// E2E coverage for the index URL->id lookup variant semantics, which the+// Dragonfly index cache (services/index-cache.ts) replicates client-side from+// index_get_recent_5. Behavior must be identical whether INDEX_CACHE_REDIS_URL+// is set or not.+describeIf(TEST_PRODUCTION)("V2 index lookup variant matching", () => {+ let identity: Identity;++ beforeAll(async () => {+ identity = await idmux({+ name: "v2-index-cache-variants",+ concurrency: 100,+ credits: 1000000,+ });+ }, 10000);++ test(+ "screenshot-capable index entry serves a request without screenshot",+ async () => {+ const id = crypto.randomUUID();+ const url = "https://firecrawl.dev/?testId=" + id;++ const data1 = await scrape(+ {+ url,+ formats: ["markdown", "screenshot"],+ },+ identity,+ );+ expect(data1.metadata.cacheState).toBe("miss");++ await new Promise(resolve => setTimeout(resolve, indexCooldown));++ // A request that doesn't need a screenshot matches entries with or+ // without one (p_feature_screenshot IS NOT TRUE OR has_screenshot).+ const data2 = await scrape(+ {+ url,+ maxAge: 10 * 60 * 1000,+ },+ identity,+ );+ expect(data2.metadata.cacheState).toBe("hit");+ },+ scrapeTimeout * 2 + indexCooldown,+ );++ test(+ "screenshot request does not get served by a screenshotless index entry",+ async () => {+ const id = crypto.randomUUID();+ const url = "https://firecrawl.dev/?testId=" + id;++ const data1 = await scrape(+ {+ url,+ },+ identity,+ );+ expect(data1.metadata.cacheState).toBe("miss");++ await new Promise(resolve => setTimeout(resolve, indexCooldown));++ const data2 = await scrape(+ {+ url,+ formats: ["markdown", "screenshot"],+ maxAge: 10 * 60 * 1000,+ },+ identity,+ );+ expect(data2.metadata.cacheState).toBe("miss");+ },+ scrapeTimeout * 2 + indexCooldown,+ );+});diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex 3b6132985..c3e1872c0 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -84,6 +84,7 @@ const configSchema = z.object({DATABASE_URL: z.string().optional(),DATABASE_REPLICA_URL: z.string().optional(),INDEX_DATABASE_URL: z.string().optional(),+ INDEX_CACHE_REDIS_URL: z.string().optional(),REDIS_URL: z.string().optional(),REDIS_EVICT_URL: z.string().optional(),REDIS_RATE_LIMIT_URL: z.string().optional(),diff --git a/apps/api/src/db/rpc.ts b/apps/api/src/db/rpc.tsindex 42b849896..6c0e9e8d3 100644--- a/apps/api/src/db/rpc.ts+++ b/apps/api/src/db/rpc.ts@@ -258,7 +258,19 @@ export function queryMaxAge();}-export function indexGetRecent4(params: {+type IndexGetRecentRow = {+ id: string;+ created_at: string;+ status: number;+ has_screenshot: boolean;+ has_screenshot_fullscreen: boolean;+ wait_time_ms: number | null;+};++// Same filters as index_get_recent_4, but also returns the per-entry+// capability columns so results can populate the Dragonfly index cache+// (services/index-cache.ts).+export async function indexGetRecent5(params: {url_hash: Buffer;max_age_ms: number;is_mobile: boolean;@@ -270,11 +282,15 @@ export function indexGetRecent4(params: {wait_time_ms: number;is_stealth: boolean;min_age_ms: number | null;-}): Promise<{ id: string; created_at: string; status: number }[]> {- return execRows(+}): Promise<IndexGetRecentRow[]> {+ const rows = await execRows<IndexGetRecentRow>(dbIndex,- sql`select * from index_get_recent_4(p_url_hash => ${params.url_hash}, p_max_age_ms => ${params.max_age_ms}, p_is_mobile => ${params.is_mobile}, p_block_ads => ${params.block_ads}, p_feature_screenshot => ${params.feature_screenshot}, p_feature_screenshot_fullscreen => ${params.feature_screenshot_fullscreen}, p_location_country => ${params.location_country}, p_location_languages => ${sql.param(params.location_languages)}::text[], p_wait_time_ms => ${params.wait_time_ms}, p_is_stealth => ${params.is_stealth}, p_min_age_ms => ${params.min_age_ms})`,+ sql`select * from index_get_recent_5(p_url_hash => ${params.url_hash}, p_max_age_ms => ${params.max_age_ms}, p_is_mobile => ${params.is_mobile}, p_block_ads => ${params.block_ads}, p_feature_screenshot => ${params.feature_screenshot}, p_feature_screenshot_fullscreen => ${params.feature_screenshot_fullscreen}, p_location_country => ${params.location_country}, p_location_languages => ${sql.param(params.location_languages)}::text[], p_wait_time_ms => ${params.wait_time_ms}, p_is_stealth => ${params.is_stealth}, p_min_age_ms => ${params.min_age_ms})`,);+ for (const row of rows) {+ row.wait_time_ms = toNum(row.wait_time_ms);+ }+ return rows;}export function queryTopUrlsForDomain<T = Record<string, any>>(diff --git a/apps/api/src/lib/index-cache-metrics.ts b/apps/api/src/lib/index-cache-metrics.tsnew file mode 100644index 000000000..4de79a2b9--- /dev/null+++ b/apps/api/src/lib/index-cache-metrics.ts@@ -0,0 +1,19 @@+import { Counter, Histogram } from "prom-client";++export const indexLookupCounter = new Counter({+ name: "firecrawl_index_lookup_total",+ help: "Index URL->id lookups by outcome (cache layer vs DB vs miss)",+ labelNames: ["outcome"],+});++export const indexCacheReadDuration = new Histogram({+ name: "firecrawl_index_cache_read_duration_seconds",+ help: "Duration of index cache (Dragonfly) reads",+ buckets: [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.15, 0.25, 0.5],+});++export const indexCacheErrorCounter = new Counter({+ name: "firecrawl_index_cache_errors_total",+ help: "Index cache (Dragonfly) operation errors",+ labelNames: ["op"],+});diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.tsindex 77ad5fc40..eb499ed67 100644--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts@@ -13,7 +13,19 @@ import {generateDomainSplits,addOMCEJob,} from "../../../../services";-import { queryMaxAge, indexGetRecent4 } from "../../../../db/rpc";+import { queryMaxAge, indexGetRecent5 } from "../../../../db/rpc";+import {+ deleteCachedIndexEntry,+ deriveIndexVariantKey,+ filterIndexEntries,+ getCachedIndexEntries,+ getCachedMaxAge,+ setCachedMaxAge,+ upsertCachedIndexEntries,+ useIndexCache,+ type IndexCacheEntry,+} from "../../../../services/index-cache";+import { indexLookupCounter } from "../../../../lib/index-cache-metrics";import {AgentIndexOnlyError,EngineError,@@ -215,9 +227,14 @@ export async function scrapeURLWithIndex(const normalizedURL = normalizeURLForIndex(meta.url);const urlHash = hashURL(normalizedURL);+ const defaultMaxAge = 2 * 24 * 60 * 60 * 1000; // 2 days++ type MaxAgeSource = "explicit" | "dynamic_cached" | "dynamic_db" | "default";let maxAge: number;+ let maxAgeSource: MaxAgeSource = "default";if (meta.options.maxAge !== undefined) {maxAge = meta.options.maxAge;+ maxAgeSource = "explicit";} else {const domainSplitsHash = generateDomainSplits(new URL(meta.url).hostname,@@ -229,63 +246,187 @@ export async function scrapeURLWithIndex(config.FIRECRAWL_INDEX_WRITE_ONLY ||config.USE_DB_AUTHENTICATION !== true) {- maxAge = 2 * 24 * 60 * 60 * 1000; // 2 days+ maxAge = defaultMaxAge;} else {try {- maxAge = await Promise.race([- (async () => {+ const resolved = await Promise.race([+ (async (): Promise<{+ value: number;+ source: MaxAgeSource;+ }> => {try {- const data = await queryMaxAge(domainSplitsHash[level]);- if (!data || data.length === 0) {- return 2 * 24 * 60 * 60 * 1000; // 2 days+ const domainHash = domainSplitsHash[level];+ if (useIndexCache) {+ const cached = await getCachedMaxAge(domainHash, meta.logger);+ if (cached !== null) {+ return {+ value: cached.maxAge ?? defaultMaxAge,+ source: "dynamic_cached",+ };+ }+ }+ const data = await queryMaxAge(domainHash);+ const value =+ !data || data.length === 0 ? null : (data[0].max_age ?? null);+ if (useIndexCache) {+ setCachedMaxAge(domainHash, value, meta.logger).catch(+ () => {},+ );}- return data[0].max_age ?? 2 * 24 * 60 * 60 * 1000; // 2 days+ return { value: value ?? defaultMaxAge, source: "dynamic_db" };} catch (error) {meta.logger.warn("Failed to get max age from DB", { error });- return 2 * 24 * 60 * 60 * 1000; // 2 days+ return { value: defaultMaxAge, source: "default" };}})(),- new Promise<number>(resolve =>- setTimeout(() => {- resolve(2 * 24 * 60 * 60 * 1000); // 2 days- }, 200),+ new Promise<{ value: number; source: MaxAgeSource }>(+ resolve =>+ setTimeout(() => {+ resolve({ value: defaultMaxAge, source: "default" });+ }, 200),),]);+ maxAge = resolved.value;+ maxAgeSource = resolved.source;} catch (e) {meta.logger.warn("Failed to get max age from DB", {error: e,});- maxAge = 2 * 24 * 60 * 60 * 1000; // 2 days+ maxAge = defaultMaxAge;}}}const checkpoint1 = Date.now();- let data: { id: string; created_at: string; status: number }[];- try {- data = await indexGetRecent4({- url_hash: urlHash,- max_age_ms: maxAge,- is_mobile: meta.options.mobile,- block_ads: meta.options.blockAds,- feature_screenshot: meta.featureFlags.has("screenshot"),- feature_screenshot_fullscreen: meta.featureFlags.has(- "screenshot@fullScreen",- ),- location_country: meta.options.location?.country ?? null,- location_languages:- (meta.options.location?.languages?.length ?? 0) > 0- ? (meta.options.location?.languages ?? null)- : null,- wait_time_ms: meta.options.waitFor,- is_stealth: meta.featureFlags.has("stealthProxy"),- min_age_ms: meta.options.minAge ?? null,- });- } catch (error) {- throw new EngineError("Failed to retrieve URL from DB index", {- cause: error,+ const variantKey = deriveIndexVariantKey({+ urlHash,+ isMobile: meta.options.mobile,+ blockAds: meta.options.blockAds,+ isStealth: meta.featureFlags.has("stealthProxy"),+ locationCountry: meta.options.location?.country ?? null,+ locationLanguages:+ (meta.options.location?.languages?.length ?? 0) > 0+ ? (meta.options.location?.languages ?? null)+ : null,+ });++ let cacheStatus: "hit" | "filtered" | "miss" | "error" | "disabled" =+ "disabled";+ let servedFromCache = false;+ let timingsCache = 0;+ let timingsDb = 0;++ // Canonical log for every index URL->id lookup. debug = normal op, warn =+ // recovered weird op (cache failure with DB fallback, GCS self-heal),+ // error = failed op.+ const logLookup = (+ level: "debug" | "warn" | "error",+ dbResult: "hit" | "miss" | "error",+ extra: Record<string, unknown> = {},+ ) => {+ const outcome =+ cacheStatus === "hit"+ ? "cache_hit"+ : cacheStatus === "disabled"+ ? `db_only_${dbResult}`+ : `cache_${cacheStatus}_db_${dbResult}`;+ indexLookupCounter.inc({ outcome });+ const effectiveLevel =+ level === "debug" && cacheStatus === "error" ? "warn" : level;+ meta.logger[effectiveLevel]("Index URL lookup", {+ module: "index",+ method: "scrapeURLWithIndex",+ canonicalLog: "index/url_lookup",+ outcome,+ urlHash: urlHash.toString("hex"),+ variantKey,+ teamId: meta.internalOptions.teamId,+ scrapeId: meta.id,+ maxAge,+ maxAgeSource,+ minAge: meta.options.minAge ?? null,+ timingsMaxAge: checkpoint1 - startTime,+ timingsCache,+ timingsDb,+ timingsFull: Date.now() - startTime,+ ...extra,});+ };++ let data: { id: string; created_at: string; status: number }[] = [];++ if (useIndexCache) {+ const cacheStart = Date.now();+ const read = await getCachedIndexEntries(variantKey, meta.logger);+ timingsCache = Date.now() - cacheStart;+ if (read.status === "hit") {+ const filtered = filterIndexEntries(read.entries, {+ maxAgeMs: maxAge,+ minAgeMs: meta.options.minAge ?? null,+ needsScreenshot: meta.featureFlags.has("screenshot"),+ needsScreenshotFullscreen: meta.featureFlags.has(+ "screenshot@fullScreen",+ ),+ waitTimeMs: meta.options.waitFor,+ });+ if (filtered.length > 0) {+ data = filtered;+ servedFromCache = true;+ cacheStatus = "hit";+ } else {+ // The capped per-key entry list may have dropped rows the DB still+ // has, so an empty filter result must fall through to the DB.+ cacheStatus = "filtered";+ }+ } else {+ cacheStatus = read.status;+ }+ }++ if (!servedFromCache) {+ const dbStart = Date.now();+ try {+ const rows = await indexGetRecent5({+ url_hash: urlHash,+ max_age_ms: maxAge,+ is_mobile: meta.options.mobile,+ block_ads: meta.options.blockAds,+ feature_screenshot: meta.featureFlags.has("screenshot"),+ feature_screenshot_fullscreen: meta.featureFlags.has(+ "screenshot@fullScreen",+ ),+ location_country: meta.options.location?.country ?? null,+ location_languages:+ (meta.options.location?.languages?.length ?? 0) > 0+ ? (meta.options.location?.languages ?? null)+ : null,+ wait_time_ms: meta.options.waitFor,+ is_stealth: meta.featureFlags.has("stealthProxy"),+ min_age_ms: meta.options.minAge ?? null,+ });+ timingsDb = Date.now() - dbStart;+ if (useIndexCache && rows.length > 0) {+ const entries: IndexCacheEntry[] = rows.map(row => ({+ id: row.id,+ created_at: row.created_at,+ status: row.status,+ has_screenshot: row.has_screenshot,+ has_screenshot_fullscreen: row.has_screenshot_fullscreen,+ wait_time_ms: row.wait_time_ms,+ }));+ upsertCachedIndexEntries(variantKey, entries, meta.logger).catch(+ () => {},+ );+ }+ data = rows;+ } catch (error) {+ timingsDb = Date.now() - dbStart;+ logLookup("error", "error", { error });+ throw new EngineError("Failed to retrieve URL from DB index", {+ cause: error,+ });+ }}let selectedRow: {@@ -307,15 +448,7 @@ export async function scrapeURLWithIndex(}if (selectedRow === null || selectedRow === undefined) {- meta.logger.debug("Index metrics", {- module: "index/metrics",- hit: false,- maxAge,- dynamicMaxAge: meta.options.maxAge === undefined,- timingsFull: Date.now() - startTime,- timingsMaxAge: checkpoint1 - startTime,- timingsSupa: Date.now() - checkpoint1,- });+ logLookup("debug", "miss");if (meta.internalOptions.agentIndexOnly) {throw new AgentIndexOnlyError();@@ -338,9 +471,12 @@ export async function scrapeURLWithIndex(meta.logger.child({ module: "index", method: "getIndexFromGCS" }),);if (!doc) {- meta.logger.warn("Index document not found in GCS", {- indexDocumentId: id,- });+ if (servedFromCache) {+ // Self-heal: drop the poisoned cache entry so it can't keep serving an+ // id whose document is gone.+ deleteCachedIndexEntry(variantKey, id, meta.logger).catch(() => {});+ }+ logLookup("warn", "hit", { gcsMiss: true, indexDocumentId: id });throw new EngineError("Document not found in GCS");}@@ -350,6 +486,7 @@ export async function scrapeURLWithIndex(// If the cached content is base64 PDF but we want parsed PDF (parsePDF:true or default)if (isCachedPdfBase64 && shouldParsePDF(meta.options.parsers)) {// Cached content is unparsed PDF, but we want parsed - report cache miss+ logLookup("debug", "hit", { pdfMismatch: "cached_unparsed_want_parsed" });throw new IndexMissError();}@@ -360,20 +497,16 @@ export async function scrapeURLWithIndex(meta.url.toLowerCase().endsWith(".pdf") || meta.url.includes(".pdf?");if (isPdfUrl) {// This is likely a parsed PDF cached, but we want unparsed - report cache miss+ logLookup("debug", "hit", { pdfMismatch: "cached_parsed_want_unparsed" });throw new IndexMissError();}}- meta.logger.debug("Index metrics", {- module: "index/metrics",- hit: true,+ logLookup("debug", "hit", {age: Date.now() - new Date(selectedRow.created_at).getTime(),- maxAge,- dynamicMaxAge: meta.options.maxAge === undefined,- timingsFull: Date.now() - startTime,- timingsMaxAge: checkpoint1 - startTime,- timingsSupa: checkpoint2 - checkpoint1,- timingsGCS: Date.now() - checkpoint2,+ status: selectedRow.status,+ indexDocumentId: id,+ timingsGcs: Date.now() - checkpoint2,});return {diff --git a/apps/api/src/services/index-cache.ts b/apps/api/src/services/index-cache.tsnew file mode 100644index 000000000..16a8fa859--- /dev/null+++ b/apps/api/src/services/index-cache.ts@@ -0,0 +1,316 @@+import crypto from "crypto";+import IORedis from "ioredis";+import { config } from "../config";+import { logger as _logger } from "../lib/logger";+import {+ indexCacheErrorCounter,+ indexCacheReadDuration,+} from "../lib/index-cache-metrics";+import type { Logger } from "winston";++// Dragonfly LRU cache in front of the index database's URL->id lookup hot+// path (index_get_recent_5) and the per-domain max age lookup (query_max_age).+// The instance runs with --cache_mode=true, so memory pressure evicts cold+// keys; TTLs here are hygiene, not correctness. Invalidation of index entries+// (invalidated_at) is a rare manual ops action handled by flushing the whole+// instance.++const ENTRY_KEY_PREFIX = "idxc:";+const MAX_AGE_KEY_PREFIX = "idxma:";+const ENTRY_TTL_SECONDS = 7 * 24 * 60 * 60;+const MAX_AGE_TTL_SECONDS = 15 * 60;+// More entries than index_get_recent_5's LIMIT 5 so that client-side+// filtering on screenshot/waitFor/minAge still has headroom to find 5 rows.+const ENTRY_CAP = 32;+const READ_TIMEOUT_MS = 150;++const indexCacheRedis: IORedis | null = config.INDEX_CACHE_REDIS_URL+ ? new IORedis(config.INDEX_CACHE_REDIS_URL, {+ enableAutoPipelining: true,+ // The cache must fail fast and fall back to Postgres, never queue+ // commands while disconnected.+ enableOfflineQueue: false,+ maxRetriesPerRequest: 1,+ })+ : null;++indexCacheRedis?.on("error", error => {+ _logger.warn("Index cache Redis connection error", {+ module: "index-cache",+ error,+ });+});++export const useIndexCache = indexCacheRedis !== null;++export type IndexCacheEntry = {+ id: string;+ created_at: string;+ status: number;+ has_screenshot: boolean;+ has_screenshot_fullscreen: boolean;+ wait_time_ms: number | null;+};++type IndexCacheReadResult =+ | { status: "hit"; entries: IndexCacheEntry[] }+ | { status: "miss" }+ | { status: "error" };++// Exact-match dimensions of index_get_recent_5. Range/capability dimensions+// (age window, screenshot capability, wait_time_ms) live on the entries and+// are applied by filterIndexEntries. Languages are sorted+deduped because the+// SQL compares them with set semantics (@> and <@); empty array and null stay+// distinct because the SQL treats them differently (IS NULL vs set equality).+export function deriveIndexVariantKey(params: {+ urlHash: Buffer;+ isMobile: boolean;+ blockAds: boolean;+ isStealth: boolean;+ locationCountry: string | null;+ locationLanguages: string[] | null;+}): string {+ const languages =+ params.locationLanguages === null+ ? null+ : [...new Set(params.locationLanguages)].sort();+ const payload = JSON.stringify([+ params.urlHash.toString("hex"),+ params.isMobile,+ params.blockAds,+ params.isStealth,+ params.locationCountry,+ languages,+ ]);+ return (+ ENTRY_KEY_PREFIX + crypto.createHash("sha256").update(payload).digest("hex")+ );+}++// Mirrors the per-entry filters of index_get_recent_5 (see PR description for+// the SQL source): age window, screenshot capability (a request that doesn't+// need a screenshot matches any entry; one that does requires it), waitFor+// (COALESCE(entry, 0) >= requested), newest-first, LIMIT 5. Keep in sync with+// the SQL function.+export function filterIndexEntries(+ entries: IndexCacheEntry[],+ opts: {+ maxAgeMs: number;+ minAgeMs: number | null;+ needsScreenshot: boolean;+ needsScreenshotFullscreen: boolean;+ waitTimeMs: number | null;+ now?: number;+ },+): IndexCacheEntry[] {+ const now = opts.now ?? Date.now();+ return entries+ .filter(entry => {+ const createdAt = new Date(entry.created_at).getTime();+ if (isNaN(createdAt)) return false;+ if (createdAt < now - opts.maxAgeMs) return false;+ if (opts.minAgeMs !== null && createdAt > now - opts.minAgeMs)+ return false;+ if (opts.needsScreenshot && !entry.has_screenshot) return false;+ if (opts.needsScreenshotFullscreen && !entry.has_screenshot_fullscreen)+ return false;+ if (+ opts.waitTimeMs !== null &&+ (entry.wait_time_ms ?? 0) < opts.waitTimeMs+ )+ return false;+ return true;+ })+ .sort(+ (a, b) =>+ new Date(b.created_at).getTime() - new Date(a.created_at).getTime(),+ )+ .slice(0, 5);+}++const TIMED_OUT = Symbol("index-cache-timeout");++function withTimeout<T>(+ promise: Promise<T>,+ ms: number,+): Promise<T | typeof TIMED_OUT> {+ return Promise.race([+ promise,+ new Promise<typeof TIMED_OUT>(resolve =>+ setTimeout(() => resolve(TIMED_OUT), ms).unref?.(),+ ),+ ]);+}++export async function getCachedIndexEntries(+ key: string,+ logger: Logger = _logger,+ client: IORedis | null = indexCacheRedis,+): Promise<IndexCacheReadResult> {+ if (client === null) {+ return { status: "error" };+ }+ const start = Date.now();+ try {+ const raw = await withTimeout(client.hgetall(key), READ_TIMEOUT_MS);+ indexCacheReadDuration.observe((Date.now() - start) / 1000);+ if (raw === TIMED_OUT) {+ indexCacheErrorCounter.inc({ op: "read_timeout" });+ return { status: "error" };+ }+ const fields = Object.values(raw);+ if (fields.length === 0) {+ return { status: "miss" };+ }+ const entries: IndexCacheEntry[] = [];+ for (const field of fields) {+ try {+ entries.push(JSON.parse(field));+ } catch {+ // Skip unparseable entries; they age out via TTL/eviction.+ }+ }+ if (entries.length === 0) {+ return { status: "miss" };+ }+ return { status: "hit", entries };+ } catch (error) {+ indexCacheReadDuration.observe((Date.now() - start) / 1000);+ indexCacheErrorCounter.inc({ op: "read" });+ logger.warn("Index cache read failed", {+ module: "index-cache",+ error,+ key,+ });+ return { status: "error" };+ }+}++export async function upsertCachedIndexEntries(+ key: string,+ entries: IndexCacheEntry[],+ logger: Logger = _logger,+ client: IORedis | null = indexCacheRedis,+): Promise<void> {+ if (client === null || entries.length === 0) {+ return;+ }+ try {+ const fields: Record<string, string> = {};+ for (const entry of entries) {+ fields[entry.id] = JSON.stringify(entry);+ }+ const pipeline = client.pipeline();+ pipeline.hset(key, fields);+ pipeline.expire(key, ENTRY_TTL_SECONDS);+ pipeline.hlen(key);+ const results = await pipeline.exec();+ const hlen = results?.[2]?.[1];+ if (typeof hlen === "number" && hlen > ENTRY_CAP) {+ const raw = await client.hgetall(key);+ const parsed = Object.entries(raw)+ .map(([id, field]) => {+ try {+ return { id, created_at: JSON.parse(field).created_at as string };+ } catch {+ return { id, created_at: "1970-01-01T00:00:00Z" };+ }+ })+ .sort(+ (a, b) =>+ new Date(b.created_at).getTime() - new Date(a.created_at).getTime(),+ );+ const toDelete = parsed.slice(ENTRY_CAP).map(x => x.id);+ if (toDelete.length > 0) {+ await client.hdel(key, ...toDelete);+ }+ }+ } catch (error) {+ indexCacheErrorCounter.inc({ op: "write" });+ logger.warn("Index cache write failed", {+ module: "index-cache",+ error,+ key,+ });+ }+}++export async function deleteCachedIndexEntry(+ key: string,+ id: string,+ logger: Logger = _logger,+ client: IORedis | null = indexCacheRedis,+): Promise<void> {+ if (client === null) {+ return;+ }+ try {+ await client.hdel(key, id);+ } catch (error) {+ indexCacheErrorCounter.inc({ op: "delete" });+ logger.warn("Index cache entry delete failed", {+ module: "index-cache",+ error,+ key,+ id,+ });+ }+}++// Caches query_max_age results per domain hash, including the "no signature"+// (null) result so absent domains don't keep hitting Postgres.+export async function getCachedMaxAge(+ domainHash: Buffer,+ logger: Logger = _logger,+ client: IORedis | null = indexCacheRedis,+): Promise<{ maxAge: number | null } | null> {+ if (client === null) {+ return null;+ }+ try {+ const raw = await withTimeout(+ client.get(MAX_AGE_KEY_PREFIX + domainHash.toString("hex")),+ READ_TIMEOUT_MS,+ );+ if (raw === TIMED_OUT) {+ indexCacheErrorCounter.inc({ op: "maxage_read_timeout" });+ return null;+ }+ if (raw === null || raw === undefined) {+ return null;+ }+ return { maxAge: JSON.parse(raw).max_age ?? null };+ } catch (error) {+ indexCacheErrorCounter.inc({ op: "maxage_read" });+ logger.warn("Index cache max age read failed", {+ module: "index-cache",+ error,+ });+ return null;+ }+}++export async function setCachedMaxAge(+ domainHash: Buffer,+ maxAge: number | null,+ logger: Logger = _logger,+ client: IORedis | null = indexCacheRedis,+): Promise<void> {+ if (client === null) {+ return;+ }+ try {+ await client.set(+ MAX_AGE_KEY_PREFIX + domainHash.toString("hex"),+ JSON.stringify({ max_age: maxAge }),+ "EX",+ MAX_AGE_TTL_SECONDS,+ );+ } catch (error) {+ indexCacheErrorCounter.inc({ op: "maxage_write" });+ logger.warn("Index cache max age write failed", {+ module: "index-cache",+ error,+ });+ }+}diff --git a/apps/api/src/services/index.ts b/apps/api/src/services/index.tsindex dbdab04c3..379a90f68 100644--- a/apps/api/src/services/index.ts+++ b/apps/api/src/services/index.ts@@ -15,6 +15,12 @@ import { configDotenv } from "dotenv";import { ApiError } from "@google-cloud/storage";import crypto from "crypto";import { redisEvictConnection } from "./redis";+import {+ deriveIndexVariantKey,+ upsertCachedIndexEntries,+ useIndexCache,+ type IndexCacheEntry,+} from "./index-cache";import type { Logger } from "winston";import psl from "psl";import { MapDocument } from "../controllers/v2/types";@@ -337,6 +343,7 @@ export async function processIndexInsertJobs() {try {await dbIndex.insert(schema.index).values(jobs);_logger.info(`Index inserter inserted jobs`, { jobCount: jobs.length });+ writeThroughIndexCache(jobs);} catch (error) {_logger.error(`Index inserter failed to insert jobs`, {error,@@ -345,6 +352,46 @@ export async function processIndexInsertJobs() {}}+// Write-through to the Dragonfly index cache after rows are durably in the+// index DB. created_at approximates the DB's defaultNow() by milliseconds,+// which is irrelevant against day-scale maxAge windows. Fire-and-forget: a+// cache failure must never affect the insert loop.+function writeThroughIndexCache(jobs: any[]) {+ if (!useIndexCache) {+ return;+ }+ const createdAt = new Date().toISOString();+ const byKey = new Map<string, IndexCacheEntry[]>();+ for (const job of jobs) {+ if (!Buffer.isBuffer(job.url_hash) || typeof job.id !== "string") {+ continue;+ }+ const key = deriveIndexVariantKey({+ urlHash: job.url_hash,+ isMobile: job.is_mobile,+ blockAds: job.block_ads,+ isStealth: job.is_stealth,+ locationCountry: job.location_country ?? null,+ locationLanguages: job.location_languages ?? null,+ });+ const entries = byKey.get(key) ?? [];+ entries.push({+ id: job.id,+ created_at: createdAt,+ status: job.status,+ has_screenshot: job.has_screenshot,+ has_screenshot_fullscreen: job.has_screenshot_fullscreen,+ wait_time_ms: job.wait_time_ms ?? null,+ });+ byKey.set(key, entries);+ }+ for (const [key, entries] of byKey) {+ upsertCachedIndexEntries(key, entries, _logger).catch(error => {+ _logger.warn("Index cache write-through failed", { error, key });+ });+ }+}+export async function getIndexInsertQueueLength(): Promise<number> {return (await redisEvictConnection.llen(INDEX_INSERT_QUEUE_KEY)) ?? 0;}