Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): Dragonfly cache for index URL->id lookups + canonical lookup log (#3770)

Gergő Móricz0f91869af58e
diff --git a/apps/api/src/__tests__/snips/index-cache.test.ts b/apps/api/src/__tests__/snips/index-cache.test.ts
new file mode 100644
index 000000000..12e0496e8
--- /dev/null
+++ b/apps/api/src/__tests__/snips/index-cache.test.ts
@@ -0,0 +1,230 @@
+import IORedis from "ioredis";
+import {
+ deriveIndexVariantKey,
+ filterIndexEntries,
+ getCachedIndexEntries,
+ getCachedMaxAge,
+ upsertCachedIndexEntries,
+ type IndexCacheEntry,
+} from "../../services/index-cache";
+import { hashURL } from "../../services";
+
+const urlHash = hashURL("https://example.com/page");
+
+const baseKeyParams = {
+ urlHash,
+ isMobile: false,
+ blockAds: true,
+ isStealth: false,
+ locationCountry: null,
+ locationLanguages: null,
+};
+
+function entry(overrides: Partial<IndexCacheEntry> = {}): IndexCacheEntry {
+ return {
+ id: crypto.randomUUID(),
+ created_at: new Date().toISOString(),
+ status: 200,
+ has_screenshot: false,
+ has_screenshot_fullscreen: false,
+ wait_time_ms: null,
+ ...overrides,
+ };
+}
+
+describe("deriveIndexVariantKey", () => {
+ it("is stable for identical params", () => {
+ expect(deriveIndexVariantKey(baseKeyParams)).toBe(
+ deriveIndexVariantKey({ ...baseKeyParams }),
+ );
+ });
+
+ it("changes when any exact-match dimension changes", () => {
+ const base = deriveIndexVariantKey(baseKeyParams);
+ expect(deriveIndexVariantKey({ ...baseKeyParams, isMobile: true })).not.toBe(
+ base,
+ );
+ expect(deriveIndexVariantKey({ ...baseKeyParams, blockAds: false })).not.toBe(
+ base,
+ );
+ expect(
+ deriveIndexVariantKey({ ...baseKeyParams, isStealth: true }),
+ ).not.toBe(base);
+ expect(
+ deriveIndexVariantKey({ ...baseKeyParams, locationCountry: "DE" }),
+ ).not.toBe(base);
+ expect(
+ deriveIndexVariantKey({
+ ...baseKeyParams,
+ urlHash: hashURL("https://example.com/other"),
+ }),
+ ).not.toBe(base);
+ });
+
+ it("treats languages as a set (order and duplicates ignored), matching the SQL @>/<@ comparison", () => {
+ const a = deriveIndexVariantKey({
+ ...baseKeyParams,
+ locationLanguages: ["en", "de"],
+ });
+ const b = deriveIndexVariantKey({
+ ...baseKeyParams,
+ locationLanguages: ["de", "en", "de"],
+ });
+ expect(a).toBe(b);
+ });
+
+ it("keeps empty language array distinct from null, matching the SQL IS NULL check", () => {
+ const empty = deriveIndexVariantKey({
+ ...baseKeyParams,
+ locationLanguages: [],
+ });
+ const nul = deriveIndexVariantKey({
+ ...baseKeyParams,
+ locationLanguages: null,
+ });
+ expect(empty).not.toBe(nul);
+ });
+});
+
+describe("filterIndexEntries", () => {
+ const now = Date.now();
+ const hour = 60 * 60 * 1000;
+
+ it("applies the maxAge window", () => {
+ const fresh = entry({ created_at: new Date(now - hour).toISOString() });
+ const stale = entry({ created_at: new Date(now - 5 * hour).toISOString() });
+ const result = filterIndexEntries([fresh, stale], {
+ maxAgeMs: 2 * hour,
+ minAgeMs: null,
+ needsScreenshot: false,
+ needsScreenshotFullscreen: false,
+ waitTimeMs: 0,
+ now,
+ });
+ expect(result.map(x => x.id)).toEqual([fresh.id]);
+ });
+
+ it("applies the minAge window", () => {
+ const tooFresh = entry({ created_at: new Date(now - hour).toISOString() });
+ const oldEnough = entry({
+ created_at: new Date(now - 5 * hour).toISOString(),
+ });
+ const result = filterIndexEntries([tooFresh, oldEnough], {
+ maxAgeMs: 24 * hour,
+ minAgeMs: 2 * hour,
+ needsScreenshot: false,
+ needsScreenshotFullscreen: false,
+ waitTimeMs: 0,
+ now,
+ });
+ expect(result.map(x => x.id)).toEqual([oldEnough.id]);
+ });
+
+ it("matches screenshot capability like the SQL (request-false matches anything, request-true requires it)", () => {
+ const withShot = entry({ has_screenshot: true });
+ const withoutShot = entry({ has_screenshot: false });
+ const opts = {
+ maxAgeMs: hour,
+ minAgeMs: null,
+ needsScreenshotFullscreen: false,
+ waitTimeMs: 0,
+ now,
+ };
+ expect(
+ filterIndexEntries([withShot, withoutShot], {
+ ...opts,
+ needsScreenshot: false,
+ }),
+ ).toHaveLength(2);
+ expect(
+ filterIndexEntries([withShot, withoutShot], {
+ ...opts,
+ needsScreenshot: true,
+ }).map(x => x.id),
+ ).toEqual([withShot.id]);
+ });
+
+ it("compares waitFor like the SQL (COALESCE(entry, 0) >= requested)", () => {
+ const noWait = entry({ wait_time_ms: null });
+ const shortWait = entry({ wait_time_ms: 500 });
+ const longWait = entry({ wait_time_ms: 5000 });
+ const opts = {
+ maxAgeMs: hour,
+ minAgeMs: null,
+ needsScreenshot: false,
+ needsScreenshotFullscreen: false,
+ now,
+ };
+ expect(
+ filterIndexEntries([noWait, shortWait, longWait], {
+ ...opts,
+ waitTimeMs: 0,
+ }),
+ ).toHaveLength(3);
+ expect(
+ filterIndexEntries([noWait, shortWait, longWait], {
+ ...opts,
+ waitTimeMs: 1000,
+ }).map(x => x.id),
+ ).toEqual([longWait.id]);
+ });
+
+ it("sorts newest-first and caps at 5 like the SQL LIMIT", () => {
+ const entries = Array.from({ length: 8 }, (_, i) =>
+ entry({ created_at: new Date(now - i * 1000).toISOString() }),
+ );
+ const shuffled = [...entries].reverse();
+ const result = filterIndexEntries(shuffled, {
+ maxAgeMs: hour,
+ minAgeMs: null,
+ needsScreenshot: false,
+ needsScreenshotFullscreen: false,
+ waitTimeMs: 0,
+ now,
+ });
+ expect(result).toHaveLength(5);
+ expect(result.map(x => x.id)).toEqual(entries.slice(0, 5).map(x => x.id));
+ });
+});
+
+describe("index cache fail-open", () => {
+ // Points at a port nothing listens on: every operation must resolve (not
+ // throw, not hang) so the read path can fall back to Postgres.
+ let deadClient: IORedis;
+
+ beforeAll(() => {
+ deadClient = new IORedis("redis://127.0.0.1:1", {
+ enableOfflineQueue: false,
+ maxRetriesPerRequest: 1,
+ retryStrategy: () => null,
+ lazyConnect: true,
+ });
+ deadClient.on("error", () => {});
+ deadClient.connect().catch(() => {});
+ });
+
+ afterAll(() => {
+ deadClient.disconnect();
+ });
+
+ it("getCachedIndexEntries resolves to error status", async () => {
+ const result = await getCachedIndexEntries(
+ "idxc:test",
+ undefined,
+ deadClient,
+ );
+ expect(result.status).toBe("error");
+ }, 5000);
+
+ it("upsertCachedIndexEntries resolves without throwing", async () => {
+ await expect(
+ upsertCachedIndexEntries("idxc:test", [entry()], undefined, deadClient),
+ ).resolves.toBeUndefined();
+ }, 5000);
+
+ it("getCachedMaxAge resolves to null", async () => {
+ await expect(
+ getCachedMaxAge(urlHash, undefined, deadClient),
+ ).resolves.toBeNull();
+ }, 5000);
+});
diff --git a/apps/api/src/__tests__/snips/v2/index-cache-variants.test.ts b/apps/api/src/__tests__/snips/v2/index-cache-variants.test.ts
new file mode 100644
index 000000000..8a779baea
--- /dev/null
+++ b/apps/api/src/__tests__/snips/v2/index-cache-variants.test.ts
@@ -0,0 +1,79 @@
+import { describeIf, TEST_PRODUCTION, indexCooldown } from "../lib";
+import { Identity, idmux, scrapeTimeout, scrape } from "./lib";
+import crypto from "crypto";
+
+// E2E coverage for the index URL->id lookup variant semantics, which the
+// Dragonfly index cache (services/index-cache.ts) replicates client-side from
+// index_get_recent_5. Behavior must be identical whether INDEX_CACHE_REDIS_URL
+// is set or not.
+describeIf(TEST_PRODUCTION)("V2 index lookup variant matching", () => {
+ let identity: Identity;
+
+ beforeAll(async () => {
+ identity = await idmux({
+ name: "v2-index-cache-variants",
+ concurrency: 100,
+ credits: 1000000,
+ });
+ }, 10000);
+
+ test(
+ "screenshot-capable index entry serves a request without screenshot",
+ async () => {
+ const id = crypto.randomUUID();
+ const url = "https://firecrawl.dev/?testId=" + id;
+
+ const data1 = await scrape(
+ {
+ url,
+ formats: ["markdown", "screenshot"],
+ },
+ identity,
+ );
+ expect(data1.metadata.cacheState).toBe("miss");
+
+ await new Promise(resolve => setTimeout(resolve, indexCooldown));
+
+ // A request that doesn't need a screenshot matches entries with or
+ // without one (p_feature_screenshot IS NOT TRUE OR has_screenshot).
+ const data2 = await scrape(
+ {
+ url,
+ maxAge: 10 * 60 * 1000,
+ },
+ identity,
+ );
+ expect(data2.metadata.cacheState).toBe("hit");
+ },
+ scrapeTimeout * 2 + indexCooldown,
+ );
+
+ test(
+ "screenshot request does not get served by a screenshotless index entry",
+ async () => {
+ const id = crypto.randomUUID();
+ const url = "https://firecrawl.dev/?testId=" + id;
+
+ const data1 = await scrape(
+ {
+ url,
+ },
+ identity,
+ );
+ expect(data1.metadata.cacheState).toBe("miss");
+
+ await new Promise(resolve => setTimeout(resolve, indexCooldown));
+
+ const data2 = await scrape(
+ {
+ url,
+ formats: ["markdown", "screenshot"],
+ maxAge: 10 * 60 * 1000,
+ },
+ identity,
+ );
+ expect(data2.metadata.cacheState).toBe("miss");
+ },
+ scrapeTimeout * 2 + indexCooldown,
+ );
+});
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index 3b6132985..c3e1872c0 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -84,6 +84,7 @@ const configSchema = z.object({
DATABASE_URL: z.string().optional(),
DATABASE_REPLICA_URL: z.string().optional(),
INDEX_DATABASE_URL: z.string().optional(),
+ INDEX_CACHE_REDIS_URL: z.string().optional(),
REDIS_URL: z.string().optional(),
REDIS_EVICT_URL: z.string().optional(),
REDIS_RATE_LIMIT_URL: z.string().optional(),
diff --git a/apps/api/src/db/rpc.ts b/apps/api/src/db/rpc.ts
index 42b849896..6c0e9e8d3 100644
--- a/apps/api/src/db/rpc.ts
+++ b/apps/api/src/db/rpc.ts
@@ -258,7 +258,19 @@ export function queryMaxAge(
);
}
-export function indexGetRecent4(params: {
+type IndexGetRecentRow = {
+ id: string;
+ created_at: string;
+ status: number;
+ has_screenshot: boolean;
+ has_screenshot_fullscreen: boolean;
+ wait_time_ms: number | null;
+};
+
+// Same filters as index_get_recent_4, but also returns the per-entry
+// capability columns so results can populate the Dragonfly index cache
+// (services/index-cache.ts).
+export async function indexGetRecent5(params: {
url_hash: Buffer;
max_age_ms: number;
is_mobile: boolean;
@@ -270,11 +282,15 @@ export function indexGetRecent4(params: {
wait_time_ms: number;
is_stealth: boolean;
min_age_ms: number | null;
-}): Promise<{ id: string; created_at: string; status: number }[]> {
- return execRows(
+}): Promise<IndexGetRecentRow[]> {
+ const rows = await execRows<IndexGetRecentRow>(
dbIndex,
- sql`select * from index_get_recent_4(p_url_hash => ${params.url_hash}, p_max_age_ms => ${params.max_age_ms}, p_is_mobile => ${params.is_mobile}, p_block_ads => ${params.block_ads}, p_feature_screenshot => ${params.feature_screenshot}, p_feature_screenshot_fullscreen => ${params.feature_screenshot_fullscreen}, p_location_country => ${params.location_country}, p_location_languages => ${sql.param(params.location_languages)}::text[], p_wait_time_ms => ${params.wait_time_ms}, p_is_stealth => ${params.is_stealth}, p_min_age_ms => ${params.min_age_ms})`,
+ sql`select * from index_get_recent_5(p_url_hash => ${params.url_hash}, p_max_age_ms => ${params.max_age_ms}, p_is_mobile => ${params.is_mobile}, p_block_ads => ${params.block_ads}, p_feature_screenshot => ${params.feature_screenshot}, p_feature_screenshot_fullscreen => ${params.feature_screenshot_fullscreen}, p_location_country => ${params.location_country}, p_location_languages => ${sql.param(params.location_languages)}::text[], p_wait_time_ms => ${params.wait_time_ms}, p_is_stealth => ${params.is_stealth}, p_min_age_ms => ${params.min_age_ms})`,
);
+ for (const row of rows) {
+ row.wait_time_ms = toNum(row.wait_time_ms);
+ }
+ return rows;
}
export function queryTopUrlsForDomain<T = Record<string, any>>(
diff --git a/apps/api/src/lib/index-cache-metrics.ts b/apps/api/src/lib/index-cache-metrics.ts
new file mode 100644
index 000000000..4de79a2b9
--- /dev/null
+++ b/apps/api/src/lib/index-cache-metrics.ts
@@ -0,0 +1,19 @@
+import { Counter, Histogram } from "prom-client";
+
+export const indexLookupCounter = new Counter({
+ name: "firecrawl_index_lookup_total",
+ help: "Index URL->id lookups by outcome (cache layer vs DB vs miss)",
+ labelNames: ["outcome"],
+});
+
+export const indexCacheReadDuration = new Histogram({
+ name: "firecrawl_index_cache_read_duration_seconds",
+ help: "Duration of index cache (Dragonfly) reads",
+ buckets: [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.15, 0.25, 0.5],
+});
+
+export const indexCacheErrorCounter = new Counter({
+ name: "firecrawl_index_cache_errors_total",
+ help: "Index cache (Dragonfly) operation errors",
+ labelNames: ["op"],
+});
diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
index 77ad5fc40..eb499ed67 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
@@ -13,7 +13,19 @@ import {
generateDomainSplits,
addOMCEJob,
} from "../../../../services";
-import { queryMaxAge, indexGetRecent4 } from "../../../../db/rpc";
+import { queryMaxAge, indexGetRecent5 } from "../../../../db/rpc";
+import {
+ deleteCachedIndexEntry,
+ deriveIndexVariantKey,
+ filterIndexEntries,
+ getCachedIndexEntries,
+ getCachedMaxAge,
+ setCachedMaxAge,
+ upsertCachedIndexEntries,
+ useIndexCache,
+ type IndexCacheEntry,
+} from "../../../../services/index-cache";
+import { indexLookupCounter } from "../../../../lib/index-cache-metrics";
import {
AgentIndexOnlyError,
EngineError,
@@ -215,9 +227,14 @@ export async function scrapeURLWithIndex(
const normalizedURL = normalizeURLForIndex(meta.url);
const urlHash = hashURL(normalizedURL);
+ const defaultMaxAge = 2 * 24 * 60 * 60 * 1000; // 2 days
+
+ type MaxAgeSource = "explicit" | "dynamic_cached" | "dynamic_db" | "default";
let maxAge: number;
+ let maxAgeSource: MaxAgeSource = "default";
if (meta.options.maxAge !== undefined) {
maxAge = meta.options.maxAge;
+ maxAgeSource = "explicit";
} else {
const domainSplitsHash = generateDomainSplits(
new URL(meta.url).hostname,
@@ -229,63 +246,187 @@ export async function scrapeURLWithIndex(
config.FIRECRAWL_INDEX_WRITE_ONLY ||
config.USE_DB_AUTHENTICATION !== true
) {
- maxAge = 2 * 24 * 60 * 60 * 1000; // 2 days
+ maxAge = defaultMaxAge;
} else {
try {
- maxAge = await Promise.race([
- (async () => {
+ const resolved = await Promise.race([
+ (async (): Promise<{
+ value: number;
+ source: MaxAgeSource;
+ }> => {
try {
- const data = await queryMaxAge(domainSplitsHash[level]);
- if (!data || data.length === 0) {
- return 2 * 24 * 60 * 60 * 1000; // 2 days
+ const domainHash = domainSplitsHash[level];
+ if (useIndexCache) {
+ const cached = await getCachedMaxAge(domainHash, meta.logger);
+ if (cached !== null) {
+ return {
+ value: cached.maxAge ?? defaultMaxAge,
+ source: "dynamic_cached",
+ };
+ }
+ }
+ const data = await queryMaxAge(domainHash);
+ const value =
+ !data || data.length === 0 ? null : (data[0].max_age ?? null);
+ if (useIndexCache) {
+ setCachedMaxAge(domainHash, value, meta.logger).catch(
+ () => {},
+ );
}
- return data[0].max_age ?? 2 * 24 * 60 * 60 * 1000; // 2 days
+ return { value: value ?? defaultMaxAge, source: "dynamic_db" };
} catch (error) {
meta.logger.warn("Failed to get max age from DB", { error });
- return 2 * 24 * 60 * 60 * 1000; // 2 days
+ return { value: defaultMaxAge, source: "default" };
}
})(),
- new Promise<number>(resolve =>
- setTimeout(() => {
- resolve(2 * 24 * 60 * 60 * 1000); // 2 days
- }, 200),
+ new Promise<{ value: number; source: MaxAgeSource }>(
+ resolve =>
+ setTimeout(() => {
+ resolve({ value: defaultMaxAge, source: "default" });
+ }, 200),
),
]);
+ maxAge = resolved.value;
+ maxAgeSource = resolved.source;
} catch (e) {
meta.logger.warn("Failed to get max age from DB", {
error: e,
});
- maxAge = 2 * 24 * 60 * 60 * 1000; // 2 days
+ maxAge = defaultMaxAge;
}
}
}
const checkpoint1 = Date.now();
- let data: { id: string; created_at: string; status: number }[];
- try {
- data = await indexGetRecent4({
- url_hash: urlHash,
- max_age_ms: maxAge,
- is_mobile: meta.options.mobile,
- block_ads: meta.options.blockAds,
- feature_screenshot: meta.featureFlags.has("screenshot"),
- feature_screenshot_fullscreen: meta.featureFlags.has(
- "screenshot@fullScreen",
- ),
- location_country: meta.options.location?.country ?? null,
- location_languages:
- (meta.options.location?.languages?.length ?? 0) > 0
- ? (meta.options.location?.languages ?? null)
- : null,
- wait_time_ms: meta.options.waitFor,
- is_stealth: meta.featureFlags.has("stealthProxy"),
- min_age_ms: meta.options.minAge ?? null,
- });
- } catch (error) {
- throw new EngineError("Failed to retrieve URL from DB index", {
- cause: error,
+ const variantKey = deriveIndexVariantKey({
+ urlHash,
+ isMobile: meta.options.mobile,
+ blockAds: meta.options.blockAds,
+ isStealth: meta.featureFlags.has("stealthProxy"),
+ locationCountry: meta.options.location?.country ?? null,
+ locationLanguages:
+ (meta.options.location?.languages?.length ?? 0) > 0
+ ? (meta.options.location?.languages ?? null)
+ : null,
+ });
+
+ let cacheStatus: "hit" | "filtered" | "miss" | "error" | "disabled" =
+ "disabled";
+ let servedFromCache = false;
+ let timingsCache = 0;
+ let timingsDb = 0;
+
+ // Canonical log for every index URL->id lookup. debug = normal op, warn =
+ // recovered weird op (cache failure with DB fallback, GCS self-heal),
+ // error = failed op.
+ const logLookup = (
+ level: "debug" | "warn" | "error",
+ dbResult: "hit" | "miss" | "error",
+ extra: Record<string, unknown> = {},
+ ) => {
+ const outcome =
+ cacheStatus === "hit"
+ ? "cache_hit"
+ : cacheStatus === "disabled"
+ ? `db_only_${dbResult}`
+ : `cache_${cacheStatus}_db_${dbResult}`;
+ indexLookupCounter.inc({ outcome });
+ const effectiveLevel =
+ level === "debug" && cacheStatus === "error" ? "warn" : level;
+ meta.logger[effectiveLevel]("Index URL lookup", {
+ module: "index",
+ method: "scrapeURLWithIndex",
+ canonicalLog: "index/url_lookup",
+ outcome,
+ urlHash: urlHash.toString("hex"),
+ variantKey,
+ teamId: meta.internalOptions.teamId,
+ scrapeId: meta.id,
+ maxAge,
+ maxAgeSource,
+ minAge: meta.options.minAge ?? null,
+ timingsMaxAge: checkpoint1 - startTime,
+ timingsCache,
+ timingsDb,
+ timingsFull: Date.now() - startTime,
+ ...extra,
});
+ };
+
+ let data: { id: string; created_at: string; status: number }[] = [];
+
+ if (useIndexCache) {
+ const cacheStart = Date.now();
+ const read = await getCachedIndexEntries(variantKey, meta.logger);
+ timingsCache = Date.now() - cacheStart;
+ if (read.status === "hit") {
+ const filtered = filterIndexEntries(read.entries, {
+ maxAgeMs: maxAge,
+ minAgeMs: meta.options.minAge ?? null,
+ needsScreenshot: meta.featureFlags.has("screenshot"),
+ needsScreenshotFullscreen: meta.featureFlags.has(
+ "screenshot@fullScreen",
+ ),
+ waitTimeMs: meta.options.waitFor,
+ });
+ if (filtered.length > 0) {
+ data = filtered;
+ servedFromCache = true;
+ cacheStatus = "hit";
+ } else {
+ // The capped per-key entry list may have dropped rows the DB still
+ // has, so an empty filter result must fall through to the DB.
+ cacheStatus = "filtered";
+ }
+ } else {
+ cacheStatus = read.status;
+ }
+ }
+
+ if (!servedFromCache) {
+ const dbStart = Date.now();
+ try {
+ const rows = await indexGetRecent5({
+ url_hash: urlHash,
+ max_age_ms: maxAge,
+ is_mobile: meta.options.mobile,
+ block_ads: meta.options.blockAds,
+ feature_screenshot: meta.featureFlags.has("screenshot"),
+ feature_screenshot_fullscreen: meta.featureFlags.has(
+ "screenshot@fullScreen",
+ ),
+ location_country: meta.options.location?.country ?? null,
+ location_languages:
+ (meta.options.location?.languages?.length ?? 0) > 0
+ ? (meta.options.location?.languages ?? null)
+ : null,
+ wait_time_ms: meta.options.waitFor,
+ is_stealth: meta.featureFlags.has("stealthProxy"),
+ min_age_ms: meta.options.minAge ?? null,
+ });
+ timingsDb = Date.now() - dbStart;
+ if (useIndexCache && rows.length > 0) {
+ const entries: IndexCacheEntry[] = rows.map(row => ({
+ id: row.id,
+ created_at: row.created_at,
+ status: row.status,
+ has_screenshot: row.has_screenshot,
+ has_screenshot_fullscreen: row.has_screenshot_fullscreen,
+ wait_time_ms: row.wait_time_ms,
+ }));
+ upsertCachedIndexEntries(variantKey, entries, meta.logger).catch(
+ () => {},
+ );
+ }
+ data = rows;
+ } catch (error) {
+ timingsDb = Date.now() - dbStart;
+ logLookup("error", "error", { error });
+ throw new EngineError("Failed to retrieve URL from DB index", {
+ cause: error,
+ });
+ }
}
let selectedRow: {
@@ -307,15 +448,7 @@ export async function scrapeURLWithIndex(
}
if (selectedRow === null || selectedRow === undefined) {
- meta.logger.debug("Index metrics", {
- module: "index/metrics",
- hit: false,
- maxAge,
- dynamicMaxAge: meta.options.maxAge === undefined,
- timingsFull: Date.now() - startTime,
- timingsMaxAge: checkpoint1 - startTime,
- timingsSupa: Date.now() - checkpoint1,
- });
+ logLookup("debug", "miss");
if (meta.internalOptions.agentIndexOnly) {
throw new AgentIndexOnlyError();
@@ -338,9 +471,12 @@ export async function scrapeURLWithIndex(
meta.logger.child({ module: "index", method: "getIndexFromGCS" }),
);
if (!doc) {
- meta.logger.warn("Index document not found in GCS", {
- indexDocumentId: id,
- });
+ if (servedFromCache) {
+ // Self-heal: drop the poisoned cache entry so it can't keep serving an
+ // id whose document is gone.
+ deleteCachedIndexEntry(variantKey, id, meta.logger).catch(() => {});
+ }
+ logLookup("warn", "hit", { gcsMiss: true, indexDocumentId: id });
throw new EngineError("Document not found in GCS");
}
@@ -350,6 +486,7 @@ export async function scrapeURLWithIndex(
// If the cached content is base64 PDF but we want parsed PDF (parsePDF:true or default)
if (isCachedPdfBase64 && shouldParsePDF(meta.options.parsers)) {
// Cached content is unparsed PDF, but we want parsed - report cache miss
+ logLookup("debug", "hit", { pdfMismatch: "cached_unparsed_want_parsed" });
throw new IndexMissError();
}
@@ -360,20 +497,16 @@ export async function scrapeURLWithIndex(
meta.url.toLowerCase().endsWith(".pdf") || meta.url.includes(".pdf?");
if (isPdfUrl) {
// This is likely a parsed PDF cached, but we want unparsed - report cache miss
+ logLookup("debug", "hit", { pdfMismatch: "cached_parsed_want_unparsed" });
throw new IndexMissError();
}
}
- meta.logger.debug("Index metrics", {
- module: "index/metrics",
- hit: true,
+ logLookup("debug", "hit", {
age: Date.now() - new Date(selectedRow.created_at).getTime(),
- maxAge,
- dynamicMaxAge: meta.options.maxAge === undefined,
- timingsFull: Date.now() - startTime,
- timingsMaxAge: checkpoint1 - startTime,
- timingsSupa: checkpoint2 - checkpoint1,
- timingsGCS: Date.now() - checkpoint2,
+ status: selectedRow.status,
+ indexDocumentId: id,
+ timingsGcs: Date.now() - checkpoint2,
});
return {
diff --git a/apps/api/src/services/index-cache.ts b/apps/api/src/services/index-cache.ts
new file mode 100644
index 000000000..16a8fa859
--- /dev/null
+++ b/apps/api/src/services/index-cache.ts
@@ -0,0 +1,316 @@
+import crypto from "crypto";
+import IORedis from "ioredis";
+import { config } from "../config";
+import { logger as _logger } from "../lib/logger";
+import {
+ indexCacheErrorCounter,
+ indexCacheReadDuration,
+} from "../lib/index-cache-metrics";
+import type { Logger } from "winston";
+
+// Dragonfly LRU cache in front of the index database's URL->id lookup hot
+// path (index_get_recent_5) and the per-domain max age lookup (query_max_age).
+// The instance runs with --cache_mode=true, so memory pressure evicts cold
+// keys; TTLs here are hygiene, not correctness. Invalidation of index entries
+// (invalidated_at) is a rare manual ops action handled by flushing the whole
+// instance.
+
+const ENTRY_KEY_PREFIX = "idxc:";
+const MAX_AGE_KEY_PREFIX = "idxma:";
+const ENTRY_TTL_SECONDS = 7 * 24 * 60 * 60;
+const MAX_AGE_TTL_SECONDS = 15 * 60;
+// More entries than index_get_recent_5's LIMIT 5 so that client-side
+// filtering on screenshot/waitFor/minAge still has headroom to find 5 rows.
+const ENTRY_CAP = 32;
+const READ_TIMEOUT_MS = 150;
+
+const indexCacheRedis: IORedis | null = config.INDEX_CACHE_REDIS_URL
+ ? new IORedis(config.INDEX_CACHE_REDIS_URL, {
+ enableAutoPipelining: true,
+ // The cache must fail fast and fall back to Postgres, never queue
+ // commands while disconnected.
+ enableOfflineQueue: false,
+ maxRetriesPerRequest: 1,
+ })
+ : null;
+
+indexCacheRedis?.on("error", error => {
+ _logger.warn("Index cache Redis connection error", {
+ module: "index-cache",
+ error,
+ });
+});
+
+export const useIndexCache = indexCacheRedis !== null;
+
+export type IndexCacheEntry = {
+ id: string;
+ created_at: string;
+ status: number;
+ has_screenshot: boolean;
+ has_screenshot_fullscreen: boolean;
+ wait_time_ms: number | null;
+};
+
+type IndexCacheReadResult =
+ | { status: "hit"; entries: IndexCacheEntry[] }
+ | { status: "miss" }
+ | { status: "error" };
+
+// Exact-match dimensions of index_get_recent_5. Range/capability dimensions
+// (age window, screenshot capability, wait_time_ms) live on the entries and
+// are applied by filterIndexEntries. Languages are sorted+deduped because the
+// SQL compares them with set semantics (@> and <@); empty array and null stay
+// distinct because the SQL treats them differently (IS NULL vs set equality).
+export function deriveIndexVariantKey(params: {
+ urlHash: Buffer;
+ isMobile: boolean;
+ blockAds: boolean;
+ isStealth: boolean;
+ locationCountry: string | null;
+ locationLanguages: string[] | null;
+}): string {
+ const languages =
+ params.locationLanguages === null
+ ? null
+ : [...new Set(params.locationLanguages)].sort();
+ const payload = JSON.stringify([
+ params.urlHash.toString("hex"),
+ params.isMobile,
+ params.blockAds,
+ params.isStealth,
+ params.locationCountry,
+ languages,
+ ]);
+ return (
+ ENTRY_KEY_PREFIX + crypto.createHash("sha256").update(payload).digest("hex")
+ );
+}
+
+// Mirrors the per-entry filters of index_get_recent_5 (see PR description for
+// the SQL source): age window, screenshot capability (a request that doesn't
+// need a screenshot matches any entry; one that does requires it), waitFor
+// (COALESCE(entry, 0) >= requested), newest-first, LIMIT 5. Keep in sync with
+// the SQL function.
+export function filterIndexEntries(
+ entries: IndexCacheEntry[],
+ opts: {
+ maxAgeMs: number;
+ minAgeMs: number | null;
+ needsScreenshot: boolean;
+ needsScreenshotFullscreen: boolean;
+ waitTimeMs: number | null;
+ now?: number;
+ },
+): IndexCacheEntry[] {
+ const now = opts.now ?? Date.now();
+ return entries
+ .filter(entry => {
+ const createdAt = new Date(entry.created_at).getTime();
+ if (isNaN(createdAt)) return false;
+ if (createdAt < now - opts.maxAgeMs) return false;
+ if (opts.minAgeMs !== null && createdAt > now - opts.minAgeMs)
+ return false;
+ if (opts.needsScreenshot && !entry.has_screenshot) return false;
+ if (opts.needsScreenshotFullscreen && !entry.has_screenshot_fullscreen)
+ return false;
+ if (
+ opts.waitTimeMs !== null &&
+ (entry.wait_time_ms ?? 0) < opts.waitTimeMs
+ )
+ return false;
+ return true;
+ })
+ .sort(
+ (a, b) =>
+ new Date(b.created_at).getTime() - new Date(a.created_at).getTime(),
+ )
+ .slice(0, 5);
+}
+
+const TIMED_OUT = Symbol("index-cache-timeout");
+
+function withTimeout<T>(
+ promise: Promise<T>,
+ ms: number,
+): Promise<T | typeof TIMED_OUT> {
+ return Promise.race([
+ promise,
+ new Promise<typeof TIMED_OUT>(resolve =>
+ setTimeout(() => resolve(TIMED_OUT), ms).unref?.(),
+ ),
+ ]);
+}
+
+export async function getCachedIndexEntries(
+ key: string,
+ logger: Logger = _logger,
+ client: IORedis | null = indexCacheRedis,
+): Promise<IndexCacheReadResult> {
+ if (client === null) {
+ return { status: "error" };
+ }
+ const start = Date.now();
+ try {
+ const raw = await withTimeout(client.hgetall(key), READ_TIMEOUT_MS);
+ indexCacheReadDuration.observe((Date.now() - start) / 1000);
+ if (raw === TIMED_OUT) {
+ indexCacheErrorCounter.inc({ op: "read_timeout" });
+ return { status: "error" };
+ }
+ const fields = Object.values(raw);
+ if (fields.length === 0) {
+ return { status: "miss" };
+ }
+ const entries: IndexCacheEntry[] = [];
+ for (const field of fields) {
+ try {
+ entries.push(JSON.parse(field));
+ } catch {
+ // Skip unparseable entries; they age out via TTL/eviction.
+ }
+ }
+ if (entries.length === 0) {
+ return { status: "miss" };
+ }
+ return { status: "hit", entries };
+ } catch (error) {
+ indexCacheReadDuration.observe((Date.now() - start) / 1000);
+ indexCacheErrorCounter.inc({ op: "read" });
+ logger.warn("Index cache read failed", {
+ module: "index-cache",
+ error,
+ key,
+ });
+ return { status: "error" };
+ }
+}
+
+export async function upsertCachedIndexEntries(
+ key: string,
+ entries: IndexCacheEntry[],
+ logger: Logger = _logger,
+ client: IORedis | null = indexCacheRedis,
+): Promise<void> {
+ if (client === null || entries.length === 0) {
+ return;
+ }
+ try {
+ const fields: Record<string, string> = {};
+ for (const entry of entries) {
+ fields[entry.id] = JSON.stringify(entry);
+ }
+ const pipeline = client.pipeline();
+ pipeline.hset(key, fields);
+ pipeline.expire(key, ENTRY_TTL_SECONDS);
+ pipeline.hlen(key);
+ const results = await pipeline.exec();
+ const hlen = results?.[2]?.[1];
+ if (typeof hlen === "number" && hlen > ENTRY_CAP) {
+ const raw = await client.hgetall(key);
+ const parsed = Object.entries(raw)
+ .map(([id, field]) => {
+ try {
+ return { id, created_at: JSON.parse(field).created_at as string };
+ } catch {
+ return { id, created_at: "1970-01-01T00:00:00Z" };
+ }
+ })
+ .sort(
+ (a, b) =>
+ new Date(b.created_at).getTime() - new Date(a.created_at).getTime(),
+ );
+ const toDelete = parsed.slice(ENTRY_CAP).map(x => x.id);
+ if (toDelete.length > 0) {
+ await client.hdel(key, ...toDelete);
+ }
+ }
+ } catch (error) {
+ indexCacheErrorCounter.inc({ op: "write" });
+ logger.warn("Index cache write failed", {
+ module: "index-cache",
+ error,
+ key,
+ });
+ }
+}
+
+export async function deleteCachedIndexEntry(
+ key: string,
+ id: string,
+ logger: Logger = _logger,
+ client: IORedis | null = indexCacheRedis,
+): Promise<void> {
+ if (client === null) {
+ return;
+ }
+ try {
+ await client.hdel(key, id);
+ } catch (error) {
+ indexCacheErrorCounter.inc({ op: "delete" });
+ logger.warn("Index cache entry delete failed", {
+ module: "index-cache",
+ error,
+ key,
+ id,
+ });
+ }
+}
+
+// Caches query_max_age results per domain hash, including the "no signature"
+// (null) result so absent domains don't keep hitting Postgres.
+export async function getCachedMaxAge(
+ domainHash: Buffer,
+ logger: Logger = _logger,
+ client: IORedis | null = indexCacheRedis,
+): Promise<{ maxAge: number | null } | null> {
+ if (client === null) {
+ return null;
+ }
+ try {
+ const raw = await withTimeout(
+ client.get(MAX_AGE_KEY_PREFIX + domainHash.toString("hex")),
+ READ_TIMEOUT_MS,
+ );
+ if (raw === TIMED_OUT) {
+ indexCacheErrorCounter.inc({ op: "maxage_read_timeout" });
+ return null;
+ }
+ if (raw === null || raw === undefined) {
+ return null;
+ }
+ return { maxAge: JSON.parse(raw).max_age ?? null };
+ } catch (error) {
+ indexCacheErrorCounter.inc({ op: "maxage_read" });
+ logger.warn("Index cache max age read failed", {
+ module: "index-cache",
+ error,
+ });
+ return null;
+ }
+}
+
+export async function setCachedMaxAge(
+ domainHash: Buffer,
+ maxAge: number | null,
+ logger: Logger = _logger,
+ client: IORedis | null = indexCacheRedis,
+): Promise<void> {
+ if (client === null) {
+ return;
+ }
+ try {
+ await client.set(
+ MAX_AGE_KEY_PREFIX + domainHash.toString("hex"),
+ JSON.stringify({ max_age: maxAge }),
+ "EX",
+ MAX_AGE_TTL_SECONDS,
+ );
+ } catch (error) {
+ indexCacheErrorCounter.inc({ op: "maxage_write" });
+ logger.warn("Index cache max age write failed", {
+ module: "index-cache",
+ error,
+ });
+ }
+}
diff --git a/apps/api/src/services/index.ts b/apps/api/src/services/index.ts
index dbdab04c3..379a90f68 100644
--- a/apps/api/src/services/index.ts
+++ b/apps/api/src/services/index.ts
@@ -15,6 +15,12 @@ import { configDotenv } from "dotenv";
import { ApiError } from "@google-cloud/storage";
import crypto from "crypto";
import { redisEvictConnection } from "./redis";
+import {
+ deriveIndexVariantKey,
+ upsertCachedIndexEntries,
+ useIndexCache,
+ type IndexCacheEntry,
+} from "./index-cache";
import type { Logger } from "winston";
import psl from "psl";
import { MapDocument } from "../controllers/v2/types";
@@ -337,6 +343,7 @@ export async function processIndexInsertJobs() {
try {
await dbIndex.insert(schema.index).values(jobs);
_logger.info(`Index inserter inserted jobs`, { jobCount: jobs.length });
+ writeThroughIndexCache(jobs);
} catch (error) {
_logger.error(`Index inserter failed to insert jobs`, {
error,
@@ -345,6 +352,46 @@ export async function processIndexInsertJobs() {
}
}
+// Write-through to the Dragonfly index cache after rows are durably in the
+// index DB. created_at approximates the DB's defaultNow() by milliseconds,
+// which is irrelevant against day-scale maxAge windows. Fire-and-forget: a
+// cache failure must never affect the insert loop.
+function writeThroughIndexCache(jobs: any[]) {
+ if (!useIndexCache) {
+ return;
+ }
+ const createdAt = new Date().toISOString();
+ const byKey = new Map<string, IndexCacheEntry[]>();
+ for (const job of jobs) {
+ if (!Buffer.isBuffer(job.url_hash) || typeof job.id !== "string") {
+ continue;
+ }
+ const key = deriveIndexVariantKey({
+ urlHash: job.url_hash,
+ isMobile: job.is_mobile,
+ blockAds: job.block_ads,
+ isStealth: job.is_stealth,
+ locationCountry: job.location_country ?? null,
+ locationLanguages: job.location_languages ?? null,
+ });
+ const entries = byKey.get(key) ?? [];
+ entries.push({
+ id: job.id,
+ created_at: createdAt,
+ status: job.status,
+ has_screenshot: job.has_screenshot,
+ has_screenshot_fullscreen: job.has_screenshot_fullscreen,
+ wait_time_ms: job.wait_time_ms ?? null,
+ });
+ byKey.set(key, entries);
+ }
+ for (const [key, entries] of byKey) {
+ upsertCachedIndexEntries(key, entries, _logger).catch(error => {
+ _logger.warn("Index cache write-through failed", { error, key });
+ });
+ }
+}
+
export async function getIndexInsertQueueLength(): Promise<number> {
return (await redisEvictConnection.llen(INDEX_INSERT_QUEUE_KEY)) ?? 0;
}