Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick: (#3879)
Nicolasc2fc90578628
diff --git a/apps/api/src/__tests__/snips/index-cache.test.ts b/apps/api/src/__tests__/snips/index-cache.test.tsindex 9d4366e41..8d1baeff9 100644--- a/apps/api/src/__tests__/snips/index-cache.test.ts+++ b/apps/api/src/__tests__/snips/index-cache.test.ts@@ -6,6 +6,7 @@ import {getCachedMaxAge,getCachedNegative,isNegativeStillValid,+ SCREENSHOT_INDEX_CUTOFF_MS,setCachedNegative,upsertCachedIndexEntries,type IndexCacheEntry,@@ -172,6 +173,82 @@ describe("filterIndexEntries", () => {).toEqual([longWait.id]);});+ it("drops pre-cutoff screenshot entries but keeps non-screenshot ones (temporary screenshot cutoff)", () => {+ const cutoff = now - hour;+ const preCutoff = entry({+ has_screenshot: true,+ created_at: new Date(cutoff - 1000).toISOString(),+ });+ const postCutoff = entry({+ has_screenshot: true,+ created_at: new Date(cutoff + 1000).toISOString(),+ });+ const opts = {+ maxAgeMs: 24 * hour,+ minAgeMs: null,+ needsScreenshotFullscreen: false,+ waitTimeMs: 0,+ screenshotCutoffMs: cutoff,+ now,+ };+ // Screenshot request: only the post-cutoff entry survives.+ expect(+ filterIndexEntries([preCutoff, postCutoff], {+ ...opts,+ needsScreenshot: true,+ }).map(x => x.id),+ ).toEqual([postCutoff.id]);+ // Non-screenshot request: the cutoff doesn't apply, both survive.+ expect(+ filterIndexEntries([preCutoff, postCutoff], {+ ...opts,+ needsScreenshot: false,+ }),+ ).toHaveLength(2);+ });++ it("applies the screenshot cutoff to fullscreen-screenshot requests too", () => {+ const cutoff = now - hour;+ const preCutoff = entry({+ has_screenshot: true,+ has_screenshot_fullscreen: true,+ created_at: new Date(cutoff - 1000).toISOString(),+ });+ expect(+ filterIndexEntries([preCutoff], {+ maxAgeMs: 24 * hour,+ minAgeMs: null,+ needsScreenshot: false,+ needsScreenshotFullscreen: true,+ waitTimeMs: 0,+ screenshotCutoffMs: cutoff,+ now,+ }),+ ).toHaveLength(0);+ });++ it("ignores the screenshot cutoff when screenshotCutoffMs is omitted", () => {+ const preCutoff = entry({+ has_screenshot: true,+ created_at: new Date(now - 5 * hour).toISOString(),+ });+ expect(+ filterIndexEntries([preCutoff], {+ maxAgeMs: 24 * hour,+ minAgeMs: null,+ needsScreenshot: true,+ needsScreenshotFullscreen: false,+ waitTimeMs: 0,+ now,+ }),+ ).toHaveLength(1);+ });++ it("exports a sane screenshot cutoff constant", () => {+ expect(Number.isNaN(SCREENSHOT_INDEX_CUTOFF_MS)).toBe(false);+ expect(SCREENSHOT_INDEX_CUTOFF_MS).toBe(Date.parse("2026-06-24T20:00:00Z"));+ });+it("sorts newest-first and caps at 5 like the SQL LIMIT", () => {const entries = Array.from({ length: 8 }, (_, i) =>entry({ created_at: new Date(now - i * 1000).toISOString() }),diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.tsindex 325a35d48..d56c60894 100644--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts@@ -22,6 +22,7 @@ import {getCachedMaxAge,getCachedNegative,isNegativeStillValid,+ SCREENSHOT_INDEX_CUTOFF_MS,setCachedMaxAge,setCachedNegative,upsertCachedIndexEntries,@@ -372,6 +373,7 @@ export async function scrapeURLWithIndex("screenshot@fullScreen",),waitTimeMs: meta.options.waitFor,+ screenshotCutoffMs: SCREENSHOT_INDEX_CUTOFF_MS,});if (filtered.length > 0) {data = filtered;@@ -463,6 +465,20 @@ export async function scrapeURLWithIndex(}}+ // TEMPORARY: screenshots indexed before SCREENSHOT_INDEX_CUTOFF_MS have+ // unreachable URLs, so never serve them for screenshot requests — drop them+ // here so we waterfall to a fresh scrape that re-saves a working URL. The+ // cache path already filters these via filterIndexEntries; this covers the+ // DB path. Remove once pre-cutoff entries have aged out.+ if (+ meta.featureFlags.has("screenshot") ||+ meta.featureFlags.has("screenshot@fullScreen")+ ) {+ data = data.filter(+ x => new Date(x.created_at).getTime() >= SCREENSHOT_INDEX_CUTOFF_MS,+ );+ }+let selectedRow: {id: string;created_at: string;diff --git a/apps/api/src/services/index-cache.ts b/apps/api/src/services/index-cache.tsindex 3786d526e..b0141f36d 100644--- a/apps/api/src/services/index-cache.ts+++ b/apps/api/src/services/index-cache.ts@@ -100,11 +100,20 @@ export function deriveIndexVariantKey(params: {);}+// TEMPORARY: screenshots indexed before this instant were stored with URLs+// that are no longer reachable, so the index must not serve them when a+// screenshot is requested — force a fresh scrape that re-saves a working URL+// instead. The default 2-day index window means pre-cutoff entries age out+// shortly after, so remove this constant (and the screenshotCutoffMs plumbing+// in filterIndexEntries / scrapeURLWithIndex) once they're gone.+export const SCREENSHOT_INDEX_CUTOFF_MS = Date.parse("2026-06-24T20:00:00Z");+// Mirrors the per-entry filters of index_get_recent_5 (see PR description for// the SQL source): age window, screenshot capability (a request that doesn't// need a screenshot matches any entry; one that does requires it), waitFor// (COALESCE(entry, 0) >= requested), newest-first, LIMIT 5. Keep in sync with-// the SQL function.+// the SQL function. screenshotCutoffMs is a temporary app-level guard (no SQL+// equivalent) that drops pre-cutoff screenshot entries.export function filterIndexEntries(entries: IndexCacheEntry[],opts: {@@ -113,6 +122,7 @@ export function filterIndexEntries(needsScreenshot: boolean;needsScreenshotFullscreen: boolean;waitTimeMs: number | null;+ screenshotCutoffMs?: number | null;now?: number;},): IndexCacheEntry[] {@@ -127,6 +137,12 @@ export function filterIndexEntries(if (opts.needsScreenshot && !entry.has_screenshot) return false;if (opts.needsScreenshotFullscreen && !entry.has_screenshot_fullscreen)return false;+ if (+ (opts.needsScreenshot || opts.needsScreenshotFullscreen) &&+ opts.screenshotCutoffMs != null &&+ createdAt < opts.screenshotCutoffMs+ )+ return false;if (opts.waitTimeMs !== null &&(entry.wait_time_ms ?? 0) < opts.waitTimeMs