Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Nick: (#3879)

Nicolasc2fc90578628
diff --git a/apps/api/src/__tests__/snips/index-cache.test.ts b/apps/api/src/__tests__/snips/index-cache.test.ts
index 9d4366e41..8d1baeff9 100644
--- a/apps/api/src/__tests__/snips/index-cache.test.ts
+++ b/apps/api/src/__tests__/snips/index-cache.test.ts
@@ -6,6 +6,7 @@ import {
getCachedMaxAge,
getCachedNegative,
isNegativeStillValid,
+ SCREENSHOT_INDEX_CUTOFF_MS,
setCachedNegative,
upsertCachedIndexEntries,
type IndexCacheEntry,
@@ -172,6 +173,82 @@ describe("filterIndexEntries", () => {
).toEqual([longWait.id]);
});
+ it("drops pre-cutoff screenshot entries but keeps non-screenshot ones (temporary screenshot cutoff)", () => {
+ const cutoff = now - hour;
+ const preCutoff = entry({
+ has_screenshot: true,
+ created_at: new Date(cutoff - 1000).toISOString(),
+ });
+ const postCutoff = entry({
+ has_screenshot: true,
+ created_at: new Date(cutoff + 1000).toISOString(),
+ });
+ const opts = {
+ maxAgeMs: 24 * hour,
+ minAgeMs: null,
+ needsScreenshotFullscreen: false,
+ waitTimeMs: 0,
+ screenshotCutoffMs: cutoff,
+ now,
+ };
+ // Screenshot request: only the post-cutoff entry survives.
+ expect(
+ filterIndexEntries([preCutoff, postCutoff], {
+ ...opts,
+ needsScreenshot: true,
+ }).map(x => x.id),
+ ).toEqual([postCutoff.id]);
+ // Non-screenshot request: the cutoff doesn't apply, both survive.
+ expect(
+ filterIndexEntries([preCutoff, postCutoff], {
+ ...opts,
+ needsScreenshot: false,
+ }),
+ ).toHaveLength(2);
+ });
+
+ it("applies the screenshot cutoff to fullscreen-screenshot requests too", () => {
+ const cutoff = now - hour;
+ const preCutoff = entry({
+ has_screenshot: true,
+ has_screenshot_fullscreen: true,
+ created_at: new Date(cutoff - 1000).toISOString(),
+ });
+ expect(
+ filterIndexEntries([preCutoff], {
+ maxAgeMs: 24 * hour,
+ minAgeMs: null,
+ needsScreenshot: false,
+ needsScreenshotFullscreen: true,
+ waitTimeMs: 0,
+ screenshotCutoffMs: cutoff,
+ now,
+ }),
+ ).toHaveLength(0);
+ });
+
+ it("ignores the screenshot cutoff when screenshotCutoffMs is omitted", () => {
+ const preCutoff = entry({
+ has_screenshot: true,
+ created_at: new Date(now - 5 * hour).toISOString(),
+ });
+ expect(
+ filterIndexEntries([preCutoff], {
+ maxAgeMs: 24 * hour,
+ minAgeMs: null,
+ needsScreenshot: true,
+ needsScreenshotFullscreen: false,
+ waitTimeMs: 0,
+ now,
+ }),
+ ).toHaveLength(1);
+ });
+
+ it("exports a sane screenshot cutoff constant", () => {
+ expect(Number.isNaN(SCREENSHOT_INDEX_CUTOFF_MS)).toBe(false);
+ expect(SCREENSHOT_INDEX_CUTOFF_MS).toBe(Date.parse("2026-06-24T20:00:00Z"));
+ });
+
it("sorts newest-first and caps at 5 like the SQL LIMIT", () => {
const entries = Array.from({ length: 8 }, (_, i) =>
entry({ created_at: new Date(now - i * 1000).toISOString() }),
diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
index 325a35d48..d56c60894 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
@@ -22,6 +22,7 @@ import {
getCachedMaxAge,
getCachedNegative,
isNegativeStillValid,
+ SCREENSHOT_INDEX_CUTOFF_MS,
setCachedMaxAge,
setCachedNegative,
upsertCachedIndexEntries,
@@ -372,6 +373,7 @@ export async function scrapeURLWithIndex(
"screenshot@fullScreen",
),
waitTimeMs: meta.options.waitFor,
+ screenshotCutoffMs: SCREENSHOT_INDEX_CUTOFF_MS,
});
if (filtered.length > 0) {
data = filtered;
@@ -463,6 +465,20 @@ export async function scrapeURLWithIndex(
}
}
+ // TEMPORARY: screenshots indexed before SCREENSHOT_INDEX_CUTOFF_MS have
+ // unreachable URLs, so never serve them for screenshot requests — drop them
+ // here so we waterfall to a fresh scrape that re-saves a working URL. The
+ // cache path already filters these via filterIndexEntries; this covers the
+ // DB path. Remove once pre-cutoff entries have aged out.
+ if (
+ meta.featureFlags.has("screenshot") ||
+ meta.featureFlags.has("screenshot@fullScreen")
+ ) {
+ data = data.filter(
+ x => new Date(x.created_at).getTime() >= SCREENSHOT_INDEX_CUTOFF_MS,
+ );
+ }
+
let selectedRow: {
id: string;
created_at: string;
diff --git a/apps/api/src/services/index-cache.ts b/apps/api/src/services/index-cache.ts
index 3786d526e..b0141f36d 100644
--- a/apps/api/src/services/index-cache.ts
+++ b/apps/api/src/services/index-cache.ts
@@ -100,11 +100,20 @@ export function deriveIndexVariantKey(params: {
);
}
+// TEMPORARY: screenshots indexed before this instant were stored with URLs
+// that are no longer reachable, so the index must not serve them when a
+// screenshot is requested — force a fresh scrape that re-saves a working URL
+// instead. The default 2-day index window means pre-cutoff entries age out
+// shortly after, so remove this constant (and the screenshotCutoffMs plumbing
+// in filterIndexEntries / scrapeURLWithIndex) once they're gone.
+export const SCREENSHOT_INDEX_CUTOFF_MS = Date.parse("2026-06-24T20:00:00Z");
+
// Mirrors the per-entry filters of index_get_recent_5 (see PR description for
// the SQL source): age window, screenshot capability (a request that doesn't
// need a screenshot matches any entry; one that does requires it), waitFor
// (COALESCE(entry, 0) >= requested), newest-first, LIMIT 5. Keep in sync with
-// the SQL function.
+// the SQL function. screenshotCutoffMs is a temporary app-level guard (no SQL
+// equivalent) that drops pre-cutoff screenshot entries.
export function filterIndexEntries(
entries: IndexCacheEntry[],
opts: {
@@ -113,6 +122,7 @@ export function filterIndexEntries(
needsScreenshot: boolean;
needsScreenshotFullscreen: boolean;
waitTimeMs: number | null;
+ screenshotCutoffMs?: number | null;
now?: number;
},
): IndexCacheEntry[] {
@@ -127,6 +137,12 @@ export function filterIndexEntries(
if (opts.needsScreenshot && !entry.has_screenshot) return false;
if (opts.needsScreenshotFullscreen && !entry.has_screenshot_fullscreen)
return false;
+ if (
+ (opts.needsScreenshot || opts.needsScreenshotFullscreen) &&
+ opts.screenshotCutoffMs != null &&
+ createdAt < opts.screenshotCutoffMs
+ )
+ return false;
if (
opts.waitTimeMs !== null &&
(entry.wait_time_ms ?? 0) < opts.waitTimeMs