Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): re-sign rotated screenshot URLs (#3880)

Gergő Móricz3117276fe4c3
diff --git a/apps/api/.env.example b/apps/api/.env.example
index 2da496768..7272572ed 100644
--- a/apps/api/.env.example
+++ b/apps/api/.env.example
@@ -40,6 +40,9 @@ SUPABASE_SERVICE_TOKEN=
# Other Optionals
# use if you've set up authentication and want to test with a real API key
TEST_API_KEY=
+# Re-sign cached index screenshot URLs for index entries created before this ISO timestamp.
+# Example: 2026-06-24T20:00:00Z
+GCS_SCREENSHOT_RESIGN_BEFORE=
# set if you'd like to test the scraping rate limit
RATE_LIMIT_TEST_API_KEY_SCRAPE=
# set if you'd like to test the crawling rate limit
@@ -126,4 +129,4 @@ CDP_API_KEY_SECRET=""
# ^ KEYS taken from -> https://portal.cdp.coinbase.com/projects/api-keys
## ENDPOINT PRICING TEST $0.01
-X402_ENDPOINT_PRICE_USD=0.01
\ No newline at end of file
+X402_ENDPOINT_PRICE_USD=0.01
diff --git a/apps/api/src/__tests__/snips/index-cache.test.ts b/apps/api/src/__tests__/snips/index-cache.test.ts
index 8d1baeff9..9d4366e41 100644
--- a/apps/api/src/__tests__/snips/index-cache.test.ts
+++ b/apps/api/src/__tests__/snips/index-cache.test.ts
@@ -6,7 +6,6 @@ import {
getCachedMaxAge,
getCachedNegative,
isNegativeStillValid,
- SCREENSHOT_INDEX_CUTOFF_MS,
setCachedNegative,
upsertCachedIndexEntries,
type IndexCacheEntry,
@@ -173,82 +172,6 @@ describe("filterIndexEntries", () => {
).toEqual([longWait.id]);
});
- it("drops pre-cutoff screenshot entries but keeps non-screenshot ones (temporary screenshot cutoff)", () => {
- const cutoff = now - hour;
- const preCutoff = entry({
- has_screenshot: true,
- created_at: new Date(cutoff - 1000).toISOString(),
- });
- const postCutoff = entry({
- has_screenshot: true,
- created_at: new Date(cutoff + 1000).toISOString(),
- });
- const opts = {
- maxAgeMs: 24 * hour,
- minAgeMs: null,
- needsScreenshotFullscreen: false,
- waitTimeMs: 0,
- screenshotCutoffMs: cutoff,
- now,
- };
- // Screenshot request: only the post-cutoff entry survives.
- expect(
- filterIndexEntries([preCutoff, postCutoff], {
- ...opts,
- needsScreenshot: true,
- }).map(x => x.id),
- ).toEqual([postCutoff.id]);
- // Non-screenshot request: the cutoff doesn't apply, both survive.
- expect(
- filterIndexEntries([preCutoff, postCutoff], {
- ...opts,
- needsScreenshot: false,
- }),
- ).toHaveLength(2);
- });
-
- it("applies the screenshot cutoff to fullscreen-screenshot requests too", () => {
- const cutoff = now - hour;
- const preCutoff = entry({
- has_screenshot: true,
- has_screenshot_fullscreen: true,
- created_at: new Date(cutoff - 1000).toISOString(),
- });
- expect(
- filterIndexEntries([preCutoff], {
- maxAgeMs: 24 * hour,
- minAgeMs: null,
- needsScreenshot: false,
- needsScreenshotFullscreen: true,
- waitTimeMs: 0,
- screenshotCutoffMs: cutoff,
- now,
- }),
- ).toHaveLength(0);
- });
-
- it("ignores the screenshot cutoff when screenshotCutoffMs is omitted", () => {
- const preCutoff = entry({
- has_screenshot: true,
- created_at: new Date(now - 5 * hour).toISOString(),
- });
- expect(
- filterIndexEntries([preCutoff], {
- maxAgeMs: 24 * hour,
- minAgeMs: null,
- needsScreenshot: true,
- needsScreenshotFullscreen: false,
- waitTimeMs: 0,
- now,
- }),
- ).toHaveLength(1);
- });
-
- it("exports a sane screenshot cutoff constant", () => {
- expect(Number.isNaN(SCREENSHOT_INDEX_CUTOFF_MS)).toBe(false);
- expect(SCREENSHOT_INDEX_CUTOFF_MS).toBe(Date.parse("2026-06-24T20:00:00Z"));
- });
-
it("sorts newest-first and caps at 5 like the SQL LIMIT", () => {
const entries = Array.from({ length: 8 }, (_, i) =>
entry({ created_at: new Date(now - i * 1000).toISOString() }),
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index 65d9e0213..b980d8bde 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -137,6 +137,7 @@ const configSchema = z.object({
GCS_FIRE_ENGINE_BUCKET_NAME: z.string().optional(),
GCS_INDEX_BUCKET_NAME: z.string().optional(),
GCS_MEDIA_BUCKET_NAME: z.string().optional(),
+ GCS_SCREENSHOT_RESIGN_BEFORE: emptyStringAsUndefined(z.string().datetime()),
GCS_PARSE_UPLOAD_BUCKET_NAME: z.string().optional(),
PARSE_UPLOAD_STORAGE_DRIVER: z.enum(["local", "gcs"]).optional(),
PARSE_UPLOAD_REF_SECRET: emptyStringAsUndefined(z.string().trim().min(1)),
diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
index d56c60894..d35566936 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
@@ -22,7 +22,6 @@ import {
getCachedMaxAge,
getCachedNegative,
isNegativeStillValid,
- SCREENSHOT_INDEX_CUTOFF_MS,
setCachedMaxAge,
setCachedNegative,
upsertCachedIndexEntries,
@@ -373,7 +372,6 @@ export async function scrapeURLWithIndex(
"screenshot@fullScreen",
),
waitTimeMs: meta.options.waitFor,
- screenshotCutoffMs: SCREENSHOT_INDEX_CUTOFF_MS,
});
if (filtered.length > 0) {
data = filtered;
@@ -465,20 +463,6 @@ export async function scrapeURLWithIndex(
}
}
- // TEMPORARY: screenshots indexed before SCREENSHOT_INDEX_CUTOFF_MS have
- // unreachable URLs, so never serve them for screenshot requests — drop them
- // here so we waterfall to a fresh scrape that re-saves a working URL. The
- // cache path already filters these via filterIndexEntries; this covers the
- // DB path. Remove once pre-cutoff entries have aged out.
- if (
- meta.featureFlags.has("screenshot") ||
- meta.featureFlags.has("screenshot@fullScreen")
- ) {
- data = data.filter(
- x => new Date(x.created_at).getTime() >= SCREENSHOT_INDEX_CUTOFF_MS,
- );
- }
-
let selectedRow: {
id: string;
created_at: string;
@@ -519,6 +503,7 @@ export async function scrapeURLWithIndex(
const doc = await getIndexFromGCS(
id + ".json",
meta.logger.child({ module: "index", method: "getIndexFromGCS" }),
+ { indexCreatedAt: selectedRow.created_at },
);
if (!doc) {
if (servedFromCache) {
diff --git a/apps/api/src/search/highlights.ts b/apps/api/src/search/highlights.ts
index 60291a196..2964d1f6e 100644
--- a/apps/api/src/search/highlights.ts
+++ b/apps/api/src/search/highlights.ts
@@ -88,6 +88,7 @@ async function getIndexedMarkdownForURL(
const doc = await getIndexFromGCS(
selected.id + ".json",
logger.child({ module: "search/highlights", method: "getIndexFromGCS" }),
+ { indexCreatedAt: selected.created_at },
);
if (!doc || !doc.html) {
return null;
diff --git a/apps/api/src/services/index-cache.ts b/apps/api/src/services/index-cache.ts
index b0141f36d..3786d526e 100644
--- a/apps/api/src/services/index-cache.ts
+++ b/apps/api/src/services/index-cache.ts
@@ -100,20 +100,11 @@ export function deriveIndexVariantKey(params: {
);
}
-// TEMPORARY: screenshots indexed before this instant were stored with URLs
-// that are no longer reachable, so the index must not serve them when a
-// screenshot is requested — force a fresh scrape that re-saves a working URL
-// instead. The default 2-day index window means pre-cutoff entries age out
-// shortly after, so remove this constant (and the screenshotCutoffMs plumbing
-// in filterIndexEntries / scrapeURLWithIndex) once they're gone.
-export const SCREENSHOT_INDEX_CUTOFF_MS = Date.parse("2026-06-24T20:00:00Z");
-
// Mirrors the per-entry filters of index_get_recent_5 (see PR description for
// the SQL source): age window, screenshot capability (a request that doesn't
// need a screenshot matches any entry; one that does requires it), waitFor
// (COALESCE(entry, 0) >= requested), newest-first, LIMIT 5. Keep in sync with
-// the SQL function. screenshotCutoffMs is a temporary app-level guard (no SQL
-// equivalent) that drops pre-cutoff screenshot entries.
+// the SQL function.
export function filterIndexEntries(
entries: IndexCacheEntry[],
opts: {
@@ -122,7 +113,6 @@ export function filterIndexEntries(
needsScreenshot: boolean;
needsScreenshotFullscreen: boolean;
waitTimeMs: number | null;
- screenshotCutoffMs?: number | null;
now?: number;
},
): IndexCacheEntry[] {
@@ -137,12 +127,6 @@ export function filterIndexEntries(
if (opts.needsScreenshot && !entry.has_screenshot) return false;
if (opts.needsScreenshotFullscreen && !entry.has_screenshot_fullscreen)
return false;
- if (
- (opts.needsScreenshot || opts.needsScreenshotFullscreen) &&
- opts.screenshotCutoffMs != null &&
- createdAt < opts.screenshotCutoffMs
- )
- return false;
if (
opts.waitTimeMs !== null &&
(entry.wait_time_ms ?? 0) < opts.waitTimeMs
diff --git a/apps/api/src/services/index-screenshot-url.test.ts b/apps/api/src/services/index-screenshot-url.test.ts
new file mode 100644
index 000000000..78b4e6e03
--- /dev/null
+++ b/apps/api/src/services/index-screenshot-url.test.ts
@@ -0,0 +1,75 @@
+import { describe, expect, it } from "vitest";
+import {
+ getGcsScreenshotUrlResignReason,
+ getGcsSignedUrlExpiration,
+} from "./index-screenshot-url";
+
+describe("index screenshot signed URLs", () => {
+ it("parses v2 and v4 expiration times", () => {
+ const v2Url = new URL(
+ "https://storage.googleapis.com/media-bucket/screenshot.png?Expires=4102444800",
+ );
+ const v4Url = new URL(
+ "https://storage.googleapis.com/media-bucket/screenshot.png?X-Goog-Date=20300101T000000Z&X-Goog-Expires=60",
+ );
+
+ expect(getGcsSignedUrlExpiration(v2Url)).toBe(4102444800000);
+ expect(getGcsSignedUrlExpiration(v4Url)).toBe(
+ Date.UTC(2030, 0, 1, 0, 1, 0),
+ );
+ });
+
+ it("requests re-signing when the URL is expired", () => {
+ const url = new URL(
+ "https://storage.googleapis.com/media-bucket/screenshot.png?GoogleAccessId=current%40project.iam.gserviceaccount.com&Expires=100",
+ );
+
+ expect(
+ getGcsScreenshotUrlResignReason(url, {
+ now: 101000,
+ }),
+ ).toBe("expired");
+ });
+
+ it("requests re-signing when the index entry predates GCS_SCREENSHOT_RESIGN_BEFORE", () => {
+ const url = new URL(
+ "https://storage.googleapis.com/media-bucket/screenshot.png?GoogleAccessId=current%40project.iam.gserviceaccount.com&Expires=4102444800",
+ );
+
+ expect(
+ getGcsScreenshotUrlResignReason(url, {
+ now: Date.UTC(2026, 5, 25),
+ indexCreatedAt: "2026-06-24T19:59:59.000Z",
+ resignBefore: "2026-06-24T20:00:00.000Z",
+ }),
+ ).toBe("resign_before");
+ });
+
+ it("does not re-sign when the index entry is on or after GCS_SCREENSHOT_RESIGN_BEFORE", () => {
+ const url = new URL(
+ "https://storage.googleapis.com/media-bucket/screenshot.png?GoogleAccessId=current%40project.iam.gserviceaccount.com&Expires=4102444800",
+ );
+
+ expect(
+ getGcsScreenshotUrlResignReason(url, {
+ now: Date.UTC(2026, 5, 25),
+ indexCreatedAt: "2026-06-24T20:00:00.000Z",
+ resignBefore: "2026-06-24T20:00:00.000Z",
+ }),
+ ).toBeNull();
+ });
+
+ it("does not re-sign non-GCS URLs before GCS_SCREENSHOT_RESIGN_BEFORE", () => {
+ const url = new URL(
+ "https://cdn.example.com/screenshot.png?GoogleAccessId=current%40project.iam.gserviceaccount.com&Expires=100",
+ );
+
+ expect(
+ getGcsScreenshotUrlResignReason(url, {
+ now: 101000,
+ indexCreatedAt: "2026-06-24T19:59:59.000Z",
+ resignBefore: "2026-06-24T20:00:00.000Z",
+ }),
+ ).toBeNull();
+ });
+});
diff --git a/apps/api/src/services/index-screenshot-url.ts b/apps/api/src/services/index-screenshot-url.ts
new file mode 100644
index 000000000..bf5a1e21c
--- /dev/null
+++ b/apps/api/src/services/index-screenshot-url.ts
@@ -0,0 +1,75 @@
+import { config } from "../config";
+
+function parseGcsSignedUrlDate(value: string | null): number {
+ if (!value) {
+ return NaN;
+ }
+
+ const compactDate = value.match(
+ /^(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})Z$/,
+ );
+ if (compactDate) {
+ const [, year, month, day, hour, minute, second] = compactDate;
+ return Date.UTC(
+ parseInt(year, 10),
+ parseInt(month, 10) - 1,
+ parseInt(day, 10),
+ parseInt(hour, 10),
+ parseInt(minute, 10),
+ parseInt(second, 10),
+ );
+ }
+
+ return new Date(value).getTime();
+}
+
+function parseTimestamp(value: string | null | undefined): number | null {
+ if (!value) {
+ return null;
+ }
+
+ const parsed = parseGcsSignedUrlDate(value);
+ return Number.isNaN(parsed) ? null : parsed;
+}
+
+export function getGcsSignedUrlExpiration(url: URL): number {
+ let expiresAt = parseInt(url.searchParams.get("Expires") ?? "0", 10) * 1000;
+ if (expiresAt === 0) {
+ expiresAt =
+ parseGcsSignedUrlDate(url.searchParams.get("X-Goog-Date")) +
+ parseInt(url.searchParams.get("X-Goog-Expires") ?? "0", 10) * 1000;
+ }
+
+ return expiresAt;
+}
+
+export function getGcsScreenshotUrlResignReason(
+ url: URL,
+ opts: {
+ indexCreatedAt?: string | null;
+ now?: number;
+ resignBefore?: string | null;
+ } = {},
+): "expired" | "resign_before" | null {
+ if (url.hostname !== "storage.googleapis.com") {
+ return null;
+ }
+
+ if (getGcsSignedUrlExpiration(url) < (opts.now ?? Date.now())) {
+ return "expired";
+ }
+
+ const resignBeforeMs = parseTimestamp(
+ opts.resignBefore ?? config.GCS_SCREENSHOT_RESIGN_BEFORE,
+ );
+ const indexCreatedAtMs = parseTimestamp(opts.indexCreatedAt);
+ if (
+ resignBeforeMs !== null &&
+ indexCreatedAtMs !== null &&
+ indexCreatedAtMs < resignBeforeMs
+ ) {
+ return "resign_before";
+ }
+
+ return null;
+}
diff --git a/apps/api/src/services/index.ts b/apps/api/src/services/index.ts
index 379a90f68..793ce4db9 100644
--- a/apps/api/src/services/index.ts
+++ b/apps/api/src/services/index.ts
@@ -28,11 +28,13 @@ import type { PdfMetadata } from "../scraper/scrapeURL/engines/pdf/types";
import { storage } from "../lib/gcs-jobs";
import { withSpan, setSpanAttributes } from "../lib/otel-tracer";
import { config } from "../config";
+import { getGcsScreenshotUrlResignReason } from "./index-screenshot-url";
configDotenv();
export async function getIndexFromGCS(
url: string,
logger?: Logger,
+ opts: { indexCreatedAt?: string | null } = {},
): Promise<any | null> {
try {
return await withSpan("firecrawl-index-get-from-gcs", async span => {
@@ -54,26 +56,11 @@ export async function getIndexFromGCS(
if (typeof parsed.screenshot === "string") {
try {
const screenshotUrl = new URL(parsed.screenshot);
- let expiresAt =
- parseInt(screenshotUrl.searchParams.get("Expires") ?? "0", 10) *
- 1000;
- if (expiresAt === 0) {
- expiresAt =
- new Date(
- screenshotUrl.searchParams.get("X-Goog-Date") ??
- "1970-01-01T00:00:00Z",
- ).getTime() +
- parseInt(
- screenshotUrl.searchParams.get("X-Goog-Expires") ?? "0",
- 10,
- ) *
- 1000;
- }
- if (
- screenshotUrl.hostname === "storage.googleapis.com" &&
- expiresAt < Date.now()
- ) {
- logger?.info("Re-signing screenshot URL");
+ const resignReason = getGcsScreenshotUrlResignReason(screenshotUrl, {
+ indexCreatedAt: opts.indexCreatedAt,
+ });
+ if (resignReason !== null) {
+ logger?.info("Re-signing screenshot URL", { reason: resignReason });
const filePath = decodeURIComponent(
screenshotUrl.pathname.split("/")[2],
);