Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): re-sign rotated screenshot URLs (#3880)
Gergő Móricz3117276fe4c3
diff --git a/apps/api/.env.example b/apps/api/.env.exampleindex 2da496768..7272572ed 100644--- a/apps/api/.env.example+++ b/apps/api/.env.example@@ -40,6 +40,9 @@ SUPABASE_SERVICE_TOKEN=# Other Optionals# use if you've set up authentication and want to test with a real API keyTEST_API_KEY=+# Re-sign cached index screenshot URLs for index entries created before this ISO timestamp.+# Example: 2026-06-24T20:00:00Z+GCS_SCREENSHOT_RESIGN_BEFORE=# set if you'd like to test the scraping rate limitRATE_LIMIT_TEST_API_KEY_SCRAPE=# set if you'd like to test the crawling rate limit@@ -126,4 +129,4 @@ CDP_API_KEY_SECRET=""# ^ KEYS taken from -> https://portal.cdp.coinbase.com/projects/api-keys## ENDPOINT PRICING TEST $0.01-X402_ENDPOINT_PRICE_USD=0.01\ No newline at end of file+X402_ENDPOINT_PRICE_USD=0.01diff --git a/apps/api/src/__tests__/snips/index-cache.test.ts b/apps/api/src/__tests__/snips/index-cache.test.tsindex 8d1baeff9..9d4366e41 100644--- a/apps/api/src/__tests__/snips/index-cache.test.ts+++ b/apps/api/src/__tests__/snips/index-cache.test.ts@@ -6,7 +6,6 @@ import {getCachedMaxAge,getCachedNegative,isNegativeStillValid,- SCREENSHOT_INDEX_CUTOFF_MS,setCachedNegative,upsertCachedIndexEntries,type IndexCacheEntry,@@ -173,82 +172,6 @@ describe("filterIndexEntries", () => {).toEqual([longWait.id]);});- it("drops pre-cutoff screenshot entries but keeps non-screenshot ones (temporary screenshot cutoff)", () => {- const cutoff = now - hour;- const preCutoff = entry({- has_screenshot: true,- created_at: new Date(cutoff - 1000).toISOString(),- });- const postCutoff = entry({- has_screenshot: true,- created_at: new Date(cutoff + 1000).toISOString(),- });- const opts = {- maxAgeMs: 24 * hour,- minAgeMs: null,- needsScreenshotFullscreen: false,- waitTimeMs: 0,- screenshotCutoffMs: cutoff,- now,- };- // Screenshot request: only the post-cutoff entry survives.- expect(- filterIndexEntries([preCutoff, postCutoff], {- ...opts,- needsScreenshot: true,- }).map(x => x.id),- ).toEqual([postCutoff.id]);- // Non-screenshot request: the cutoff doesn't apply, both survive.- expect(- filterIndexEntries([preCutoff, postCutoff], {- ...opts,- needsScreenshot: false,- }),- ).toHaveLength(2);- });-- it("applies the screenshot cutoff to fullscreen-screenshot requests too", () => {- const cutoff = now - hour;- const preCutoff = entry({- has_screenshot: true,- has_screenshot_fullscreen: true,- created_at: new Date(cutoff - 1000).toISOString(),- });- expect(- filterIndexEntries([preCutoff], {- maxAgeMs: 24 * hour,- minAgeMs: null,- needsScreenshot: false,- needsScreenshotFullscreen: true,- waitTimeMs: 0,- screenshotCutoffMs: cutoff,- now,- }),- ).toHaveLength(0);- });-- it("ignores the screenshot cutoff when screenshotCutoffMs is omitted", () => {- const preCutoff = entry({- has_screenshot: true,- created_at: new Date(now - 5 * hour).toISOString(),- });- expect(- filterIndexEntries([preCutoff], {- maxAgeMs: 24 * hour,- minAgeMs: null,- needsScreenshot: true,- needsScreenshotFullscreen: false,- waitTimeMs: 0,- now,- }),- ).toHaveLength(1);- });-- it("exports a sane screenshot cutoff constant", () => {- expect(Number.isNaN(SCREENSHOT_INDEX_CUTOFF_MS)).toBe(false);- expect(SCREENSHOT_INDEX_CUTOFF_MS).toBe(Date.parse("2026-06-24T20:00:00Z"));- });-it("sorts newest-first and caps at 5 like the SQL LIMIT", () => {const entries = Array.from({ length: 8 }, (_, i) =>entry({ created_at: new Date(now - i * 1000).toISOString() }),diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex 65d9e0213..b980d8bde 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -137,6 +137,7 @@ const configSchema = z.object({GCS_FIRE_ENGINE_BUCKET_NAME: z.string().optional(),GCS_INDEX_BUCKET_NAME: z.string().optional(),GCS_MEDIA_BUCKET_NAME: z.string().optional(),+ GCS_SCREENSHOT_RESIGN_BEFORE: emptyStringAsUndefined(z.string().datetime()),GCS_PARSE_UPLOAD_BUCKET_NAME: z.string().optional(),PARSE_UPLOAD_STORAGE_DRIVER: z.enum(["local", "gcs"]).optional(),PARSE_UPLOAD_REF_SECRET: emptyStringAsUndefined(z.string().trim().min(1)),diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.tsindex d56c60894..d35566936 100644--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts@@ -22,7 +22,6 @@ import {getCachedMaxAge,getCachedNegative,isNegativeStillValid,- SCREENSHOT_INDEX_CUTOFF_MS,setCachedMaxAge,setCachedNegative,upsertCachedIndexEntries,@@ -373,7 +372,6 @@ export async function scrapeURLWithIndex("screenshot@fullScreen",),waitTimeMs: meta.options.waitFor,- screenshotCutoffMs: SCREENSHOT_INDEX_CUTOFF_MS,});if (filtered.length > 0) {data = filtered;@@ -465,20 +463,6 @@ export async function scrapeURLWithIndex(}}- // TEMPORARY: screenshots indexed before SCREENSHOT_INDEX_CUTOFF_MS have- // unreachable URLs, so never serve them for screenshot requests — drop them- // here so we waterfall to a fresh scrape that re-saves a working URL. The- // cache path already filters these via filterIndexEntries; this covers the- // DB path. Remove once pre-cutoff entries have aged out.- if (- meta.featureFlags.has("screenshot") ||- meta.featureFlags.has("screenshot@fullScreen")- ) {- data = data.filter(- x => new Date(x.created_at).getTime() >= SCREENSHOT_INDEX_CUTOFF_MS,- );- }-let selectedRow: {id: string;created_at: string;@@ -519,6 +503,7 @@ export async function scrapeURLWithIndex(const doc = await getIndexFromGCS(id + ".json",meta.logger.child({ module: "index", method: "getIndexFromGCS" }),+ { indexCreatedAt: selectedRow.created_at },);if (!doc) {if (servedFromCache) {diff --git a/apps/api/src/search/highlights.ts b/apps/api/src/search/highlights.tsindex 60291a196..2964d1f6e 100644--- a/apps/api/src/search/highlights.ts+++ b/apps/api/src/search/highlights.ts@@ -88,6 +88,7 @@ async function getIndexedMarkdownForURL(const doc = await getIndexFromGCS(selected.id + ".json",logger.child({ module: "search/highlights", method: "getIndexFromGCS" }),+ { indexCreatedAt: selected.created_at },);if (!doc || !doc.html) {return null;diff --git a/apps/api/src/services/index-cache.ts b/apps/api/src/services/index-cache.tsindex b0141f36d..3786d526e 100644--- a/apps/api/src/services/index-cache.ts+++ b/apps/api/src/services/index-cache.ts@@ -100,20 +100,11 @@ export function deriveIndexVariantKey(params: {);}-// TEMPORARY: screenshots indexed before this instant were stored with URLs-// that are no longer reachable, so the index must not serve them when a-// screenshot is requested — force a fresh scrape that re-saves a working URL-// instead. The default 2-day index window means pre-cutoff entries age out-// shortly after, so remove this constant (and the screenshotCutoffMs plumbing-// in filterIndexEntries / scrapeURLWithIndex) once they're gone.-export const SCREENSHOT_INDEX_CUTOFF_MS = Date.parse("2026-06-24T20:00:00Z");-// Mirrors the per-entry filters of index_get_recent_5 (see PR description for// the SQL source): age window, screenshot capability (a request that doesn't// need a screenshot matches any entry; one that does requires it), waitFor// (COALESCE(entry, 0) >= requested), newest-first, LIMIT 5. Keep in sync with-// the SQL function. screenshotCutoffMs is a temporary app-level guard (no SQL-// equivalent) that drops pre-cutoff screenshot entries.+// the SQL function.export function filterIndexEntries(entries: IndexCacheEntry[],opts: {@@ -122,7 +113,6 @@ export function filterIndexEntries(needsScreenshot: boolean;needsScreenshotFullscreen: boolean;waitTimeMs: number | null;- screenshotCutoffMs?: number | null;now?: number;},): IndexCacheEntry[] {@@ -137,12 +127,6 @@ export function filterIndexEntries(if (opts.needsScreenshot && !entry.has_screenshot) return false;if (opts.needsScreenshotFullscreen && !entry.has_screenshot_fullscreen)return false;- if (- (opts.needsScreenshot || opts.needsScreenshotFullscreen) &&- opts.screenshotCutoffMs != null &&- createdAt < opts.screenshotCutoffMs- )- return false;if (opts.waitTimeMs !== null &&(entry.wait_time_ms ?? 0) < opts.waitTimeMsdiff --git a/apps/api/src/services/index-screenshot-url.test.ts b/apps/api/src/services/index-screenshot-url.test.tsnew file mode 100644index 000000000..78b4e6e03--- /dev/null+++ b/apps/api/src/services/index-screenshot-url.test.ts@@ -0,0 +1,75 @@+import { describe, expect, it } from "vitest";+import {+ getGcsScreenshotUrlResignReason,+ getGcsSignedUrlExpiration,+} from "./index-screenshot-url";++describe("index screenshot signed URLs", () => {+ it("parses v2 and v4 expiration times", () => {+ const v2Url = new URL(+ "https://storage.googleapis.com/media-bucket/screenshot.png?Expires=4102444800",+ );+ const v4Url = new URL(+ "https://storage.googleapis.com/media-bucket/screenshot.png?X-Goog-Date=20300101T000000Z&X-Goog-Expires=60",+ );++ expect(getGcsSignedUrlExpiration(v2Url)).toBe(4102444800000);+ expect(getGcsSignedUrlExpiration(v4Url)).toBe(+ Date.UTC(2030, 0, 1, 0, 1, 0),+ );+ });++ it("requests re-signing when the URL is expired", () => {+ const url = new URL(+ "https://storage.googleapis.com/media-bucket/screenshot.png?GoogleAccessId=current%40project.iam.gserviceaccount.com&Expires=100",+ );++ expect(+ getGcsScreenshotUrlResignReason(url, {+ now: 101000,+ }),+ ).toBe("expired");+ });++ it("requests re-signing when the index entry predates GCS_SCREENSHOT_RESIGN_BEFORE", () => {+ const url = new URL(+ "https://storage.googleapis.com/media-bucket/screenshot.png?GoogleAccessId=current%40project.iam.gserviceaccount.com&Expires=4102444800",+ );++ expect(+ getGcsScreenshotUrlResignReason(url, {+ now: Date.UTC(2026, 5, 25),+ indexCreatedAt: "2026-06-24T19:59:59.000Z",+ resignBefore: "2026-06-24T20:00:00.000Z",+ }),+ ).toBe("resign_before");+ });++ it("does not re-sign when the index entry is on or after GCS_SCREENSHOT_RESIGN_BEFORE", () => {+ const url = new URL(+ "https://storage.googleapis.com/media-bucket/screenshot.png?GoogleAccessId=current%40project.iam.gserviceaccount.com&Expires=4102444800",+ );++ expect(+ getGcsScreenshotUrlResignReason(url, {+ now: Date.UTC(2026, 5, 25),+ indexCreatedAt: "2026-06-24T20:00:00.000Z",+ resignBefore: "2026-06-24T20:00:00.000Z",+ }),+ ).toBeNull();+ });++ it("does not re-sign non-GCS URLs before GCS_SCREENSHOT_RESIGN_BEFORE", () => {+ const url = new URL(+ "https://cdn.example.com/screenshot.png?GoogleAccessId=current%40project.iam.gserviceaccount.com&Expires=100",+ );++ expect(+ getGcsScreenshotUrlResignReason(url, {+ now: 101000,+ indexCreatedAt: "2026-06-24T19:59:59.000Z",+ resignBefore: "2026-06-24T20:00:00.000Z",+ }),+ ).toBeNull();+ });+});diff --git a/apps/api/src/services/index-screenshot-url.ts b/apps/api/src/services/index-screenshot-url.tsnew file mode 100644index 000000000..bf5a1e21c--- /dev/null+++ b/apps/api/src/services/index-screenshot-url.ts@@ -0,0 +1,75 @@+import { config } from "../config";++function parseGcsSignedUrlDate(value: string | null): number {+ if (!value) {+ return NaN;+ }++ const compactDate = value.match(+ /^(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})Z$/,+ );+ if (compactDate) {+ const [, year, month, day, hour, minute, second] = compactDate;+ return Date.UTC(+ parseInt(year, 10),+ parseInt(month, 10) - 1,+ parseInt(day, 10),+ parseInt(hour, 10),+ parseInt(minute, 10),+ parseInt(second, 10),+ );+ }++ return new Date(value).getTime();+}++function parseTimestamp(value: string | null | undefined): number | null {+ if (!value) {+ return null;+ }++ const parsed = parseGcsSignedUrlDate(value);+ return Number.isNaN(parsed) ? null : parsed;+}++export function getGcsSignedUrlExpiration(url: URL): number {+ let expiresAt = parseInt(url.searchParams.get("Expires") ?? "0", 10) * 1000;+ if (expiresAt === 0) {+ expiresAt =+ parseGcsSignedUrlDate(url.searchParams.get("X-Goog-Date")) ++ parseInt(url.searchParams.get("X-Goog-Expires") ?? "0", 10) * 1000;+ }++ return expiresAt;+}++export function getGcsScreenshotUrlResignReason(+ url: URL,+ opts: {+ indexCreatedAt?: string | null;+ now?: number;+ resignBefore?: string | null;+ } = {},+): "expired" | "resign_before" | null {+ if (url.hostname !== "storage.googleapis.com") {+ return null;+ }++ if (getGcsSignedUrlExpiration(url) < (opts.now ?? Date.now())) {+ return "expired";+ }++ const resignBeforeMs = parseTimestamp(+ opts.resignBefore ?? config.GCS_SCREENSHOT_RESIGN_BEFORE,+ );+ const indexCreatedAtMs = parseTimestamp(opts.indexCreatedAt);+ if (+ resignBeforeMs !== null &&+ indexCreatedAtMs !== null &&+ indexCreatedAtMs < resignBeforeMs+ ) {+ return "resign_before";+ }++ return null;+}diff --git a/apps/api/src/services/index.ts b/apps/api/src/services/index.tsindex 379a90f68..793ce4db9 100644--- a/apps/api/src/services/index.ts+++ b/apps/api/src/services/index.ts@@ -28,11 +28,13 @@ import type { PdfMetadata } from "../scraper/scrapeURL/engines/pdf/types";import { storage } from "../lib/gcs-jobs";import { withSpan, setSpanAttributes } from "../lib/otel-tracer";import { config } from "../config";+import { getGcsScreenshotUrlResignReason } from "./index-screenshot-url";configDotenv();export async function getIndexFromGCS(url: string,logger?: Logger,+ opts: { indexCreatedAt?: string | null } = {},): Promise<any | null> {try {return await withSpan("firecrawl-index-get-from-gcs", async span => {@@ -54,26 +56,11 @@ export async function getIndexFromGCS(if (typeof parsed.screenshot === "string") {try {const screenshotUrl = new URL(parsed.screenshot);- let expiresAt =- parseInt(screenshotUrl.searchParams.get("Expires") ?? "0", 10) *- 1000;- if (expiresAt === 0) {- expiresAt =- new Date(- screenshotUrl.searchParams.get("X-Goog-Date") ??- "1970-01-01T00:00:00Z",- ).getTime() +- parseInt(- screenshotUrl.searchParams.get("X-Goog-Expires") ?? "0",- 10,- ) *- 1000;- }- if (- screenshotUrl.hostname === "storage.googleapis.com" &&- expiresAt < Date.now()- ) {- logger?.info("Re-signing screenshot URL");+ const resignReason = getGcsScreenshotUrlResignReason(screenshotUrl, {+ indexCreatedAt: opts.indexCreatedAt,+ });+ if (resignReason !== null) {+ logger?.info("Re-signing screenshot URL", { reason: resignReason });const filePath = decodeURIComponent(screenshotUrl.pathname.split("/")[2],);