Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api/scrapeURL/index): re-sign expired screenshot URLs (#2080)

Gergő Móricz4291af2c3f7e
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.yml
index 7c62260b3..2202fcf27 100644
--- a/.github/workflows/test-server.yml
+++ b/.github/workflows/test-server.yml
@@ -36,6 +36,7 @@ env:
GCS_CREDENTIALS: ${{ secrets.GCS_CREDENTIALS }}
GCS_BUCKET_NAME: ${{ secrets.GCS_BUCKET_NAME }}
GCS_INDEX_BUCKET_NAME: ${{ secrets.GCS_INDEX_BUCKET_NAME }}
+ GCS_MEDIA_BUCKET_NAME: ${{ secrets.GCS_MEDIA_BUCKET_NAME }}
GOOGLE_GENERATIVE_AI_API_KEY: ${{ secrets.GOOGLE_GENERATIVE_AI_API_KEY }}
GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
diff --git a/apps/api/src/lib/gcs-jobs.ts b/apps/api/src/lib/gcs-jobs.ts
index 79e02a2f0..ccfbc62ba 100644
--- a/apps/api/src/lib/gcs-jobs.ts
+++ b/apps/api/src/lib/gcs-jobs.ts
@@ -4,7 +4,7 @@ import { logger } from "./logger";
import { Document } from "../controllers/v1/types";
const credentials = process.env.GCS_CREDENTIALS ? JSON.parse(atob(process.env.GCS_CREDENTIALS)) : undefined;
-const storage = new Storage({ credentials });
+export const storage = new Storage({ credentials });
export async function saveJobToGCS(job: FirecrawlJob): Promise<void> {
try {
diff --git a/apps/api/src/lib/gcs-pdf-cache.ts b/apps/api/src/lib/gcs-pdf-cache.ts
index d7446f600..fb0241f99 100644
--- a/apps/api/src/lib/gcs-pdf-cache.ts
+++ b/apps/api/src/lib/gcs-pdf-cache.ts
@@ -1,6 +1,6 @@
-import { Storage } from "@google-cloud/storage";
import { logger } from "./logger";
import crypto from "crypto";
+import { storage } from "./gcs-jobs";
const credentials = process.env.GCS_CREDENTIALS ? JSON.parse(atob(process.env.GCS_CREDENTIALS)) : undefined;
const PDF_CACHE_PREFIX = "pdf-cache-v2/";
@@ -29,7 +29,6 @@ export async function savePdfResultToCache(
}
const cacheKey = createPdfCacheKey(pdfContent);
- const storage = new Storage({ credentials });
const bucket = storage.bucket(process.env.GCS_BUCKET_NAME);
const blob = bucket.file(`${PDF_CACHE_PREFIX}${cacheKey}.json`);
@@ -83,7 +82,6 @@ export async function getPdfResultFromCache(
}
const cacheKey = createPdfCacheKey(pdfContent);
- const storage = new Storage({ credentials });
const bucket = storage.bucket(process.env.GCS_BUCKET_NAME);
const blob = bucket.file(`${PDF_CACHE_PREFIX}${cacheKey}.json`);
diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
index 06364cb6a..6f8a1df1e 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
@@ -1,10 +1,11 @@
+import crypto from "crypto";
import { Document } from "../../../../controllers/v1/types";
import { EngineScrapeResult } from "..";
import { Meta } from "../..";
import { getIndexFromGCS, hashURL, index_supabase_service, normalizeURLForIndex, saveIndexToGCS, generateURLSplits, addIndexInsertJob, generateDomainSplits, addOMCEJob, addDomainFrequencyJob } from "../../../../services";
import { EngineError, IndexMissError } from "../../error";
import { shouldParsePDF } from "../../../../controllers/v2/types";
-import crypto from "crypto";
+import { storage } from "../../../../lib/gcs-jobs";
export async function sendDocumentToIndex(meta: Meta, document: Document) {
const shouldCache = meta.options.storeInCache
diff --git a/apps/api/src/services/index.ts b/apps/api/src/services/index.ts
index a16a229fc..dc8a803f1 100644
--- a/apps/api/src/services/index.ts
+++ b/apps/api/src/services/index.ts
@@ -1,13 +1,14 @@
import { createClient, SupabaseClient } from "@supabase/supabase-js";
import { logger as _logger } from "../lib/logger";
import { configDotenv } from "dotenv";
-import { ApiError, Storage } from "@google-cloud/storage";
+import { ApiError } from "@google-cloud/storage";
import crypto from "crypto";
import { redisEvictConnection } from "./redis";
import type { Logger } from "winston";
import psl from "psl";
import { MapDocument } from "../controllers/v2/types";
import { PDFMetadata } from "../lib/pdf-parser";
+import { storage } from "../lib/gcs-jobs";
configDotenv();
// SupabaseService class initializes the Supabase client conditionally based on environment variables.
@@ -71,11 +72,41 @@ export async function getIndexFromGCS(url: string, logger?: Logger): Promise<any
return null;
}
- const storage = new Storage({ credentials });
const bucket = storage.bucket(process.env.GCS_INDEX_BUCKET_NAME);
const blob = bucket.file(`${url}`);
const [blobContent] = await blob.download();
const parsed = JSON.parse(blobContent.toString());
+
+ try {
+ if (
+ typeof parsed.screenshot === "string"
+ ) {
+ const screenshotUrl = new URL(parsed.screenshot);
+ let expiresAt = parseInt(screenshotUrl.searchParams.get("Expires") ?? "0", 10) * 1000;
+ if (expiresAt === 0) {
+ expiresAt = new Date(screenshotUrl.searchParams.get("X-Goog-Date") ?? "1970-01-01T00:00:00Z").getTime() + parseInt(screenshotUrl.searchParams.get("X-Goog-Expires") ?? "0", 10) * 1000;
+ }
+ if (screenshotUrl.hostname === "storage.googleapis.com" && expiresAt < Date.now()) {
+ logger?.info("Re-signing screenshot URL");
+ const [url] = await storage.bucket(process.env.GCS_MEDIA_BUCKET_NAME!).file(decodeURIComponent(screenshotUrl.pathname.split("/")[2])).getSignedUrl({
+ action: "read",
+ expires: Date.now() + 1000 * 60 * 60 * 24 * 7, // 7 days
+ });
+ parsed.screenshot = url;
+
+ // Update the blob
+ await blob.save(JSON.stringify(parsed), {
+ contentType: "application/json",
+ });
+ }
+ }
+ } catch (error) {
+ logger?.warn("Error re-signing screenshot URL", {
+ error,
+ url,
+ });
+ }
+
return parsed;
} catch (error) {
if (error instanceof ApiError && error.code === 404 && error.message.includes("No such object:")) {
@@ -106,7 +137,6 @@ export async function saveIndexToGCS(id: string, doc: {
return;
}
- const storage = new Storage({ credentials });
const bucket = storage.bucket(process.env.GCS_INDEX_BUCKET_NAME);
const blob = bucket.file(`${id}.json`);
for (let i = 0; i < 3; i++) {