Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api/scrapeURL/index): re-sign expired screenshot URLs (#2080)
Gergő Móricz4291af2c3f7e
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.ymlindex 7c62260b3..2202fcf27 100644--- a/.github/workflows/test-server.yml+++ b/.github/workflows/test-server.yml@@ -36,6 +36,7 @@ env:GCS_CREDENTIALS: ${{ secrets.GCS_CREDENTIALS }}GCS_BUCKET_NAME: ${{ secrets.GCS_BUCKET_NAME }}GCS_INDEX_BUCKET_NAME: ${{ secrets.GCS_INDEX_BUCKET_NAME }}+ GCS_MEDIA_BUCKET_NAME: ${{ secrets.GCS_MEDIA_BUCKET_NAME }}GOOGLE_GENERATIVE_AI_API_KEY: ${{ secrets.GOOGLE_GENERATIVE_AI_API_KEY }}GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}diff --git a/apps/api/src/lib/gcs-jobs.ts b/apps/api/src/lib/gcs-jobs.tsindex 79e02a2f0..ccfbc62ba 100644--- a/apps/api/src/lib/gcs-jobs.ts+++ b/apps/api/src/lib/gcs-jobs.ts@@ -4,7 +4,7 @@ import { logger } from "./logger";import { Document } from "../controllers/v1/types";const credentials = process.env.GCS_CREDENTIALS ? JSON.parse(atob(process.env.GCS_CREDENTIALS)) : undefined;-const storage = new Storage({ credentials });+export const storage = new Storage({ credentials });export async function saveJobToGCS(job: FirecrawlJob): Promise<void> {try {diff --git a/apps/api/src/lib/gcs-pdf-cache.ts b/apps/api/src/lib/gcs-pdf-cache.tsindex d7446f600..fb0241f99 100644--- a/apps/api/src/lib/gcs-pdf-cache.ts+++ b/apps/api/src/lib/gcs-pdf-cache.ts@@ -1,6 +1,6 @@-import { Storage } from "@google-cloud/storage";import { logger } from "./logger";import crypto from "crypto";+import { storage } from "./gcs-jobs";const credentials = process.env.GCS_CREDENTIALS ? JSON.parse(atob(process.env.GCS_CREDENTIALS)) : undefined;const PDF_CACHE_PREFIX = "pdf-cache-v2/";@@ -29,7 +29,6 @@ export async function savePdfResultToCache(}const cacheKey = createPdfCacheKey(pdfContent);- const storage = new Storage({ credentials });const bucket = storage.bucket(process.env.GCS_BUCKET_NAME);const blob = bucket.file(`${PDF_CACHE_PREFIX}${cacheKey}.json`);@@ -83,7 +82,6 @@ export async function getPdfResultFromCache(}const cacheKey = createPdfCacheKey(pdfContent);- const storage = new Storage({ credentials });const bucket = storage.bucket(process.env.GCS_BUCKET_NAME);const blob = bucket.file(`${PDF_CACHE_PREFIX}${cacheKey}.json`);diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.tsindex 06364cb6a..6f8a1df1e 100644--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts@@ -1,10 +1,11 @@+import crypto from "crypto";import { Document } from "../../../../controllers/v1/types";import { EngineScrapeResult } from "..";import { Meta } from "../..";import { getIndexFromGCS, hashURL, index_supabase_service, normalizeURLForIndex, saveIndexToGCS, generateURLSplits, addIndexInsertJob, generateDomainSplits, addOMCEJob, addDomainFrequencyJob } from "../../../../services";import { EngineError, IndexMissError } from "../../error";import { shouldParsePDF } from "../../../../controllers/v2/types";-import crypto from "crypto";+import { storage } from "../../../../lib/gcs-jobs";export async function sendDocumentToIndex(meta: Meta, document: Document) {const shouldCache = meta.options.storeInCachediff --git a/apps/api/src/services/index.ts b/apps/api/src/services/index.tsindex a16a229fc..dc8a803f1 100644--- a/apps/api/src/services/index.ts+++ b/apps/api/src/services/index.ts@@ -1,13 +1,14 @@import { createClient, SupabaseClient } from "@supabase/supabase-js";import { logger as _logger } from "../lib/logger";import { configDotenv } from "dotenv";-import { ApiError, Storage } from "@google-cloud/storage";+import { ApiError } from "@google-cloud/storage";import crypto from "crypto";import { redisEvictConnection } from "./redis";import type { Logger } from "winston";import psl from "psl";import { MapDocument } from "../controllers/v2/types";import { PDFMetadata } from "../lib/pdf-parser";+import { storage } from "../lib/gcs-jobs";configDotenv();// SupabaseService class initializes the Supabase client conditionally based on environment variables.@@ -71,11 +72,41 @@ export async function getIndexFromGCS(url: string, logger?: Logger): Promise<anyreturn null;}- const storage = new Storage({ credentials });const bucket = storage.bucket(process.env.GCS_INDEX_BUCKET_NAME);const blob = bucket.file(`${url}`);const [blobContent] = await blob.download();const parsed = JSON.parse(blobContent.toString());++ try {+ if (+ typeof parsed.screenshot === "string"+ ) {+ const screenshotUrl = new URL(parsed.screenshot);+ let expiresAt = parseInt(screenshotUrl.searchParams.get("Expires") ?? "0", 10) * 1000;+ if (expiresAt === 0) {+ expiresAt = new Date(screenshotUrl.searchParams.get("X-Goog-Date") ?? "1970-01-01T00:00:00Z").getTime() + parseInt(screenshotUrl.searchParams.get("X-Goog-Expires") ?? "0", 10) * 1000;+ }+ if (screenshotUrl.hostname === "storage.googleapis.com" && expiresAt < Date.now()) {+ logger?.info("Re-signing screenshot URL");+ const [url] = await storage.bucket(process.env.GCS_MEDIA_BUCKET_NAME!).file(decodeURIComponent(screenshotUrl.pathname.split("/")[2])).getSignedUrl({+ action: "read",+ expires: Date.now() + 1000 * 60 * 60 * 24 * 7, // 7 days+ });+ parsed.screenshot = url;++ // Update the blob+ await blob.save(JSON.stringify(parsed), {+ contentType: "application/json",+ });+ }+ }+ } catch (error) {+ logger?.warn("Error re-signing screenshot URL", {+ error,+ url,+ });+ }+return parsed;} catch (error) {if (error instanceof ApiError && error.code === 404 && error.message.includes("No such object:")) {@@ -106,7 +137,6 @@ export async function saveIndexToGCS(id: string, doc: {return;}- const storage = new Storage({ credentials });const bucket = storage.bucket(process.env.GCS_INDEX_BUCKET_NAME);const blob = bucket.file(`${id}.json`);for (let i = 0; i < 3; i++) {