Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick:
Nicolasf242cd3034ba
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.tsindex edb088d02..a28c24bff 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts@@ -154,7 +154,8 @@ export async function fireEngineCheckStatus(const doc = await getDocFromGCS(status.docUrl.split("/").pop() ?? "");if (doc) {status = { ...status, ...doc };- delete status.docUrl;+ // Keep docUrl so handlers can extract ID from it+ // It's converted to gcsPath in handlers and not carried forward in EngineScrapeResult}}diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.tsindex b8d575112..d99067ee9 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts@@ -358,6 +358,7 @@ export async function scrapeURLWithFireEngineChromeCDP(proxyUsed: response.usedMobileProxy ? "stealth" : "basic",youtubeTranscriptContent: response.youtubeTranscriptContent,+ gcsPath: response.docUrl?.split("/").pop(), // Extract just the filename/ID from the path};});}@@ -434,6 +435,7 @@ export async function scrapeURLWithFireEnginePlaywright(: {}),proxyUsed: response.usedMobileProxy ? "stealth" : "basic",+ gcsPath: response.docUrl?.split("/").pop(), // Extract just the filename/ID from the path};});}@@ -499,6 +501,7 @@ export async function scrapeURLWithFireEngineTLSClient() ?? [])[1] ?? undefined,proxyUsed: response.usedMobileProxy ? "stealth" : "basic",+ gcsPath: response.docUrl?.split("/").pop(), // Extract just the filename/ID from the path};});}diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.tsindex 743f744ed..86145ff3c 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts@@ -202,7 +202,8 @@ export async function fireEngineScrape<const doc = await getDocFromGCS(status.docUrl.split("/").pop() ?? "");if (doc) {status = { ...status, ...doc };- delete status.docUrl;+ // Keep docUrl so handlers can extract ID from it+ // It's converted to gcsPath in handlers and not carried forward in EngineScrapeResult}}diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex 309a7c2aa..bad6464e1 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -131,6 +131,8 @@ export type EngineScrapeResult = {postprocessorsUsed?: string[];proxyUsed: "basic" | "stealth";++ gcsPath?: string; // GCS path for raw HTML storage};const engineHandlers: {diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 633b9706f..7dedc0d40 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -100,6 +100,7 @@ export type Meta = {| undefined; // undefined: no prefetch yet, null: prefetch came back emptycostTracking: CostTracking;winnerEngine?: Engine;+ gcsPath?: string; // GCS path for raw HTML storage (from fire-engine)};function buildFeatureFlags(@@ -660,6 +661,11 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {meta.winnerEngine = result.engine;let engineResult: EngineScrapeResult = result.result;++ // Store GCS path from engine result for downstream use (e.g., search index)+ if (engineResult.gcsPath) {+ meta.gcsPath = engineResult.gcsPath;+ }for (const postprocessor of postprocessors) {if (diff --git a/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts b/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.tsindex 4fa44075b..350674de9 100644--- a/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts@@ -131,7 +131,7 @@ export async function sendDocumentToSearchIndex(markdown: document.markdown ?? "",html: document.rawHtml ?? "",statusCode: document.metadata.statusCode,- gcsPath: undefined, // Can be populated from GCS if needed+ gcsPath: meta.gcsPath ?? undefined, // GCS path from fire-enginescreenshotUrl: document.screenshot ?? undefined,language: document.metadata.language ?? "en",country: meta.options.location?.country ?? undefined,