Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Revert "Nick:"
Nicolascb764b6dc0c1
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.tsindex a28c24bff..edb088d02 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts@@ -154,8 +154,7 @@ export async function fireEngineCheckStatus(const doc = await getDocFromGCS(status.docUrl.split("/").pop() ?? "");if (doc) {status = { ...status, ...doc };- // Keep docUrl so handlers can extract ID from it- // It's converted to gcsPath in handlers and not carried forward in EngineScrapeResult+ delete status.docUrl;}}diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.tsindex d99067ee9..b8d575112 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts@@ -358,7 +358,6 @@ export async function scrapeURLWithFireEngineChromeCDP(proxyUsed: response.usedMobileProxy ? "stealth" : "basic",youtubeTranscriptContent: response.youtubeTranscriptContent,- gcsPath: response.docUrl?.split("/").pop(), // Extract just the filename/ID from the path};});}@@ -435,7 +434,6 @@ export async function scrapeURLWithFireEnginePlaywright(: {}),proxyUsed: response.usedMobileProxy ? "stealth" : "basic",- gcsPath: response.docUrl?.split("/").pop(), // Extract just the filename/ID from the path};});}@@ -501,7 +499,6 @@ export async function scrapeURLWithFireEngineTLSClient() ?? [])[1] ?? undefined,proxyUsed: response.usedMobileProxy ? "stealth" : "basic",- gcsPath: response.docUrl?.split("/").pop(), // Extract just the filename/ID from the path};});}diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.tsindex 86145ff3c..743f744ed 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts@@ -202,8 +202,7 @@ export async function fireEngineScrape<const doc = await getDocFromGCS(status.docUrl.split("/").pop() ?? "");if (doc) {status = { ...status, ...doc };- // Keep docUrl so handlers can extract ID from it- // It's converted to gcsPath in handlers and not carried forward in EngineScrapeResult+ delete status.docUrl;}}diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex bad6464e1..309a7c2aa 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -131,8 +131,6 @@ export type EngineScrapeResult = {postprocessorsUsed?: string[];proxyUsed: "basic" | "stealth";-- gcsPath?: string; // GCS path for raw HTML storage};const engineHandlers: {diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 7dedc0d40..633b9706f 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -100,7 +100,6 @@ export type Meta = {| undefined; // undefined: no prefetch yet, null: prefetch came back emptycostTracking: CostTracking;winnerEngine?: Engine;- gcsPath?: string; // GCS path for raw HTML storage (from fire-engine)};function buildFeatureFlags(@@ -661,11 +660,6 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {meta.winnerEngine = result.engine;let engineResult: EngineScrapeResult = result.result;-- // Store GCS path from engine result for downstream use (e.g., search index)- if (engineResult.gcsPath) {- meta.gcsPath = engineResult.gcsPath;- }for (const postprocessor of postprocessors) {if (diff --git a/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts b/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.tsindex 350674de9..4fa44075b 100644--- a/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts@@ -131,7 +131,7 @@ export async function sendDocumentToSearchIndex(markdown: document.markdown ?? "",html: document.rawHtml ?? "",statusCode: document.metadata.statusCode,- gcsPath: meta.gcsPath ?? undefined, // GCS path from fire-engine+ gcsPath: undefined, // Can be populated from GCS if neededscreenshotUrl: document.screenshot ?? undefined,language: document.metadata.language ?? "en",country: meta.options.location?.country ?? undefined,