Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Revert "Nick:"
Nicolas634c521224bf
diff --git a/apps/api/src/lib/search-index/service.ts b/apps/api/src/lib/search-index/service.tsindex 552a50a02..fe2b943e2 100644--- a/apps/api/src/lib/search-index/service.ts+++ b/apps/api/src/lib/search-index/service.ts@@ -257,7 +257,6 @@ export async function indexDocumentForSearch(description: input.description?.slice(0, 500) ?? null,language: input.language ?? "en",gcs_path: input.gcsPath ?? null,- gcp_path_id: input.gcsPath ?? null,screenshot_url: input.screenshotUrl ?? null,content_ts: contentTsVector,country: input.country ?? null,@@ -478,6 +477,8 @@ function normalizeSearchURL(url: string): string {urlObj.hash = "";urlObj.search = ""; // Remove query params for canonical URL// Note: Preserve original protocol (don't force https) to avoid:+ // 1. Making HTTP-only pages unreachable+ // 2. Collisions between distinct HTTP/HTTPS resourcesif (urlObj.hostname.startsWith("www.")) {urlObj.hostname = urlObj.hostname.slice(4);diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.tsindex 44cfc4c7d..d99067ee9 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts@@ -358,7 +358,7 @@ export async function scrapeURLWithFireEngineChromeCDP(proxyUsed: response.usedMobileProxy ? "stealth" : "basic",youtubeTranscriptContent: response.youtubeTranscriptContent,- gcsPath: response.docUrl?.split("/").pop()?.replace(/\.json$/, ""), // Extract UUID, strip .json extension+ gcsPath: response.docUrl?.split("/").pop(), // Extract just the filename/ID from the path};});}@@ -435,7 +435,7 @@ export async function scrapeURLWithFireEnginePlaywright(: {}),proxyUsed: response.usedMobileProxy ? "stealth" : "basic",- gcsPath: response.docUrl?.split("/").pop()?.replace(/\.json$/, ""), // Extract UUID, strip .json extension+ gcsPath: response.docUrl?.split("/").pop(), // Extract just the filename/ID from the path};});}@@ -501,7 +501,7 @@ export async function scrapeURLWithFireEngineTLSClient() ?? [])[1] ?? undefined,proxyUsed: response.usedMobileProxy ? "stealth" : "basic",- gcsPath: response.docUrl?.split("/").pop()?.replace(/\.json$/, ""), // Extract UUID, strip .json extension+ gcsPath: response.docUrl?.split("/").pop(), // Extract just the filename/ID from the path};});}