Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick:
Nicolas061bf699e795
diff --git a/apps/api/src/lib/search-index/service.ts b/apps/api/src/lib/search-index/service.tsindex fe2b943e2..552a50a02 100644--- a/apps/api/src/lib/search-index/service.ts+++ b/apps/api/src/lib/search-index/service.ts@@ -257,6 +257,7 @@ export async function indexDocumentForSearch(description: input.description?.slice(0, 500) ?? null,language: input.language ?? "en",gcs_path: input.gcsPath ?? null,+ gcp_path_id: input.gcsPath ?? null,screenshot_url: input.screenshotUrl ?? null,content_ts: contentTsVector,country: input.country ?? null,@@ -477,8 +478,6 @@ function normalizeSearchURL(url: string): string {urlObj.hash = "";urlObj.search = ""; // Remove query params for canonical URL// Note: Preserve original protocol (don't force https) to avoid:- // 1. Making HTTP-only pages unreachable- // 2. Collisions between distinct HTTP/HTTPS resourcesif (urlObj.hostname.startsWith("www.")) {urlObj.hostname = urlObj.hostname.slice(4);diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.tsindex d99067ee9..44cfc4c7d 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts@@ -358,7 +358,7 @@ export async function scrapeURLWithFireEngineChromeCDP(proxyUsed: response.usedMobileProxy ? "stealth" : "basic",youtubeTranscriptContent: response.youtubeTranscriptContent,- gcsPath: response.docUrl?.split("/").pop(), // Extract just the filename/ID from the path+ gcsPath: response.docUrl?.split("/").pop()?.replace(/\.json$/, ""), // Extract UUID, strip .json extension};});}@@ -435,7 +435,7 @@ export async function scrapeURLWithFireEnginePlaywright(: {}),proxyUsed: response.usedMobileProxy ? "stealth" : "basic",- gcsPath: response.docUrl?.split("/").pop(), // Extract just the filename/ID from the path+ gcsPath: response.docUrl?.split("/").pop()?.replace(/\.json$/, ""), // Extract UUID, strip .json extension};});}@@ -501,7 +501,7 @@ export async function scrapeURLWithFireEngineTLSClient() ?? [])[1] ?? undefined,proxyUsed: response.usedMobileProxy ? "stealth" : "basic",- gcsPath: response.docUrl?.split("/").pop(), // Extract just the filename/ID from the path+ gcsPath: response.docUrl?.split("/").pop()?.replace(/\.json$/, ""), // Extract UUID, strip .json extension};});}