Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Revert "Nick:"

Nicolas634c521224bf
diff --git a/apps/api/src/lib/search-index/service.ts b/apps/api/src/lib/search-index/service.ts
index 552a50a02..fe2b943e2 100644
--- a/apps/api/src/lib/search-index/service.ts
+++ b/apps/api/src/lib/search-index/service.ts
@@ -257,7 +257,6 @@ export async function indexDocumentForSearch(
description: input.description?.slice(0, 500) ?? null,
language: input.language ?? "en",
gcs_path: input.gcsPath ?? null,
- gcp_path_id: input.gcsPath ?? null,
screenshot_url: input.screenshotUrl ?? null,
content_ts: contentTsVector,
country: input.country ?? null,
@@ -478,6 +477,8 @@ function normalizeSearchURL(url: string): string {
urlObj.hash = "";
urlObj.search = ""; // Remove query params for canonical URL
// Note: Preserve original protocol (don't force https) to avoid:
+ // 1. Making HTTP-only pages unreachable
+ // 2. Collisions between distinct HTTP/HTTPS resources
if (urlObj.hostname.startsWith("www.")) {
urlObj.hostname = urlObj.hostname.slice(4);
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
index 44cfc4c7d..d99067ee9 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
@@ -358,7 +358,7 @@ export async function scrapeURLWithFireEngineChromeCDP(
proxyUsed: response.usedMobileProxy ? "stealth" : "basic",
youtubeTranscriptContent: response.youtubeTranscriptContent,
- gcsPath: response.docUrl?.split("/").pop()?.replace(/\.json$/, ""), // Extract UUID, strip .json extension
+ gcsPath: response.docUrl?.split("/").pop(), // Extract just the filename/ID from the path
};
});
}
@@ -435,7 +435,7 @@ export async function scrapeURLWithFireEnginePlaywright(
: {}),
proxyUsed: response.usedMobileProxy ? "stealth" : "basic",
- gcsPath: response.docUrl?.split("/").pop()?.replace(/\.json$/, ""), // Extract UUID, strip .json extension
+ gcsPath: response.docUrl?.split("/").pop(), // Extract just the filename/ID from the path
};
});
}
@@ -501,7 +501,7 @@ export async function scrapeURLWithFireEngineTLSClient(
) ?? [])[1] ?? undefined,
proxyUsed: response.usedMobileProxy ? "stealth" : "basic",
- gcsPath: response.docUrl?.split("/").pop()?.replace(/\.json$/, ""), // Extract UUID, strip .json extension
+ gcsPath: response.docUrl?.split("/").pop(), // Extract just the filename/ID from the path
};
});
}