Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Nick:

Nicolas061bf699e795
diff --git a/apps/api/src/lib/search-index/service.ts b/apps/api/src/lib/search-index/service.ts
index fe2b943e2..552a50a02 100644
--- a/apps/api/src/lib/search-index/service.ts
+++ b/apps/api/src/lib/search-index/service.ts
@@ -257,6 +257,7 @@ export async function indexDocumentForSearch(
description: input.description?.slice(0, 500) ?? null,
language: input.language ?? "en",
gcs_path: input.gcsPath ?? null,
+ gcp_path_id: input.gcsPath ?? null,
screenshot_url: input.screenshotUrl ?? null,
content_ts: contentTsVector,
country: input.country ?? null,
@@ -477,8 +478,6 @@ function normalizeSearchURL(url: string): string {
urlObj.hash = "";
urlObj.search = ""; // Remove query params for canonical URL
// Note: Preserve original protocol (don't force https) to avoid:
- // 1. Making HTTP-only pages unreachable
- // 2. Collisions between distinct HTTP/HTTPS resources
if (urlObj.hostname.startsWith("www.")) {
urlObj.hostname = urlObj.hostname.slice(4);
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
index d99067ee9..44cfc4c7d 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
@@ -358,7 +358,7 @@ export async function scrapeURLWithFireEngineChromeCDP(
proxyUsed: response.usedMobileProxy ? "stealth" : "basic",
youtubeTranscriptContent: response.youtubeTranscriptContent,
- gcsPath: response.docUrl?.split("/").pop(), // Extract just the filename/ID from the path
+ gcsPath: response.docUrl?.split("/").pop()?.replace(/\.json$/, ""), // Extract UUID, strip .json extension
};
});
}
@@ -435,7 +435,7 @@ export async function scrapeURLWithFireEnginePlaywright(
: {}),
proxyUsed: response.usedMobileProxy ? "stealth" : "basic",
- gcsPath: response.docUrl?.split("/").pop(), // Extract just the filename/ID from the path
+ gcsPath: response.docUrl?.split("/").pop()?.replace(/\.json$/, ""), // Extract UUID, strip .json extension
};
});
}
@@ -501,7 +501,7 @@ export async function scrapeURLWithFireEngineTLSClient(
) ?? [])[1] ?? undefined,
proxyUsed: response.usedMobileProxy ? "stealth" : "basic",
- gcsPath: response.docUrl?.split("/").pop(), // Extract just the filename/ID from the path
+ gcsPath: response.docUrl?.split("/").pop()?.replace(/\.json$/, ""), // Extract UUID, strip .json extension
};
});
}