Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Update index.ts

Nicolas8252e82cad89
diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
index 7327a92f4..201b18464 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
@@ -45,6 +45,11 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {
return document;
}
+ // Generate indexId synchronously and set it on document immediately
+ // so it's available to other transformers (e.g., search index)
+ const indexId = crypto.randomUUID();
+ document.metadata.indexId = indexId;
+
(async () => {
try {
const normalizedURL = normalizeURLForIndex(meta.url);
@@ -60,8 +65,6 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {
const domainSplits = generateDomainSplits(hostname, fakeDomain);
const domainSplitsHash = domainSplits.map(split => hashURL(split));
- const indexId = crypto.randomUUID();
-
try {
await saveIndexToGCS(indexId, {
url:
@@ -84,9 +87,6 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {
contentType: document.metadata.contentType,
postprocessorsUsed: document.metadata.postprocessorsUsed,
});
-
- // Store the indexId in document metadata so it can be used by other transformers (e.g., search index)
- document.metadata.indexId = indexId;
} catch (error) {
meta.logger.error("Failed to save document to index", {
error,