Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Update index.ts
Nicolas8252e82cad89
diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.tsindex 7327a92f4..201b18464 100644--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts@@ -45,6 +45,11 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {return document;}+ // Generate indexId synchronously and set it on document immediately+ // so it's available to other transformers (e.g., search index)+ const indexId = crypto.randomUUID();+ document.metadata.indexId = indexId;+(async () => {try {const normalizedURL = normalizeURLForIndex(meta.url);@@ -60,8 +65,6 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {const domainSplits = generateDomainSplits(hostname, fakeDomain);const domainSplitsHash = domainSplits.map(split => hashURL(split));- const indexId = crypto.randomUUID();-try {await saveIndexToGCS(indexId, {url:@@ -84,9 +87,6 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {contentType: document.metadata.contentType,postprocessorsUsed: document.metadata.postprocessorsUsed,});-- // Store the indexId in document metadata so it can be used by other transformers (e.g., search index)- document.metadata.indexId = indexId;} catch (error) {meta.logger.error("Failed to save document to index", {error,