Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(scrapeURL/index): horrible no-good very bad index url bug (#1780)
Gergő Móriczd7204e48aad5
diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.tsindex b286a5ea9..63504dba9 100644--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts@@ -46,7 +46,7 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {try {await saveIndexToGCS(indexId, {- url: normalizedURL,+ url: document.metadata.url ?? document.metadata.sourceURL ?? meta.rewrittenUrl ?? meta.url,html: document.rawHtml!,statusCode: document.metadata.statusCode,error: document.metadata.error,