Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(index): store short titles and descriptions (#1677)
Gergő Móricz5ca8e2e98e43
diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.tsindex f3a766178..fe2043fe0 100644--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts@@ -60,6 +60,27 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {return document;}+ let title = document.metadata.title ?? document.metadata.ogTitle ?? null;+ let description = document.metadata.description ?? document.metadata.ogDescription ?? document.metadata.dcDescription ?? null;++ if (typeof title === "string") {+ title = title.trim();+ if (title.length > 60) {+ title = title.slice(0, 57) + "...";+ }+ } else {+ title = null;+ }++ if (typeof description === "string") {+ description = description.trim();+ if (description.length > 160) {+ description = description.slice(0, 157) + "...";+ }+ } else {+ description = null;+ }+try {await addIndexInsertJob({id: indexId,@@ -82,6 +103,8 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {...a,[`domain_splits_${i}_hash`]: x,}), {})),+ ...(title ? { title } : {}),+ ...(description ? { description } : {}),});} catch (error) {meta.logger.error("Failed to add document to index insert queue", {