Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Update extract.ts
Nicolasb693c6c23bc1
diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.tsindex 0654860ce..e527b5b6e 100644--- a/apps/api/src/controllers/v1/extract.ts+++ b/apps/api/src/controllers/v1/extract.ts@@ -93,14 +93,20 @@ export async function extractController(// }if (req.body.prompt) {+ // Get similarity scores between the search query and each link's contextconst linksAndScores : { link: string, linkWithContext: string, score: number, originalIndex: number }[] = await performRanking(mappedLinksRerank, mappedLinks.map(l => l.url), mapUrl);+mappedLinks = linksAndScores+ // Only keep links that have a similarity score above the threshold.filter(x => x.score > SCORE_THRESHOLD)+ // Map back to the original link objects.map(x => mappedLinks.find(link => link.url === x.link))+ // Remove any undefined links, links without URLs, and blocked URLs.filter((x): x is MapDocument => x !== undefined && x.url !== undefined && !isUrlBlocked(x.url))+ // Limit the number of results.slice(0, MAX_RANKING_LIMIT);- // console.log("linksAndScores", linksAndScores);- // console.log("linksAndScores", linksAndScores.length);++ // TODO: handle case where no links are returned}return mappedLinks.map(x => x.url) as string[];