Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Update extract.ts
Nicolas2513efc9718e
diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.tsindex 477ad959a..8202a5134 100644--- a/apps/api/src/controllers/v1/extract.ts+++ b/apps/api/src/controllers/v1/extract.ts@@ -30,7 +30,9 @@ const redis = new Redis(process.env.REDIS_URL!);const MAX_EXTRACT_LIMIT = 100;const MAX_RANKING_LIMIT = 10;-const SCORE_THRESHOLD = 0.75;+const INITIAL_SCORE_THRESHOLD = 0.75;+const FALLBACK_SCORE_THRESHOLD = 0.5;+const MIN_REQUIRED_LINKS = 3;/*** Extracts data from the provided URLs based on the request parameters.@@ -94,19 +96,28 @@ export async function extractController(if (req.body.prompt) {// Get similarity scores between the search query and each link's context- const linksAndScores : { link: string, linkWithContext: string, score: number, originalIndex: number }[] = await performRanking(mappedLinksRerank, mappedLinks.map(l => l.url), mapUrl);+ const linksAndScores = await performRanking(mappedLinksRerank, mappedLinks.map(l => l.url), mapUrl);- mappedLinks = linksAndScores- // Only keep links that have a similarity score above the threshold- .filter(x => x.score > SCORE_THRESHOLD)- // Map back to the original link objects- .map(x => mappedLinks.find(link => link.url === x.link))- // Remove any undefined links, links without URLs, and blocked URLs- .filter((x): x is MapDocument => x !== undefined && x.url !== undefined && !isUrlBlocked(x.url))- // Limit the number of results- .slice(0, MAX_RANKING_LIMIT);-- // TODO: handle case where no links are returned+ // First try with high threshold+ let filteredLinks = filterAndProcessLinks(mappedLinks, linksAndScores, INITIAL_SCORE_THRESHOLD);++ // If we don't have enough high-quality links, try with lower threshold+ if (filteredLinks.length < MIN_REQUIRED_LINKS) {+ logger.info(`Only found ${filteredLinks.length} links with score > ${INITIAL_SCORE_THRESHOLD}. Trying lower threshold...`);+ filteredLinks = filterAndProcessLinks(mappedLinks, linksAndScores, FALLBACK_SCORE_THRESHOLD);++ if (filteredLinks.length === 0) {+ // If still no results, take top N results regardless of score+ logger.warn(`No links found with score > ${FALLBACK_SCORE_THRESHOLD}. Taking top ${MIN_REQUIRED_LINKS} results.`);+ filteredLinks = linksAndScores+ .sort((a, b) => b.score - a.score)+ .slice(0, MIN_REQUIRED_LINKS)+ .map(x => mappedLinks.find(link => link.url === x.link))+ .filter((x): x is MapDocument => x !== undefined && x.url !== undefined && !isUrlBlocked(x.url));+ }+ }++ mappedLinks = filteredLinks.slice(0, MAX_RANKING_LIMIT);}return mappedLinks.map(x => x.url) as string[];@@ -124,8 +135,14 @@ export async function extractController(const processedUrls = await Promise.all(urlPromises);links.push(...processedUrls.flat());- // console.log("links", links.length);- // Scrape all links in parallel+ if (links.length === 0) {+ return res.status(400).json({+ success: false,+ error: "No valid URLs found to scrape. Try adjusting your search criteria or including more URLs."+ });+ }++ // Scrape all links in parallel with retriesconst scrapePromises = links.map(async (url) => {const origin = req.body.origin || "api";const timeout = Math.floor((req.body.timeout || 40000) * 0.7) || 30000; // Use 70% of total timeout for individual scrapes@@ -210,9 +227,6 @@ export async function extractController(// Optionally, you could notify an admin or add to a retry queue here});-- // console.log("completions.extract", completions.extract);-let data: any;let warning = completions.warning ?? "";try {@@ -244,4 +258,15 @@ export async function extractController(scrape_id: id,warning: warning});+}++function filterAndProcessLinks(+ mappedLinks: MapDocument[],+ linksAndScores: { link: string, linkWithContext: string, score: number, originalIndex: number }[],+ threshold: number+): MapDocument[] {+ return linksAndScores+ .filter(x => x.score > threshold)+ .map(x => mappedLinks.find(link => link.url === x.link))+ .filter((x): x is MapDocument => x !== undefined && x.url !== undefined && !isUrlBlocked(x.url));}\ No newline at end of file