Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Update extract.ts
Nicolas796cd0746db5
diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.tsindex be7feaa9d..71b1f9eb4 100644--- a/apps/api/src/controllers/v1/extract.ts+++ b/apps/api/src/controllers/v1/extract.ts@@ -42,7 +42,8 @@ export async function extractController(let docs: Document[] = [];const earlyReturn = false;- for (const url of req.body.urls) {+ // Process all URLs in parallel+ const urlPromises = req.body.urls.map(async (url) => {if (url.includes('/*')) {// Handle glob pattern URLsconst baseUrl = url.replace('/*', '');@@ -84,15 +85,20 @@ export async function extractController(.slice(0, MAX_RANKING_LIMIT);}- links.push(...mappedLinks);+ return mappedLinks;} else {// Handle direct URLs without glob patternif (!isUrlBlocked(url)) {- links.push(url);+ return [url];}+ return [];}- }+ });++ // Wait for all URL processing to complete and flatten results+ const processedUrls = await Promise.all(urlPromises);+ links.push(...processedUrls.flat());// Scrape all links in parallelconst scrapePromises = links.map(async (url) => {