Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(v1/map): parallelize
Gergő Móricz7a83099e5776
diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.tsindex 61e2c4fa3..3cb2ef00c 100644--- a/apps/api/src/controllers/v1/map.ts+++ b/apps/api/src/controllers/v1/map.ts@@ -53,8 +53,10 @@ async function queryIndex(url: string, limit: number, useIndex: boolean, includeconst hostname = urlObj.hostname;// TEMP: this should be altered on June 15th 2025 7AM PT - mogery- const domainLinks = includeSubdomains ? await queryIndexAtDomainSplitLevel(hostname, limit, 14 * 24 * 60 * 60 * 1000) : [];- const splitLinks = await queryIndexAtSplitLevel(url, limit, 14 * 24 * 60 * 60 * 1000);+ const [domainLinks, splitLinks] = await Promise.all([+ includeSubdomains ? queryIndexAtDomainSplitLevel(hostname, limit, 14 * 24 * 60 * 60 * 1000) : [],+ queryIndexAtSplitLevel(url, limit, 14 * 24 * 60 * 60 * 1000),+ ]);return Array.from(new Set([...domainLinks, ...splitLinks]));} else {