Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): only load more pages if we already have one (#2767)
tomsideguidea422e1e14c56
diff --git a/apps/api/src/lib/map-utils.ts b/apps/api/src/lib/map-utils.tsindex ae1012dfe..5a0a511e8 100644--- a/apps/api/src/lib/map-utils.ts+++ b/apps/api/src/lib/map-utils.ts@@ -210,30 +210,35 @@ export async function getMapResults({? null: await redisEvictConnection.get(cacheKey);- let pagePromises: (Promise<any> | any)[];-- if (cachedResult) {- pagePromises = JSON.parse(cachedResult);- } else {- const fetchPage = async (page: number) => {- return await fireEngineMap(- mapUrl,- {- numResults: resultsPerPage,- page: page,- },- abort,- );- };+ const fetchPage = async (page: number) => {+ return await fireEngineMap(+ mapUrl,+ {+ numResults: resultsPerPage,+ page,+ },+ abort,+ );+ };- pagePromises = Array.from({ length: maxPages }, (_, i) =>- fetchPage(i + 1),+ const fetchAllPages = async (): Promise<any[]> => {+ if (cachedResult) {+ return JSON.parse(cachedResult);+ }+ // if page 1 has no results, don't fetch remaining pages+ const page1Result = await fetchPage(1);+ if (!page1Result || page1Result.length === 0 || maxPages === 1) {+ return [page1Result];+ }+ const remainingPages = await Promise.all(+ Array.from({ length: maxPages - 1 }, (_, i) => fetchPage(i + 2)),);- }+ return [page1Result, ...remainingPages];+ };const [indexResults, searchResults] = await Promise.all([queryIndex(url, limit, useIndex, includeSubdomains),- Promise.all(pagePromises),+ fetchAllPages(),]);if (!zeroDataRetention) {