Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(map): make sitemapOnly simpler
Gergő Móricz1b032b05fa18
diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.tsindex 0d0449aa9..ab9a5ed7f 100644--- a/apps/api/src/controllers/v1/map.ts+++ b/apps/api/src/controllers/v1/map.ts@@ -61,6 +61,15 @@ export async function mapController(sitemap.forEach((x) => {links.push(x.url);});+ links = links.slice(1)+ .map((x) => {+ try {+ return checkAndUpdateURLForMap(x).url.trim();+ } catch (_) {+ return null;+ }+ })+ .filter((x) => x !== null) as string[];// links = links.slice(1, limit); // don't slice, unnecessary}} else {@@ -139,36 +148,36 @@ export async function mapController(}}-- }- // Perform cosine similarity between the search query and the list of links- if (req.body.search) {- const searchQuery = req.body.search.toLowerCase();-- links = performCosineSimilarity(links, searchQuery);- }+ // Perform cosine similarity between the search query and the list of links+ if (req.body.search) {+ const searchQuery = req.body.search.toLowerCase();- links = links- .map((x) => {- try {- return checkAndUpdateURLForMap(x).url.trim();- } catch (_) {- return null;- }- })- .filter((x) => x !== null) as string[];+ links = performCosineSimilarity(links, searchQuery);+ }- // allows for subdomains to be included- links = links.filter((x) => isSameDomain(x, req.body.url));+ links = links+ .map((x) => {+ try {+ return checkAndUpdateURLForMap(x).url.trim();+ } catch (_) {+ return null;+ }+ })+ .filter((x) => x !== null) as string[];++ // allows for subdomains to be included+ links = links.filter((x) => isSameDomain(x, req.body.url));++ // if includeSubdomains is false, filter out subdomains+ if (!req.body.includeSubdomains) {+ links = links.filter((x) => isSameSubdomain(x, req.body.url));+ }- // if includeSubdomains is false, filter out subdomains- if (!req.body.includeSubdomains) {- links = links.filter((x) => isSameSubdomain(x, req.body.url));+ // remove duplicates that could be due to http/https or www+ links = removeDuplicateUrls(links);+ links.slice(0, limit);}- // remove duplicates that could be due to http/https or www- links = removeDuplicateUrls(links);-billTeam(req.auth.team_id, req.acuc?.sub_id, 1).catch((error) => {logger.error(`Failed to bill team ${req.auth.team_id} for 1 credit: ${error}`@@ -179,14 +188,12 @@ export async function mapController(const endTime = new Date().getTime();const timeTakenInSeconds = (endTime - startTime) / 1000;- const linksToReturn = links.slice(0, limit);-logJob({job_id: id,success: links.length > 0,message: "Map completed",- num_docs: linksToReturn.length,- docs: linksToReturn,+ num_docs: links.length,+ docs: links,time_taken: timeTakenInSeconds,team_id: req.auth.team_id,mode: "map",@@ -199,7 +206,7 @@ export async function mapController(return res.status(200).json({success: true,- links: linksToReturn,+ links: links,scrape_id: req.body.origin?.includes("website") ? id : undefined,});}