Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
(fix/map) Map failed to filter by path if indexed (#1333)
Nicolas134de67a3b7d
diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.tsindex 30245aa56..ebb0b3246 100644--- a/apps/api/src/controllers/v1/map.ts+++ b/apps/api/src/controllers/v1/map.ts@@ -56,6 +56,7 @@ export async function getMapResults({allowExternalLinks,abort = new AbortController().signal, // noopmock,+ filterByPath = true,}: {url: string;search?: string;@@ -70,6 +71,7 @@ export async function getMapResults({allowExternalLinks?: boolean;abort?: AbortSignal;mock?: string;+ filterByPath?: boolean;}): Promise<MapResult> {const id = uuidv4();let links: string[] = [url];@@ -247,6 +249,29 @@ export async function getMapResults({links = links.filter((x) => isSameSubdomain(x, url));}+ // Filter by path if enabled+ if (filterByPath && !allowExternalLinks) {+ try {+ const urlObj = new URL(url);+ const urlPath = urlObj.pathname;+ // Only apply path filtering if the URL has a significant path (not just '/' or empty)+ // This means we only filter by path if the user has not selected a root domain+ if (urlPath && urlPath !== '/' && urlPath.length > 1) {+ links = links.filter(link => {+ try {+ const linkObj = new URL(link);+ return linkObj.pathname.startsWith(urlPath);+ } catch (e) {+ return false;+ }+ });+ }+ } catch (e) {+ // If URL parsing fails, continue without path filtering+ logger.warn(`Failed to parse URL for path filtering: ${url}`, { error: e });+ }+ }+// remove duplicates that could be due to http/https or wwwlinks = removeDuplicateUrls(links);}@@ -300,6 +325,7 @@ export async function mapController(plan: req.auth.plan,abort: abort.signal,mock: req.body.useMock,+ filterByPath: req.body.filterByPath !== false,}),...(req.body.timeout !== undefined ? [new Promise((resolve, reject) => setTimeout(() => {diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 5a71da9c6..1e4625494 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -506,6 +506,7 @@ export const mapRequestSchema = crawlerOptionslimit: z.number().min(1).max(30000).default(5000),timeout: z.number().positive().finite().optional(),useMock: z.string().optional(),+ filterByPath: z.boolean().default(true),}).strict(strictMessage);