Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(map): returning excessive query parameters
Gergő Móricz6e59bbd265ac
diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.tsindex 6145beced..c33f07fc3 100644--- a/apps/api/src/controllers/v1/map.ts+++ b/apps/api/src/controllers/v1/map.ts@@ -258,7 +258,7 @@ export async function getMapResults({links = links.map((x) => {try {- return checkAndUpdateURLForMap(x).url.trim();+ return checkAndUpdateURLForMap(x, crawlerOptions.ignoreQueryParameters ?? true).url.trim();} catch (_) {return null;}diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex cf859271f..8571612ee 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -795,12 +795,14 @@ export type CrawlRequest = z.infer<typeof crawlRequestSchema>;export type CrawlRequestInput = z.input<typeof crawlRequestSchema>;export const mapRequestSchema = crawlerOptions+ .omit({ ignoreQueryParameters: true }).extend({url,origin: z.string().optional().default("api"),integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || null),includeSubdomains: z.boolean().default(true),search: z.string().optional(),+ ignoreQueryParameters: z.boolean().default(true),ignoreSitemap: z.boolean().default(false),sitemapOnly: z.boolean().default(false),limit: z.number().min(1).max(30000).default(5000),diff --git a/apps/api/src/lib/validateUrl.ts b/apps/api/src/lib/validateUrl.tsindex 6216e0ec5..e8fe866b5 100644--- a/apps/api/src/lib/validateUrl.ts+++ b/apps/api/src/lib/validateUrl.ts@@ -126,7 +126,7 @@ export function isSameSubdomain(url: string, baseUrl: string) {return domain1 === domain2 && subdomain1 === subdomain2;}-export const checkAndUpdateURLForMap = (url: string) => {+export const checkAndUpdateURLForMap = (url: string, ignoreQueryParameters: boolean = false) => {if (!protocolIncluded(url)) {url = `http://${url}`;}@@ -147,7 +147,10 @@ export const checkAndUpdateURLForMap = (url: string) => {}// remove any query params- // url = url.split("?")[0].trim();+ if (ignoreQueryParameters) {+ url = url.split("?")[0].trim();+ typedUrlObj.search = "";+ }return { urlObj: typedUrlObj, url: url };};