Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(map): returning excessive query parameters

Gergő Móricz6e59bbd265ac
diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.ts
index 6145beced..c33f07fc3 100644
--- a/apps/api/src/controllers/v1/map.ts
+++ b/apps/api/src/controllers/v1/map.ts
@@ -258,7 +258,7 @@ export async function getMapResults({
links = links
.map((x) => {
try {
- return checkAndUpdateURLForMap(x).url.trim();
+ return checkAndUpdateURLForMap(x, crawlerOptions.ignoreQueryParameters ?? true).url.trim();
} catch (_) {
return null;
}
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index cf859271f..8571612ee 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -795,12 +795,14 @@ export type CrawlRequest = z.infer<typeof crawlRequestSchema>;
export type CrawlRequestInput = z.input<typeof crawlRequestSchema>;
export const mapRequestSchema = crawlerOptions
+ .omit({ ignoreQueryParameters: true })
.extend({
url,
origin: z.string().optional().default("api"),
integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || null),
includeSubdomains: z.boolean().default(true),
search: z.string().optional(),
+ ignoreQueryParameters: z.boolean().default(true),
ignoreSitemap: z.boolean().default(false),
sitemapOnly: z.boolean().default(false),
limit: z.number().min(1).max(30000).default(5000),
diff --git a/apps/api/src/lib/validateUrl.ts b/apps/api/src/lib/validateUrl.ts
index 6216e0ec5..e8fe866b5 100644
--- a/apps/api/src/lib/validateUrl.ts
+++ b/apps/api/src/lib/validateUrl.ts
@@ -126,7 +126,7 @@ export function isSameSubdomain(url: string, baseUrl: string) {
return domain1 === domain2 && subdomain1 === subdomain2;
}
-export const checkAndUpdateURLForMap = (url: string) => {
+export const checkAndUpdateURLForMap = (url: string, ignoreQueryParameters: boolean = false) => {
if (!protocolIncluded(url)) {
url = `http://${url}`;
}
@@ -147,7 +147,10 @@ export const checkAndUpdateURLForMap = (url: string) => {
}
// remove any query params
- // url = url.split("?")[0].trim();
+ if (ignoreQueryParameters) {
+ url = url.split("?")[0].trim();
+ typedUrlObj.search = "";
+ }
return { urlObj: typedUrlObj, url: url };
};