Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api/map): unify limits

Gergő Móricz7363ffef731d
diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.ts
index 0500c90c9..6b1e21a68 100644
--- a/apps/api/src/controllers/v1/map.ts
+++ b/apps/api/src/controllers/v1/map.ts
@@ -8,7 +8,7 @@ import {
} from "./types";
import { scrapeOptions } from "../v2/types";
import { crawlToCrawler, StoredCrawl } from "../../lib/crawl-redis";
-import { MapResponse, MapRequest } from "./types";
+import { MapResponse, MapRequest, MAX_MAP_LIMIT } from "./types";
import { configDotenv } from "dotenv";
import {
checkAndUpdateURLForMap,
@@ -28,8 +28,6 @@ import { MapTimeoutError } from "../../lib/error";
configDotenv();
const redis = new Redis(process.env.REDIS_URL!);
-// Max Links that /map can return
-const MAX_MAP_LIMIT = 30000;
// Max Links that "Smart /map" can return
const MAX_FIRE_ENGINE_RESULTS = 500;
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index e9beab961..f3e80fdc9 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -793,6 +793,8 @@ export const crawlRequestSchema = crawlerOptions
export type CrawlRequest = z.infer<typeof crawlRequestSchema>;
export type CrawlRequestInput = z.input<typeof crawlRequestSchema>;
+export const MAX_MAP_LIMIT = 100000;
+
// Note: Map types have been transitioned to v2/types.ts while maintaining backwards compatibility
export const mapRequestSchema = crawlerOptions
.omit({ ignoreQueryParameters: true })
@@ -805,7 +807,7 @@ export const mapRequestSchema = crawlerOptions
ignoreQueryParameters: z.boolean().default(true),
ignoreSitemap: z.boolean().default(false),
sitemapOnly: z.boolean().default(false),
- limit: z.number().min(1).max(30000).default(5000),
+ limit: z.number().min(1).max(MAX_MAP_LIMIT).default(5000),
timeout: z.number().positive().finite().optional(),
useMock: z.string().optional(),
filterByPath: z.boolean().default(true),
diff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.ts
index 28fadcd4b..fd94343d2 100644
--- a/apps/api/src/controllers/v2/map.ts
+++ b/apps/api/src/controllers/v2/map.ts
@@ -8,6 +8,7 @@ import {
MapRequest,
MapDocument,
MapResponse,
+ MAX_MAP_LIMIT,
} from "./types";
import { crawlToCrawler, StoredCrawl } from "../../lib/crawl-redis";
import { configDotenv } from "dotenv";
@@ -27,8 +28,6 @@ import { MapTimeoutError } from "../../lib/error";
configDotenv();
-// Max Links that /map can return
-const MAX_MAP_LIMIT = 30000;
// Max Links that "Smart /map" can return
const MAX_FIRE_ENGINE_RESULTS = 500;
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 6a3685cea..f7057df56 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -611,6 +611,8 @@ export const crawlRequestSchema = crawlerOptions
export type CrawlRequest = z.infer<typeof crawlRequestSchema>;
export type CrawlRequestInput = z.input<typeof crawlRequestSchema>;
+export const MAX_MAP_LIMIT = 100000;
+
export const mapRequestSchema = crawlerOptions
.omit({ sitemap: true, ignoreQueryParameters: true })
.extend({
@@ -621,7 +623,7 @@ export const mapRequestSchema = crawlerOptions
ignoreQueryParameters: z.boolean().default(true),
search: z.string().optional(),
sitemap: z.enum(["only", "include", "skip"]).default("include"),
- limit: z.number().min(1).max(100000).default(5000),
+ limit: z.number().min(1).max(MAX_MAP_LIMIT).default(5000),
timeout: z.number().positive().finite().optional(),
useMock: z.string().optional(),
filterByPath: z.boolean().default(true),