Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api/map): unify limits
Gergő Móricz7363ffef731d
diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.tsindex 0500c90c9..6b1e21a68 100644--- a/apps/api/src/controllers/v1/map.ts+++ b/apps/api/src/controllers/v1/map.ts@@ -8,7 +8,7 @@ import {} from "./types";import { scrapeOptions } from "../v2/types";import { crawlToCrawler, StoredCrawl } from "../../lib/crawl-redis";-import { MapResponse, MapRequest } from "./types";+import { MapResponse, MapRequest, MAX_MAP_LIMIT } from "./types";import { configDotenv } from "dotenv";import {checkAndUpdateURLForMap,@@ -28,8 +28,6 @@ import { MapTimeoutError } from "../../lib/error";configDotenv();const redis = new Redis(process.env.REDIS_URL!);-// Max Links that /map can return-const MAX_MAP_LIMIT = 30000;// Max Links that "Smart /map" can returnconst MAX_FIRE_ENGINE_RESULTS = 500;diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex e9beab961..f3e80fdc9 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -793,6 +793,8 @@ export const crawlRequestSchema = crawlerOptionsexport type CrawlRequest = z.infer<typeof crawlRequestSchema>;export type CrawlRequestInput = z.input<typeof crawlRequestSchema>;+export const MAX_MAP_LIMIT = 100000;+// Note: Map types have been transitioned to v2/types.ts while maintaining backwards compatibilityexport const mapRequestSchema = crawlerOptions.omit({ ignoreQueryParameters: true })@@ -805,7 +807,7 @@ export const mapRequestSchema = crawlerOptionsignoreQueryParameters: z.boolean().default(true),ignoreSitemap: z.boolean().default(false),sitemapOnly: z.boolean().default(false),- limit: z.number().min(1).max(30000).default(5000),+ limit: z.number().min(1).max(MAX_MAP_LIMIT).default(5000),timeout: z.number().positive().finite().optional(),useMock: z.string().optional(),filterByPath: z.boolean().default(true),diff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.tsindex 28fadcd4b..fd94343d2 100644--- a/apps/api/src/controllers/v2/map.ts+++ b/apps/api/src/controllers/v2/map.ts@@ -8,6 +8,7 @@ import {MapRequest,MapDocument,MapResponse,+ MAX_MAP_LIMIT,} from "./types";import { crawlToCrawler, StoredCrawl } from "../../lib/crawl-redis";import { configDotenv } from "dotenv";@@ -27,8 +28,6 @@ import { MapTimeoutError } from "../../lib/error";configDotenv();-// Max Links that /map can return-const MAX_MAP_LIMIT = 30000;// Max Links that "Smart /map" can returnconst MAX_FIRE_ENGINE_RESULTS = 500;diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 6a3685cea..f7057df56 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -611,6 +611,8 @@ export const crawlRequestSchema = crawlerOptionsexport type CrawlRequest = z.infer<typeof crawlRequestSchema>;export type CrawlRequestInput = z.input<typeof crawlRequestSchema>;+export const MAX_MAP_LIMIT = 100000;+export const mapRequestSchema = crawlerOptions.omit({ sitemap: true, ignoreQueryParameters: true }).extend({@@ -621,7 +623,7 @@ export const mapRequestSchema = crawlerOptionsignoreQueryParameters: z.boolean().default(true),search: z.string().optional(),sitemap: z.enum(["only", "include", "skip"]).default("include"),- limit: z.number().min(1).max(100000).default(5000),+ limit: z.number().min(1).max(MAX_MAP_LIMIT).default(5000),timeout: z.number().positive().finite().optional(),useMock: z.string().optional(),filterByPath: z.boolean().default(true),