Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api/map): use new RPCs + set limit max to 100k (#2065)
Gergő Móriczb111da4e4723
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 4104edd53..6a3685cea 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -621,7 +621,7 @@ export const mapRequestSchema = crawlerOptionsignoreQueryParameters: z.boolean().default(true),search: z.string().optional(),sitemap: z.enum(["only", "include", "skip"]).default("include"),- limit: z.number().min(1).max(30000).default(5000),+ limit: z.number().min(1).max(100000).default(5000),timeout: z.number().positive().finite().optional(),useMock: z.string().optional(),filterByPath: z.boolean().default(true),diff --git a/apps/api/src/services/index.ts b/apps/api/src/services/index.tsindex a846e0736..a16a229fc 100644--- a/apps/api/src/services/index.ts+++ b/apps/api/src/services/index.ts@@ -645,10 +645,9 @@ export async function queryIndexAtSplitLevelWithMeta(url: string, limit: number)while (true) {// Query the index for the next set of linksconst { data: _data, error } = await index_supabase_service- .rpc("query_index_at_split_level_with_meta", {+ .rpc("query_index_at_split_level_with_meta_2", {i_level: level,i_url_hash: urlSplitsHash[level],- i_newer_than: new Date(Date.now() - 2 * 24 * 60 * 60 * 1000).toISOString(),}).range(iteration * 1000, (iteration + 1) * 1000)@@ -698,10 +697,9 @@ export async function queryIndexAtDomainSplitLevelWithMeta(hostname: string, limwhile (true) {// Query the index for the next set of linksconst { data: _data, error } = await index_supabase_service- .rpc("query_index_at_domain_split_level_with_meta", {+ .rpc("query_index_at_domain_split_level_with_meta_2", {i_level: level,i_domain_hash: domainSplitsHash[level],- i_newer_than: new Date(Date.now() - 2 * 24 * 60 * 60 * 1000).toISOString(),}).range(iteration * 1000, (iteration + 1) * 1000)