Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Nick:

Nicolasb8c0c52340e8
diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.ts
index a297c55e5..01fd13889 100644
--- a/apps/api/src/controllers/v2/crawl.ts
+++ b/apps/api/src/controllers/v2/crawl.ts
@@ -73,7 +73,7 @@ export async function crawlController(
flags: req.acuc?.flags ?? null,
logger,
limit: 120,
- includeSubdomains: true,
+ includeSubdomains: false,
allowExternalLinks: false,
useIndex: true,
maxFireEngineResults: 500,
diff --git a/apps/api/src/lib/map-utils.ts b/apps/api/src/lib/map-utils.ts
index 06fc262ec..25bfe8cbe 100644
--- a/apps/api/src/lib/map-utils.ts
+++ b/apps/api/src/lib/map-utils.ts
@@ -330,7 +330,8 @@ export async function buildPromptWithWebsiteStructure({
teamId,
flags,
logger,
- limit = 120,
+ limit = 50,
+
includeSubdomains = true,
allowExternalLinks = false,
useIndex = true,
@@ -353,10 +354,11 @@ export async function buildPromptWithWebsiteStructure({
url,
limit,
includeSubdomains,
- crawlerOptions: {},
+ crawlerOptions: { sitemap: "include" },
teamId,
flags,
allowExternalLinks,
+ filterByPath: false,
useIndex,
maxFireEngineResults,
});