Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Nick:

Nicolas4434e0693ddf
diff --git a/apps/api/.husky/pre-commit b/apps/api/.husky/pre-commit
index 629a923ce..e69de29bb 100644
--- a/apps/api/.husky/pre-commit
+++ b/apps/api/.husky/pre-commit
@@ -1 +0,0 @@
-cd apps/api && pnpm knip --cache && pnpm lint-staged
\ No newline at end of file
diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.ts
index a297c55e5..01fd13889 100644
--- a/apps/api/src/controllers/v2/crawl.ts
+++ b/apps/api/src/controllers/v2/crawl.ts
@@ -73,7 +73,7 @@ export async function crawlController(
flags: req.acuc?.flags ?? null,
logger,
limit: 120,
- includeSubdomains: true,
+ includeSubdomains: false,
allowExternalLinks: false,
useIndex: true,
maxFireEngineResults: 500,
diff --git a/apps/api/src/lib/map-utils.ts b/apps/api/src/lib/map-utils.ts
index 06fc262ec..25986b347 100644
--- a/apps/api/src/lib/map-utils.ts
+++ b/apps/api/src/lib/map-utils.ts
@@ -98,6 +98,7 @@ export async function getMapResults({
}): Promise<MapResult> {
const functionStartTime = Date.now();
+
const id = uuidv4();
let mapResults: MapDocument[] = [];
const zeroDataRetention = flags?.forceZDR ?? false;
@@ -330,7 +331,7 @@ export async function buildPromptWithWebsiteStructure({
teamId,
flags,
logger,
- limit = 120,
+ limit = 50,
includeSubdomains = true,
allowExternalLinks = false,
useIndex = true,
@@ -353,10 +354,11 @@ export async function buildPromptWithWebsiteStructure({
url,
limit,
includeSubdomains,
- crawlerOptions: {},
+ crawlerOptions: { sitemap: "include" },
teamId,
flags,
allowExternalLinks,
+ filterByPath: false,
useIndex,
maxFireEngineResults,
});