Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Revert "Nick:"

Nicolas46b2bdedcc32
diff --git a/apps/api/.husky/pre-commit b/apps/api/.husky/pre-commit
index e69de29bb..629a923ce 100644
--- a/apps/api/.husky/pre-commit
+++ b/apps/api/.husky/pre-commit
@@ -0,0 +1 @@
+cd apps/api && pnpm knip --cache && pnpm lint-staged
\ No newline at end of file
diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.ts
index 01fd13889..a297c55e5 100644
--- a/apps/api/src/controllers/v2/crawl.ts
+++ b/apps/api/src/controllers/v2/crawl.ts
@@ -73,7 +73,7 @@ export async function crawlController(
flags: req.acuc?.flags ?? null,
logger,
limit: 120,
- includeSubdomains: false,
+ includeSubdomains: true,
allowExternalLinks: false,
useIndex: true,
maxFireEngineResults: 500,
diff --git a/apps/api/src/lib/map-utils.ts b/apps/api/src/lib/map-utils.ts
index 25986b347..06fc262ec 100644
--- a/apps/api/src/lib/map-utils.ts
+++ b/apps/api/src/lib/map-utils.ts
@@ -98,7 +98,6 @@ export async function getMapResults({
}): Promise<MapResult> {
const functionStartTime = Date.now();
-
const id = uuidv4();
let mapResults: MapDocument[] = [];
const zeroDataRetention = flags?.forceZDR ?? false;
@@ -331,7 +330,7 @@ export async function buildPromptWithWebsiteStructure({
teamId,
flags,
logger,
- limit = 50,
+ limit = 120,
includeSubdomains = true,
allowExternalLinks = false,
useIndex = true,
@@ -354,11 +353,10 @@ export async function buildPromptWithWebsiteStructure({
url,
limit,
includeSubdomains,
- crawlerOptions: { sitemap: "include" },
+ crawlerOptions: {},
teamId,
flags,
allowExternalLinks,
- filterByPath: false,
useIndex,
maxFireEngineResults,
});