Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Revert "Nick:"
Nicolas46b2bdedcc32
diff --git a/apps/api/.husky/pre-commit b/apps/api/.husky/pre-commitindex e69de29bb..629a923ce 100644--- a/apps/api/.husky/pre-commit+++ b/apps/api/.husky/pre-commit@@ -0,0 +1 @@+cd apps/api && pnpm knip --cache && pnpm lint-staged\ No newline at end of filediff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.tsindex 01fd13889..a297c55e5 100644--- a/apps/api/src/controllers/v2/crawl.ts+++ b/apps/api/src/controllers/v2/crawl.ts@@ -73,7 +73,7 @@ export async function crawlController(flags: req.acuc?.flags ?? null,logger,limit: 120,- includeSubdomains: false,+ includeSubdomains: true,allowExternalLinks: false,useIndex: true,maxFireEngineResults: 500,diff --git a/apps/api/src/lib/map-utils.ts b/apps/api/src/lib/map-utils.tsindex 25986b347..06fc262ec 100644--- a/apps/api/src/lib/map-utils.ts+++ b/apps/api/src/lib/map-utils.ts@@ -98,7 +98,6 @@ export async function getMapResults({}): Promise<MapResult> {const functionStartTime = Date.now();-const id = uuidv4();let mapResults: MapDocument[] = [];const zeroDataRetention = flags?.forceZDR ?? false;@@ -331,7 +330,7 @@ export async function buildPromptWithWebsiteStructure({teamId,flags,logger,- limit = 50,+ limit = 120,includeSubdomains = true,allowExternalLinks = false,useIndex = true,@@ -354,11 +353,10 @@ export async function buildPromptWithWebsiteStructure({url,limit,includeSubdomains,- crawlerOptions: { sitemap: "include" },+ crawlerOptions: {},teamId,flags,allowExternalLinks,- filterByPath: false,useIndex,maxFireEngineResults,});