Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick:
Nicolas4434e0693ddf
diff --git a/apps/api/.husky/pre-commit b/apps/api/.husky/pre-commitindex 629a923ce..e69de29bb 100644--- a/apps/api/.husky/pre-commit+++ b/apps/api/.husky/pre-commit@@ -1 +0,0 @@-cd apps/api && pnpm knip --cache && pnpm lint-staged\ No newline at end of filediff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.tsindex a297c55e5..01fd13889 100644--- a/apps/api/src/controllers/v2/crawl.ts+++ b/apps/api/src/controllers/v2/crawl.ts@@ -73,7 +73,7 @@ export async function crawlController(flags: req.acuc?.flags ?? null,logger,limit: 120,- includeSubdomains: true,+ includeSubdomains: false,allowExternalLinks: false,useIndex: true,maxFireEngineResults: 500,diff --git a/apps/api/src/lib/map-utils.ts b/apps/api/src/lib/map-utils.tsindex 06fc262ec..25986b347 100644--- a/apps/api/src/lib/map-utils.ts+++ b/apps/api/src/lib/map-utils.ts@@ -98,6 +98,7 @@ export async function getMapResults({}): Promise<MapResult> {const functionStartTime = Date.now();+const id = uuidv4();let mapResults: MapDocument[] = [];const zeroDataRetention = flags?.forceZDR ?? false;@@ -330,7 +331,7 @@ export async function buildPromptWithWebsiteStructure({teamId,flags,logger,- limit = 120,+ limit = 50,includeSubdomains = true,allowExternalLinks = false,useIndex = true,@@ -353,10 +354,11 @@ export async function buildPromptWithWebsiteStructure({url,limit,includeSubdomains,- crawlerOptions: {},+ crawlerOptions: { sitemap: "include" },teamId,flags,allowExternalLinks,+ filterByPath: false,useIndex,maxFireEngineResults,});