Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
bugfix: self-host crawling doesnt respect limit
busaudc6ebbc6f6afd
diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.tsindex 4efe279ae..c8e449f01 100644--- a/apps/api/src/controllers/v1/crawl.ts+++ b/apps/api/src/controllers/v1/crawl.ts@@ -34,7 +34,11 @@ export async function crawlController(await logCrawl(id, req.auth.team_id);- const { remainingCredits } = req.account;+ let { remainingCredits } = req.account;+ const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === 'true';+ if(!useDbAuthentication){+ remainingCredits = Infinity;+ }const crawlerOptions = legacyCrawlerOptions(req.body);const pageOptions = legacyScrapeOptions(req.body.scrapeOptions);