Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api/crawl): do not prefill Crawl-Delay when crawling
Gergő Móricz9ca532fb2d57
diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.tsindex e96f36608..e83489826 100644--- a/apps/api/src/controllers/v1/crawl.ts+++ b/apps/api/src/controllers/v1/crawl.ts@@ -125,10 +125,10 @@ export async function crawlController(try {sc.robots = await crawler.getRobotsTxt(scrapeOptions.skipTlsVerification);- const robotsCrawlDelay = crawler.getRobotsCrawlDelay();- if (robotsCrawlDelay !== null && !sc.crawlerOptions.delay) {- sc.crawlerOptions.delay = robotsCrawlDelay;- }+ // const robotsCrawlDelay = crawler.getRobotsCrawlDelay();+ // if (robotsCrawlDelay !== null && !sc.crawlerOptions.delay) {+ // sc.crawlerOptions.delay = robotsCrawlDelay;+ // }} catch (e) {logger.debug("Failed to get robots.txt (this is probably fine!)", {error: e,diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.tsindex b202519c7..2a74b00fa 100644--- a/apps/api/src/controllers/v2/crawl.ts+++ b/apps/api/src/controllers/v2/crawl.ts@@ -192,10 +192,10 @@ export async function crawlController(try {sc.robots = await crawler.getRobotsTxt(scrapeOptions.skipTlsVerification);- const robotsCrawlDelay = crawler.getRobotsCrawlDelay();- if (robotsCrawlDelay !== null && !sc.crawlerOptions.delay) {- sc.crawlerOptions.delay = robotsCrawlDelay;- }+ // const robotsCrawlDelay = crawler.getRobotsCrawlDelay();+ // if (robotsCrawlDelay !== null && !sc.crawlerOptions.delay) {+ // sc.crawlerOptions.delay = robotsCrawlDelay;+ // }} catch (e) {logger.debug("Failed to get robots.txt (this is probably fine!)", {error: e,diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.tsindex 11ba02239..59d5b0037 100644--- a/apps/api/src/services/indexing/index-worker.ts+++ b/apps/api/src/services/indexing/index-worker.ts@@ -191,10 +191,10 @@ const processPrecrawlJobInternal = async (token: string, job: Job) => {sc.robots = await crawler.getRobotsTxt(scrapeOptions.skipTlsVerification,);- const robotsCrawlDelay = crawler.getRobotsCrawlDelay();- if (robotsCrawlDelay !== null && !sc.crawlerOptions.delay) {- sc.crawlerOptions.delay = robotsCrawlDelay;- }+ // const robotsCrawlDelay = crawler.getRobotsCrawlDelay();+ // if (robotsCrawlDelay !== null && !sc.crawlerOptions.delay) {+ // sc.crawlerOptions.delay = robotsCrawlDelay;+ // }} catch (e) {logger.debug("Failed to get robots.txt (this is probably fine!)", {error: e,