Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
chore: limit maxRequestsPerCrawl number in starter templates (#2165)
Jan Bucharb59ee5050d0d
diff --git a/packages/templates/templates/cheerio-js/src/main.js b/packages/templates/templates/cheerio-js/src/main.jsindex 354cfbf23..f8b62e352 100644--- a/packages/templates/templates/cheerio-js/src/main.js+++ b/packages/templates/templates/cheerio-js/src/main.js@@ -7,6 +7,8 @@ const startUrls = ['https://crawlee.dev'];const crawler = new CheerioCrawler({// proxyConfiguration: new ProxyConfiguration({ proxyUrls: ['...'] }),requestHandler: router,+ // Comment this option to scrape the full website.+ maxRequestsPerCrawl: 20,});await crawler.run(startUrls);diff --git a/packages/templates/templates/cheerio-ts/src/main.ts b/packages/templates/templates/cheerio-ts/src/main.tsindex f36b0ec5f..fb586fb6c 100644--- a/packages/templates/templates/cheerio-ts/src/main.ts+++ b/packages/templates/templates/cheerio-ts/src/main.ts@@ -8,6 +8,8 @@ const startUrls = ['https://crawlee.dev'];const crawler = new CheerioCrawler({// proxyConfiguration: new ProxyConfiguration({ proxyUrls: ['...'] }),requestHandler: router,+ // Comment this option to scrape the full website.+ maxRequestsPerCrawl: 20,});await crawler.run(startUrls);diff --git a/packages/templates/templates/getting-started-js/src/main.js b/packages/templates/templates/getting-started-js/src/main.jsindex 66844e759..d8e39bb84 100644--- a/packages/templates/templates/getting-started-js/src/main.js+++ b/packages/templates/templates/getting-started-js/src/main.js@@ -16,6 +16,8 @@ const crawler = new PlaywrightCrawler({// and add them to the crawling queue.await enqueueLinks();},+ // Comment this option to scrape the full website.+ maxRequestsPerCrawl: 20,// Uncomment this option to see the browser window.// headless: false,});diff --git a/packages/templates/templates/getting-started-ts/src/main.ts b/packages/templates/templates/getting-started-ts/src/main.tsindex 66844e759..d8e39bb84 100644--- a/packages/templates/templates/getting-started-ts/src/main.ts+++ b/packages/templates/templates/getting-started-ts/src/main.ts@@ -16,6 +16,8 @@ const crawler = new PlaywrightCrawler({// and add them to the crawling queue.await enqueueLinks();},+ // Comment this option to scrape the full website.+ maxRequestsPerCrawl: 20,// Uncomment this option to see the browser window.// headless: false,});diff --git a/packages/templates/templates/playwright-js/src/main.js b/packages/templates/templates/playwright-js/src/main.jsindex 8076b10b5..ccda8b9ea 100644--- a/packages/templates/templates/playwright-js/src/main.js+++ b/packages/templates/templates/playwright-js/src/main.js@@ -7,6 +7,8 @@ const startUrls = ['https://crawlee.dev'];const crawler = new PlaywrightCrawler({// proxyConfiguration: new ProxyConfiguration({ proxyUrls: ['...'] }),requestHandler: router,+ // Comment this option to scrape the full website.+ maxRequestsPerCrawl: 20,});await crawler.run(startUrls);diff --git a/packages/templates/templates/playwright-ts/src/main.ts b/packages/templates/templates/playwright-ts/src/main.tsindex 77083d5a9..53ecb8a76 100644--- a/packages/templates/templates/playwright-ts/src/main.ts+++ b/packages/templates/templates/playwright-ts/src/main.ts@@ -8,6 +8,8 @@ const startUrls = ['https://crawlee.dev'];const crawler = new PlaywrightCrawler({// proxyConfiguration: new ProxyConfiguration({ proxyUrls: ['...'] }),requestHandler: router,+ // Comment this option to scrape the full website.+ maxRequestsPerCrawl: 20,});await crawler.run(startUrls);diff --git a/packages/templates/templates/puppeteer-js/src/main.js b/packages/templates/templates/puppeteer-js/src/main.jsindex a494a18c8..d7d2882ce 100644--- a/packages/templates/templates/puppeteer-js/src/main.js+++ b/packages/templates/templates/puppeteer-js/src/main.js@@ -7,6 +7,8 @@ const startUrls = ['https://crawlee.dev'];const crawler = new PuppeteerCrawler({// proxyConfiguration: new ProxyConfiguration({ proxyUrls: ['...'] }),requestHandler: router,+ // Comment this option to scrape the full website.+ maxRequestsPerCrawl: 20,});await crawler.run(startUrls);diff --git a/packages/templates/templates/puppeteer-ts/src/main.ts b/packages/templates/templates/puppeteer-ts/src/main.tsindex 1c3c9a441..d613d1e88 100644--- a/packages/templates/templates/puppeteer-ts/src/main.ts+++ b/packages/templates/templates/puppeteer-ts/src/main.ts@@ -8,6 +8,8 @@ const startUrls = ['https://crawlee.dev'];const crawler = new PuppeteerCrawler({// proxyConfiguration: new ProxyConfiguration({ proxyUrls: ['...'] }),requestHandler: router,+ // Comment this option to scrape the full website.+ maxRequestsPerCrawl: 20,});await crawler.run(startUrls);