Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

chore: limit maxRequestsPerCrawl number in starter templates (#2165)

Jan Bucharb59ee5050d0d
diff --git a/packages/templates/templates/cheerio-js/src/main.js b/packages/templates/templates/cheerio-js/src/main.js
index 354cfbf23..f8b62e352 100644
--- a/packages/templates/templates/cheerio-js/src/main.js
+++ b/packages/templates/templates/cheerio-js/src/main.js
@@ -7,6 +7,8 @@ const startUrls = ['https://crawlee.dev'];
const crawler = new CheerioCrawler({
// proxyConfiguration: new ProxyConfiguration({ proxyUrls: ['...'] }),
requestHandler: router,
+ // Comment this option to scrape the full website.
+ maxRequestsPerCrawl: 20,
});
await crawler.run(startUrls);
diff --git a/packages/templates/templates/cheerio-ts/src/main.ts b/packages/templates/templates/cheerio-ts/src/main.ts
index f36b0ec5f..fb586fb6c 100644
--- a/packages/templates/templates/cheerio-ts/src/main.ts
+++ b/packages/templates/templates/cheerio-ts/src/main.ts
@@ -8,6 +8,8 @@ const startUrls = ['https://crawlee.dev'];
const crawler = new CheerioCrawler({
// proxyConfiguration: new ProxyConfiguration({ proxyUrls: ['...'] }),
requestHandler: router,
+ // Comment this option to scrape the full website.
+ maxRequestsPerCrawl: 20,
});
await crawler.run(startUrls);
diff --git a/packages/templates/templates/getting-started-js/src/main.js b/packages/templates/templates/getting-started-js/src/main.js
index 66844e759..d8e39bb84 100644
--- a/packages/templates/templates/getting-started-js/src/main.js
+++ b/packages/templates/templates/getting-started-js/src/main.js
@@ -16,6 +16,8 @@ const crawler = new PlaywrightCrawler({
// and add them to the crawling queue.
await enqueueLinks();
},
+ // Comment this option to scrape the full website.
+ maxRequestsPerCrawl: 20,
// Uncomment this option to see the browser window.
// headless: false,
});
diff --git a/packages/templates/templates/getting-started-ts/src/main.ts b/packages/templates/templates/getting-started-ts/src/main.ts
index 66844e759..d8e39bb84 100644
--- a/packages/templates/templates/getting-started-ts/src/main.ts
+++ b/packages/templates/templates/getting-started-ts/src/main.ts
@@ -16,6 +16,8 @@ const crawler = new PlaywrightCrawler({
// and add them to the crawling queue.
await enqueueLinks();
},
+ // Comment this option to scrape the full website.
+ maxRequestsPerCrawl: 20,
// Uncomment this option to see the browser window.
// headless: false,
});
diff --git a/packages/templates/templates/playwright-js/src/main.js b/packages/templates/templates/playwright-js/src/main.js
index 8076b10b5..ccda8b9ea 100644
--- a/packages/templates/templates/playwright-js/src/main.js
+++ b/packages/templates/templates/playwright-js/src/main.js
@@ -7,6 +7,8 @@ const startUrls = ['https://crawlee.dev'];
const crawler = new PlaywrightCrawler({
// proxyConfiguration: new ProxyConfiguration({ proxyUrls: ['...'] }),
requestHandler: router,
+ // Comment this option to scrape the full website.
+ maxRequestsPerCrawl: 20,
});
await crawler.run(startUrls);
diff --git a/packages/templates/templates/playwright-ts/src/main.ts b/packages/templates/templates/playwright-ts/src/main.ts
index 77083d5a9..53ecb8a76 100644
--- a/packages/templates/templates/playwright-ts/src/main.ts
+++ b/packages/templates/templates/playwright-ts/src/main.ts
@@ -8,6 +8,8 @@ const startUrls = ['https://crawlee.dev'];
const crawler = new PlaywrightCrawler({
// proxyConfiguration: new ProxyConfiguration({ proxyUrls: ['...'] }),
requestHandler: router,
+ // Comment this option to scrape the full website.
+ maxRequestsPerCrawl: 20,
});
await crawler.run(startUrls);
diff --git a/packages/templates/templates/puppeteer-js/src/main.js b/packages/templates/templates/puppeteer-js/src/main.js
index a494a18c8..d7d2882ce 100644
--- a/packages/templates/templates/puppeteer-js/src/main.js
+++ b/packages/templates/templates/puppeteer-js/src/main.js
@@ -7,6 +7,8 @@ const startUrls = ['https://crawlee.dev'];
const crawler = new PuppeteerCrawler({
// proxyConfiguration: new ProxyConfiguration({ proxyUrls: ['...'] }),
requestHandler: router,
+ // Comment this option to scrape the full website.
+ maxRequestsPerCrawl: 20,
});
await crawler.run(startUrls);
diff --git a/packages/templates/templates/puppeteer-ts/src/main.ts b/packages/templates/templates/puppeteer-ts/src/main.ts
index 1c3c9a441..d613d1e88 100644
--- a/packages/templates/templates/puppeteer-ts/src/main.ts
+++ b/packages/templates/templates/puppeteer-ts/src/main.ts
@@ -8,6 +8,8 @@ const startUrls = ['https://crawlee.dev'];
const crawler = new PuppeteerCrawler({
// proxyConfiguration: new ProxyConfiguration({ proxyUrls: ['...'] }),
requestHandler: router,
+ // Comment this option to scrape the full website.
+ maxRequestsPerCrawl: 20,
});
await crawler.run(startUrls);