Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Adding maxRequestsPerCrawl parameter to BasicCrawler (#120)

Marek Trunkátd40fda2f37b9
diff --git a/CHANGELOG.md b/CHANGELOG.md
index bd9100799..3388c60e1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,7 @@
+xxxxxxxxxxxxxxxxxxx
+===================
+- Parameter `maxRequestsPerCrawl` added to `BasicCrawler` and `PuppeteerCrawler` classes.
+
0.5.38 / 2018-06-22
===================
- Revereted back - `Apify.getApifyProxyUrl()` accepts again `session` and `groups` options instead of
diff --git a/src/basic_crawler.js b/src/basic_crawler.js
index 3f3d49d46..4e97ef89c 100644
--- a/src/basic_crawler.js
+++ b/src/basic_crawler.js
@@ -71,6 +71,10 @@ const DEFAULT_OPTIONS = {
* Function that handles requests that failed more then `option.maxRequestRetries` times.
* @param {Number} [options.maxRequestRetries=3]
* How many times the request is retried if `handleRequestFunction` failed.
+ * @param {Number} [options.maxRequestsPerCrawl]
+ * Maximum number of pages that the crawler will open. The crawl will stop when this limit is reached.
+ * Always set this value in order to prevent infinite loops in misconfigured crawlers.
+ * Note that in cases of parallel crawling, the actual number of pages visited might be slightly higher than this value.
* @param {Number} [options.maxMemoryMbytes]
* Maximum memory available in the system
* See `AutoscaledPool` for details.
@@ -103,6 +107,7 @@ export default class BasicCrawler {
handleRequestFunction,
handleFailedRequestFunction,
maxRequestRetries,
+ maxRequestsPerCrawl,
// AutoscaledPool options
maxMemoryMbytes,
@@ -118,6 +123,7 @@ export default class BasicCrawler {
checkParamOrThrow(handleRequestFunction, 'opts.handleRequestFunction', 'Function');
checkParamOrThrow(handleFailedRequestFunction, 'opts.handleFailedRequestFunction', 'Function');
checkParamOrThrow(maxRequestRetries, 'opts.maxRequestRetries', 'Number');
+ checkParamOrThrow(maxRequestsPerCrawl, 'opts.maxRequestsPerCrawl', 'Maybe Number');
if (!requestList && !requestQueue) {
throw new Error('At least one of the parameters "opts.requestList" and "opts.requestQueue" must be provided!');
@@ -128,6 +134,9 @@ export default class BasicCrawler {
this.handleRequestFunction = handleRequestFunction;
this.handleFailedRequestFunction = handleFailedRequestFunction;
this.maxRequestRetries = maxRequestRetries;
+ this.handledRequestsCount = 0;
+
+ const isMaxPagesExceeded = () => maxRequestsPerCrawl && maxRequestsPerCrawl <= this.handledRequestsCount;
this.autoscaledPool = new AutoscaledPool({
maxMemoryMbytes,
@@ -135,12 +144,18 @@ export default class BasicCrawler {
minConcurrency,
minFreeMemoryRatio,
runTaskFunction: () => this._runTaskFunction(),
- isTaskReadyFunction: () => this._isTaskReadyFunction(),
- isFinishedFunction: () => (
- isFinishedFunction
+ isTaskReadyFunction: () => {
+ if (isMaxPagesExceeded()) return Promise.resolve(false);
+
+ return this._isTaskReadyFunction();
+ },
+ isFinishedFunction: () => {
+ if (isMaxPagesExceeded()) return Promise.resolve(true);
+
+ return isFinishedFunction
? isFinishedFunction()
- : this._defaultIsFinishedFunction()
- ),
+ : this._defaultIsFinishedFunction();
+ },
ignoreMainProcess,
});
}
@@ -204,6 +219,7 @@ export default class BasicCrawler {
.then((request) => {
if (!request) return;
+ let willBeRetried = false;
const handlePromise = this.handleRequestFunction({ request });
if (!isPromise(handlePromise)) throw new Error('User provided handleRequestFunction must return a Promise.');
@@ -228,6 +244,7 @@ export default class BasicCrawler {
url: request.url,
retryCount: request.retryCount,
});
+ willBeRetried = true;
return source.reclaimRequest(request);
}
@@ -241,6 +258,9 @@ export default class BasicCrawler {
return source
.markRequestHandled(request)
.then(() => this.handleFailedRequestFunction({ request, error }));
+ })
+ .finally(() => {
+ if (!willBeRetried) this.handledRequestsCount++;
});
});
}
diff --git a/src/puppeteer_crawler.js b/src/puppeteer_crawler.js
index 0126c1cfc..7587a9c12 100644
--- a/src/puppeteer_crawler.js
+++ b/src/puppeteer_crawler.js
@@ -74,6 +74,11 @@ const DEFAULT_OPTIONS = {
* @param {Number} [options.maxRequestRetries=3]
* Indicates how many times each request is retried if `handleRequestFunction` failed.
* See `maxRequestRetries` parameter of `BasicCrawler`.
+ * @param {Number} [options.maxRequestsPerCrawl]
+ * Maximum number of pages that the crawler will open. The crawl will stop when this limit is reached.
+ * Always set this value in order to prevent infinite loops in misconfigured crawlers.
+ * Note that in cases of parallel crawling, the actual number of pages visited might be slightly higher than this value.
+ * See `maxRequestsPerCrawl` parameter of `BasicCrawler`.
* @param {Number} [options.maxMemoryMbytes]
* Maximum memory available for crawling. See `maxMemoryMbytes` parameter of `AutoscaledPool`.
* @param {Number} [options.maxConcurrency=1000]
@@ -140,6 +145,7 @@ export default class PuppeteerCrawler {
requestList,
requestQueue,
maxRequestRetries,
+ maxRequestsPerCrawl,
handleFailedRequestFunction,
// Puppeteer Pool options
@@ -175,6 +181,7 @@ export default class PuppeteerCrawler {
requestList,
requestQueue,
maxRequestRetries,
+ maxRequestsPerCrawl,
handleRequestFunction: (...args) => this._handleRequestFunction(...args),
handleFailedRequestFunction,
diff --git a/test/basic_crawler.js b/test/basic_crawler.js
index 9c79746af..d748d3dcc 100644
--- a/test/basic_crawler.js
+++ b/test/basic_crawler.js
@@ -376,4 +376,56 @@ describe('BasicCrawler', () => {
expect(processed).to.be.eql([request0, request1]);
});
+
+ it('should support maxRequestsPerCrawl parameter', async () => {
+ const sources = [
+ { url: 'http://example.com/1' },
+ { url: 'http://example.com/2' },
+ { url: 'http://example.com/3' },
+ { url: 'http://example.com/4' },
+ { url: 'http://example.com/5' },
+ ];
+ const processed = {};
+ const requestList = new Apify.RequestList({ sources });
+
+ const handleRequestFunction = async ({ request }) => {
+ await delayPromise(10);
+ processed[request.url] = request;
+ if (request.url === 'http://example.com/2') throw Error();
+ request.userData.foo = 'bar';
+ };
+
+ let handleFailedRequestFunctionCalls = 0;
+ const handleFailedRequestFunction = () => {
+ handleFailedRequestFunctionCalls++;
+ };
+
+ const basicCrawler = new Apify.BasicCrawler({
+ requestList,
+ maxRequestRetries: 3,
+ maxRequestsPerCrawl: 3,
+ maxConcurrency: 1,
+ handleRequestFunction,
+ handleFailedRequestFunction,
+ });
+
+ await requestList.initialize();
+ await basicCrawler.run();
+
+ expect(processed['http://example.com/1'].userData.foo).to.be.eql('bar');
+ expect(processed['http://example.com/1'].errorMessages).to.be.a('null');
+ expect(processed['http://example.com/1'].retryCount).to.be.eql(0);
+ expect(processed['http://example.com/3'].userData.foo).to.be.eql('bar');
+ expect(processed['http://example.com/3'].errorMessages).to.be.a('null');
+ expect(processed['http://example.com/3'].retryCount).to.be.eql(0);
+
+ expect(processed['http://example.com/2'].userData.foo).to.be.eql(undefined);
+ expect(processed['http://example.com/2'].errorMessages).to.have.lengthOf(4);
+ expect(processed['http://example.com/2'].retryCount).to.be.eql(3);
+
+ expect(handleFailedRequestFunctionCalls).to.be.eql(1);
+
+ expect(await requestList.isFinished()).to.be.eql(false);
+ expect(await requestList.isEmpty()).to.be.eql(false);
+ });
});