Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Fix memory leak

Ondra Urban3010caf5f559
diff --git a/src/autoscaling/autoscaled_pool.js b/src/autoscaling/autoscaled_pool.js
index 4894c703e..43d382487 100644
--- a/src/autoscaling/autoscaled_pool.js
+++ b/src/autoscaling/autoscaled_pool.js
@@ -161,7 +161,7 @@ class AutoscaledPool {
this.isTaskReadyFunction = isTaskReadyFunction;
// Internal properties.
- this.isPaused = false;
+ this.isStopped = false;
this.desiredConcurrency = this.minConcurrency;
this.currentConcurrency = 0;
this.lastLoggingTime = 0;
@@ -233,6 +233,7 @@ class AutoscaledPool {
* @return {Promise}
*/
async abort() {
+ this.isStopped = true;
if (this.resolve) {
this.resolve();
await this._destroy();
@@ -250,7 +251,8 @@ class AutoscaledPool {
* @return {Promise}
*/
async pause(timeoutSecs) {
- this.isPaused = true;
+ if (this.isStopped) return;
+ this.isStopped = true;
return new Promise((resolve, reject) => {
let timeout;
if (timeoutSecs) {
@@ -279,7 +281,7 @@ class AutoscaledPool {
* Tasks will automatically start running again in `options.maybeRunIntervalSecs`.
*/
resume() {
- this.isPaused = false;
+ this.isStopped = false;
}
/**
@@ -297,8 +299,8 @@ class AutoscaledPool {
const done = intervalCallback || (() => {});
// Prevent starting a new task if:
- // - the pool is paused
- if (this.isPaused) return done();
+ // - the pool is paused or aborted
+ if (this.isStopped) return done();
// - we are already querying for a task.
if (this.queryingIsTaskReady) return done();
// - we would exceed desired concurrency.
@@ -365,7 +367,7 @@ class AutoscaledPool {
*/
_autoscale(intervalCallback) {
// Don't scale if paused.
- if (this.isPaused) return intervalCallback();
+ if (this.isStopped) return intervalCallback();
// Only scale up if:
// - system has not been overloaded lately.
diff --git a/src/basic_crawler.js b/src/basic_crawler.js
index e7b2e89d2..5e7451863 100644
--- a/src/basic_crawler.js
+++ b/src/basic_crawler.js
@@ -187,18 +187,13 @@ class BasicCrawler {
const basicCrawlerAutoscaledPoolConfiguration = {
minConcurrency,
maxConcurrency,
- runTaskFunction: async () => {
- if (!this.isRunning) return null;
-
- return this._runTaskFunction();
- },
+ runTaskFunction: this._runTaskFunction.bind(this),
isTaskReadyFunction: async () => {
- if (isMaxPagesExceeded() || !this.isRunning) return false;
+ if (isMaxPagesExceeded()) return false;
return this._isTaskReadyFunction();
},
isFinishedFunction: async () => {
- if (!this.isRunning) return true;
if (isMaxPagesExceeded()) {
log.info('BasicCrawler: Crawler reached the max requests per crawl limit by crawling '
+ `${this.handledRequestsCount} requests and will shut down.`);
@@ -222,6 +217,8 @@ class BasicCrawler {
this.autoscaledPoolOptions = _.defaults({}, basicCrawlerAutoscaledPoolConfiguration, autoscaledPoolOptions);
+ this.isRunningPromise = null;
+
// Attach a listener to handle migration events gracefully.
events.on(ACTOR_EVENT_NAMES.MIGRATING, this._pauseOnMigration.bind(this));
}
@@ -232,35 +229,15 @@ class BasicCrawler {
* @return {Promise}
*/
async run() {
- if (this.isRunning) return this.isRunningPromise;
+ if (this.isRunningPromise) return this.isRunningPromise;
await this._loadHandledRequestCount();
this.autoscaledPool = new AutoscaledPool(this.autoscaledPoolOptions);
- this.isRunning = true;
- this.rejectOnAbortPromise = new Promise((r, reject) => { this.rejectOnAbort = reject; });
this.isRunningPromise = this.autoscaledPool.run();
- try {
- await this.isRunningPromise;
- this.isRunning = false;
- } catch (err) {
- this.isRunning = false; // Doing this before rejecting to make sure it's set when error handlers fire.
- this.rejectOnAbort(err);
- }
- }
-
- /**
- * Aborts the crawler by preventing additional requests and terminating the running ones.
- *
- * @return {Promise}
- */
- async abort() {
- this.isRunning = false;
- await this.autoscaledPool.abort();
- this.rejectOnAbort(new Error('BasicCrawler: .abort() function has been called. Aborting the crawler.'));
+ await this.isRunningPromise;
}
async _pauseOnMigration() {
- if (!this.isRunning) return;
await this.autoscaledPool.pause(SAFE_MIGRATION_WAIT_MILLIS)
.catch(() => {
log.error('BasicCrawler: The crawler was paused due to migration to another host, '
@@ -268,10 +245,15 @@ class BasicCrawler {
});
if (this.requestList) {
await this.requestList.persistState()
- .catch(() => {
- log.error('BasicCrawler: The crawler attempted to persist it\'s request list\'s state and failed due to invalid config. '
- + 'Make sure to use either Apify.openRequestList() or the "stateKeyPrefix" option of RequestList constructor '
- + 'to ensure your crawling state is persisted through host migrations and restarts.');
+ .catch((err) => {
+ if (err.message.includes('Cannot persist state.')) {
+ log.error('BasicCrawler: The crawler attempted to persist it\'s request list\'s state and failed due to invalid config. '
+ + 'Make sure to use either Apify.openRequestList() or the "stateKeyPrefix" option of RequestList constructor '
+ + 'to ensure your crawling state is persisted through host migrations and restarts.');
+ } else {
+ log.exception(err, 'BasicCrawler: An unexpected error occured when the crawler '
+ + 'attempted to persist it\'s request list\'s state. ');
+ }
});
}
}
@@ -317,9 +299,7 @@ class BasicCrawler {
if (!request) return;
try {
- // rejectOnAbortPromise rejects when .abort() is called or AutoscaledPool throws.
- // All running tasks are therefore terminated with an error to be reclaimed and retried.
- await Promise.race([this.handleRequestFunction({ request }), this.rejectOnAbortPromise]);
+ await this.handleRequestFunction({ request, autoscaledPool: this.autoscaledPool });
await source.markRequestHandled(request);
this.handledRequestsCount++;
} catch (err) {
@@ -381,11 +361,6 @@ class BasicCrawler {
* @ignore
*/
async _requestFunctionErrorHandler(error, request, source) {
- // Handles case where the crawler was aborted.
- // All running requests are reclaimed and will be retried.
- if (!this.isRunning) return source.reclaimRequest(request);
-
- // If we got here, it means we actually want to process the error.
request.pushErrorMessage(error);
// Reclaim and retry request if flagged as retriable and retryCount is not exceeded.
diff --git a/src/cheerio_crawler.js b/src/cheerio_crawler.js
index e6af53c3f..bceb0884e 100644
--- a/src/cheerio_crawler.js
+++ b/src/cheerio_crawler.js
@@ -231,17 +231,10 @@ class CheerioCrawler {
if (proxyUrls && !proxyUrls.length) throw new Error('Parameter "options.proxyUrls" of type Array must not be empty');
if (useApifyProxy && proxyUrls) throw new Error('Cannot combine "options.useApifyProxy" with "options.proxyUrls"!');
- this.requestFunction = async ({ request }) => {
- if (!this.isRunning) throw new Error('CheerioCrawler is stopped.');
- const rfPromise = requestFunction
- ? requestFunction({ request })
- : this._defaultRequestFunction({ request });
-
- // Return the response of requestFunction or throw.
- return Promise.race([
- rfPromise,
- this.rejectOnAbortPromise,
- ]);
+ this.requestFunction = async ({ request, autoscaledPool }) => {
+ return requestFunction
+ ? requestFunction({ request, autoscaledPool })
+ : this._defaultRequestFunction({ request, autoscaledPool });
};
this.requestOptions = requestOptions;
this.handlePageFunction = handlePageFunction;
@@ -271,6 +264,7 @@ class CheerioCrawler {
// See the _suppressTunnelAgentAssertError function.
this.tunnelAgentExceptionListener = null;
+ this.isRunningPromise = null;
}
/**
@@ -279,32 +273,13 @@ class CheerioCrawler {
* @return {Promise}
*/
async run() {
- if (this.isRunning) return this.isRunningPromise;
+ if (this.isRunningPromise) return this.isRunningPromise;
- this.isRunning = true;
- this.rejectOnAbortPromise = new Promise((r, reject) => { this.rejectOnAbort = reject; });
this._suppressTunnelAgentAssertError();
- try {
- this.isRunningPromise = this.basicCrawler.run();
- await this.isRunningPromise;
- this.isRunning = false;
- process.removeListener('uncaughtException', this.tunnelAgentExceptionListener);
- this.tunnelAgentExceptionListener = null;
- } catch (err) {
- this.isRunning = false; // Doing this before rejecting to make sure it's set when error handlers fire.
- this.rejectOnAbort(err);
- }
- }
-
- /**
- * Aborts the crawler by preventing crawls of additional pages and terminating the running ones.
- *
- * @return {Promise}
- */
- async abort() {
- this.isRunning = false;
- await this.basicCrawler.abort();
- this.rejectOnAbort(new Error('CheerioCrawler: .abort() function has been called. Aborting the crawler.'));
+ this.isRunningPromise = this.basicCrawler.run();
+ await this.isRunningPromise;
+ process.removeListener('uncaughtException', this.tunnelAgentExceptionListener);
+ this.tunnelAgentExceptionListener = null;
}
/**
@@ -312,17 +287,12 @@ class CheerioCrawler {
*
* @ignore
*/
- async _handleRequestFunction({ request }) {
- if (!this.isRunning) throw new Error('CheerioCrawler is stopped.');
-
- const rfPromise = this.requestFunction({ request });
- // rejectOnAbortPromise rejects when .abort() is called or BasicCrawler throws.
- // All running pages are therefore terminated with an error to be reclaimed and retried.
- const response = await Promise.race([
- addTimeoutToPromise(rfPromise, this.requestTimeoutMillis, 'CheerioCrawler: requestFunction timed out.'),
- this.rejectOnAbortPromise,
- ]);
-
+ async _handleRequestFunction({ request, autoscaledPool }) {
+ const response = await addTimeoutToPromise(
+ this.requestFunction({ request, autoscaledPool }),
+ this.requestTimeoutMillis,
+ 'CheerioCrawler: requestFunction timed out.',
+ );
let html;
if (typeof response === 'string') html = response;
@@ -330,14 +300,11 @@ class CheerioCrawler {
else throw new Error('CheerioCrawler: requestFunction returned neither string, nor an object with a body property of type string.');
const $ = cheerio.load(html);
- await Promise.race([
- addTimeoutToPromise(
- this.handlePageFunction({ $, html, request, response }),
- this.handlePageTimeoutMillis,
- 'CheerioCrawler: handlePageFunction timed out.',
- ),
- this.rejectOnAbortPromise,
- ]);
+ return addTimeoutToPromise(
+ this.handlePageFunction({ $, html, request, response, autoscaledPool }),
+ this.handlePageTimeoutMillis,
+ 'CheerioCrawler: handlePageFunction timed out.',
+ );
}
/**
@@ -502,7 +469,7 @@ class CheerioCrawler {
_suppressTunnelAgentAssertError() {
// Only set the handler if it's not already set.
if (this.tunnelAgentExceptionListener) return;
- this.tunnelAgentExceptionListener = process.on('uncaughtException', (err) => {
+ this.tunnelAgentExceptionListener = (err) => {
try {
const code = err.code === 'ERR_ASSERTION';
const name = err.name === 'AssertionError [ERR_ASSERTION]';
@@ -522,7 +489,8 @@ class CheerioCrawler {
}
// Rethrow the original error if it's not a match.
throw err;
- });
+ };
+ process.on('uncaughtException', this.tunnelAgentExceptionListener);
}
}
diff --git a/src/puppeteer_crawler.js b/src/puppeteer_crawler.js
index 0bc6155f9..bf278a4c2 100644
--- a/src/puppeteer_crawler.js
+++ b/src/puppeteer_crawler.js
@@ -270,59 +270,38 @@ class PuppeteerCrawler {
* @return {Promise}
*/
async run() {
- if (this.isRunning) return this.isRunningPromise;
+ if (this.isRunningPromise) return this.isRunningPromise;
this.puppeteerPool = new PuppeteerPool(this.puppeteerPoolOptions);
- this.isRunning = true;
- this.rejectOnAbortPromise = new Promise((r, reject) => { this.rejectOnAbort = reject; });
try {
this.isRunningPromise = this.basicCrawler.run();
await this.isRunningPromise;
- this.isRunning = false;
- } catch (err) {
- this.isRunning = false; // Doing this before rejecting to make sure it's set when error handlers fire.
- this.rejectOnAbort(err);
} finally {
this.puppeteerPool.destroy();
}
}
- /**
- * Aborts the crawler by preventing crawls of additional pages and terminating the running ones.
- *
- * @return {Promise}
- */
- async abort() {
- this.isRunning = false;
- await this.basicCrawler.abort();
- this.rejectOnAbort(new Error('PuppeteerCrawler: .abort() function has been called. Aborting the crawler.'));
- }
-
/**
* Wrapper around handlePageFunction that opens and closes pages etc.
*
* @ignore
*/
- async _handleRequestFunction({ request }) {
- if (!this.isRunning) throw new Error('PuppeteerCrawler is stopped.'); // Pool will be destroyed.
-
+ async _handleRequestFunction({ request, autoscaledPool }) {
const page = await this.puppeteerPool.newPage();
-
try {
- const pageOperationsPromise = this
- .gotoFunction({ page, request, puppeteerPool: this.puppeteerPool })
- .then(response => addTimeoutToPromise(
- this.handlePageFunction({ page, request, puppeteerPool: this.puppeteerPool, response }),
- this.handlePageTimeoutMillis,
- 'PuppeteerCrawler: handlePageFunction timed out.',
- ));
-
- // rejectOnAbortPromise rejects when .abort() is called or BasicCrawler throws.
- // All running pages are therefore terminated with an error to be reclaimed and retried.
- return await Promise.race([pageOperationsPromise, this.rejectOnAbortPromise]);
+ const response = await this.gotoFunction({ page, request, autoscaledPool, puppeteerPool: this.puppeteerPool });
+ await addTimeoutToPromise(
+ this.handlePageFunction({ page, request, puppeteerPool: this.puppeteerPool, response }),
+ this.handlePageTimeoutMillis,
+ 'PuppeteerCrawler: handlePageFunction timed out.',
+ );
} finally {
try {
- await addTimeoutToPromise(this.puppeteerPool.recyclePage(page), PAGE_CLOSE_TIMEOUT_MILLIS, 'Operation timed out.');
+ await addTimeoutToPromise(
+ this.puppeteerPool.recyclePage(page),
+ PAGE_CLOSE_TIMEOUT_MILLIS,
+ 'Operation timed out.',
+ );
} catch (err) {
log.debug('PuppeteerCrawler: Page.close() failed.', { reason: err && err.message });
}
diff --git a/test/basic_crawler.js b/test/basic_crawler.js
index c639dcf44..1e918963e 100644
--- a/test/basic_crawler.js
+++ b/test/basic_crawler.js
@@ -51,50 +51,6 @@ describe('BasicCrawler', () => {
expect(await requestList.isEmpty()).to.be.eql(true);
});
- it('should abort and resume', async () => {
- const sources = _.range(500).map(index => ({ url: `https://example.com/${index + 1}` }));
-
- let basicCrawler;
- let isStopped;
- let resolveAbortPromise;
- const abortPromise = new Promise((res) => { resolveAbortPromise = res; });
- const processed = [];
- const requestList = new Apify.RequestList({ sources });
- const handleRequestFunction = async ({ request }) => {
- if (request.url.endsWith('200') && !isStopped) {
- await basicCrawler.abort().then(resolveAbortPromise);
- isStopped = true;
- } else {
- await delayPromise(10);
- processed.push(_.pick(request, 'url'));
- }
- };
-
- basicCrawler = new Apify.BasicCrawler({
- requestList,
- minConcurrency: 25,
- maxConcurrency: 25,
- handleRequestFunction,
- });
-
- await requestList.initialize();
-
- // The crawler will stop after 200 requests
- await basicCrawler.run();
-
- expect(processed.length).to.be.within(175, 200);
- expect(await requestList.isFinished()).to.be.eql(false);
- expect(await requestList.isEmpty()).to.be.eql(false);
-
- // We need to check if the abort procedure finished before restarting, otherwise everything goes crazy.
- await abortPromise;
- await basicCrawler.run();
- expect(processed.length).to.be.within(500, 525);
- expect(new Set(processed.map(p => p.url))).to.be.eql(new Set(sources.map(s => s.url)));
- expect(await requestList.isFinished()).to.be.eql(true);
- expect(await requestList.isEmpty()).to.be.eql(true);
- });
-
it('should pause on migration event and persist RequestList state', async () => {
const sources = _.range(500).map(index => ({ url: `https://example.com/${index + 1}` }));
diff --git a/test/cheerio_crawler.js b/test/cheerio_crawler.js
index 7c833bda4..e3a2090ea 100644
--- a/test/cheerio_crawler.js
+++ b/test/cheerio_crawler.js
@@ -121,62 +121,6 @@ describe('CheerioCrawler', () => {
});
});
- it('should abort and resume', async () => {
- const sources = _.range(100).map(index => ({ url: `https://example.com/?q=${index + 1}` }));
- let cheerioCrawler;
- let isStopped = false;
- const processed = [];
- const failed = [];
- const requestList = new Apify.RequestList({ sources });
- const requestFunction = async ({ request }) => {
- await delayPromise(2);
- return `<html><head><title>${request.url[request.url.length - 1]}</title></head><body>Body</body></html>`;
- };
- const handlePageFunction = async ({ $, html, request }) => {
- if (request.url.endsWith('45') && !isStopped) {
- await cheerioCrawler.abort();
- isStopped = true;
- } else {
- request.userData.title = $('title').text();
- request.userData.html = html;
- processed.push(request);
- }
- };
-
- cheerioCrawler = new Apify.CheerioCrawler({
- requestList,
- minConcurrency: 5,
- maxConcurrency: 5,
- requestFunction,
- handlePageFunction,
- handleFailedRequestFunction: ({ request }) => failed.push(request),
- });
-
- await requestList.initialize();
- await cheerioCrawler.run();
-
- expect(processed.length).to.be.within(40, 50);
- expect(failed).to.have.lengthOf(0);
-
- processed.sort(comparator);
-
- for (let i = 0; i < 40; i++) {
- const request = processed[i];
- expect(request.url).to.be.eql(sources[i].url);
- expect(request.userData.title).to.be.eql(request.url[request.url.length - 1]);
- }
-
- await Apify.utils.sleep(10); // Wait for event loop to unwind.
- await cheerioCrawler.run();
-
- expect(processed.length).to.be.within(100, 110);
- expect(failed).to.have.lengthOf(0);
- expect(new Set(processed.map(p => p.url))).to.be.eql(new Set(sources.map(s => s.url)));
- processed.forEach((request) => {
- expect(request.userData.title).to.be.eql(request.url[request.url.length - 1]);
- });
- });
-
describe('should timeout', () => {
let ll;
before(() => {
diff --git a/test/puppeteer_crawler.js b/test/puppeteer_crawler.js
index 7f697904a..c35247242 100644
--- a/test/puppeteer_crawler.js
+++ b/test/puppeteer_crawler.js
@@ -63,63 +63,6 @@ describe('PuppeteerCrawler', () => {
});
});
- it('should stop and resume', async () => {
- const comparator = (a, b) => {
- a = Number(/q=(\d+)$/.exec(a.url)[1]);
- b = Number(/q=(\d+)$/.exec(b.url)[1]);
- return a - b;
- };
- const sources = _.range(30).map(index => ({ url: `https://example.com/?q=${index + 1}` }));
- let puppeteerCrawler;
- let isStopped = false;
- const processed = [];
- const failed = [];
- const requestList = new Apify.RequestList({ sources });
- const handlePageFunction = async ({ page, request, response }) => {
- if (request.url.endsWith('15') && !isStopped) {
- await puppeteerCrawler.abort();
- isStopped = true;
- } else {
- await page.waitFor('title');
- expect(await response.status()).to.be.eql(200);
- request.userData.title = await page.title();
- processed.push(request);
- }
- };
-
- puppeteerCrawler = new Apify.PuppeteerCrawler({
- requestList,
- minConcurrency: 3,
- maxConcurrency: 3,
- handlePageFunction,
- handleFailedRequestFunction: ({ request }) => failed.push(request),
- });
-
- await requestList.initialize();
- await puppeteerCrawler.run();
-
- expect(processed.length).to.be.within(12, 15);
- expect(failed).to.have.lengthOf(0);
-
- processed.sort(comparator);
-
- for (let i = 0; i < 12; i++) {
- const request = processed[i];
- expect(request.url).to.be.eql(sources[i].url);
- expect(request.userData.title).to.be.eql('Example Domain');
- }
-
- await Apify.utils.sleep(10); // Wait for event loop to unwind.
- await puppeteerCrawler.run();
-
- expect(processed.length).to.be.within(30, 33);
- expect(failed).to.have.lengthOf(0);
- expect(new Set(processed.map(p => p.url))).to.be.eql(new Set(sources.map(s => s.url)));
- processed.forEach((request) => {
- expect(request.userData.title).to.be.eql('Example Domain');
- });
- });
-
it('should ignore errors in Page.close()', async () => {
for (let i = 0; i < 2; i++) {
const requestList = new Apify.RequestList({