Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Refactor PuppeteerCrawler, add .stop() to CheerioCrawler; update changelog

Ondra Urbancffd6a8fe068
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 926b2a5fc..051273400 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,8 +1,11 @@
xxxxxxxxxxxxxxxxxxx
===================
- Added `getInfo()` method to Dataset to get meta-information about a dataset.
-- Added CheerioCrawler, a specialized class for crawling the web using `cheerio`
-- Added `keepDuplicateUrls` option to RequestList to allow duplicate URLs
+- Added CheerioCrawler, a specialized class for crawling the web using `cheerio`.
+- Added `keepDuplicateUrls` option to RequestList to allow duplicate URLs.
+- Added `.stop()` method to all Crawler classes to enable stopping the crawl programmatically.
+- Deprecated `pageOpsTimeoutMillis` option. Use `handlePageTimeoutSecs`.
+- Bluebird promises are being phased out of `apify` in favor of `async-await`.
0.6.1 / 2018-08-17
===================
diff --git a/package.json b/package.json
index 396a907a0..835b6e968 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
{
"name": "apify",
- "version": "0.6.1",
+ "version": "0.6.2",
"description": "Web scraping and automation SDK",
"engines": {
"node": ">=8.0.0"
@@ -49,7 +49,7 @@
},
"dependencies": {
"apify-client": "^0.2.9",
- "apify-shared": "0.0.48",
+ "apify-shared": "0.0.49",
"bluebird": "^3.5.0",
"content-type": "^1.0.3",
"fs-extra": "^7.0.0",
diff --git a/src/basic_crawler.js b/src/basic_crawler.js
index 24f47eac3..f0a67ed59 100644
--- a/src/basic_crawler.js
+++ b/src/basic_crawler.js
@@ -213,7 +213,7 @@ export default class BasicCrawler {
// If requestQueue.addRequest() fails here then we must reclaim it back to
// the RequestList because probably it's not yet in the queue!
log.exception(err, 'RequestQueue.addRequest() failed, reclaiming request back to the list', { request });
- await this.requestList.reclaimRequest();
+ await this.requestList.reclaimRequest(request);
return null;
}
const [nextRequest] = await Promise.all([
diff --git a/src/cheerio_crawler.js b/src/cheerio_crawler.js
index 3ca1493fc..29275158e 100644
--- a/src/cheerio_crawler.js
+++ b/src/cheerio_crawler.js
@@ -159,8 +159,14 @@ export default class CheerioCrawler {
this.ignoreSslErrors = ignoreSslErrors;
+ this.requestFunction = async ({ request }) => {
+ if (!this.isRunning) throw new Error('CheerioCrawler is stopped.');
+
+ return requestFunction
+ ? requestFunction({ request })
+ : this._defaultRequestFunction({ request });
+ };
this.handlePageFunction = handlePageFunction;
- this.requestFunction = requestFunction || this._defaultRequestFunction;
this.handlePageTimeoutSecs = handlePageTimeoutSecs;
this.basicCrawler = new BasicCrawler({
@@ -188,7 +194,27 @@ export default class CheerioCrawler {
* @return {Promise}
*/
async run() {
- return this.basicCrawler.run();
+ if (this.isRunning) return this.isRunningPromise;
+
+ this.isRunning = true;
+ this.rejectOnStopPromise = new Promise((r, reject) => { this.rejectOnStop = reject; });
+ try {
+ this.isRunningPromise = this.basicCrawler.run();
+ await this.isRunningPromise;
+ this.isRunning = false;
+ } catch (err) {
+ this.isRunning = false; // Doing this before rejecting to make sure it's set when error handlers fire.
+ this.rejectOnStop(err);
+ }
+ }
+
+ /**
+ * Stops the crawler by preventing crawls of additional pages. Pages already running are terminated.
+ */
+ stop() {
+ this.isRunning = false;
+ this.basicCrawler.stop();
+ this.rejectOnStop(new Error('PuppeteerCrawler: .stop() function has been called. Stopping the crawler.'));
}
/**
@@ -197,12 +223,18 @@ export default class CheerioCrawler {
* @ignore
*/
async _handleRequestFunction({ request }) {
+ if (!this.isRunning) throw new Error('CheerioCrawler is stopped.');
+
const html = await this.requestFunction({ request });
const $ = cheerio.load(html);
- await Promise.race([
+ const pageHandledOrTimedOutPromise = Promise.race([
this.handlePageFunction({ $, html, request }),
createTimeoutPromise(this.handlePageTimeoutSecs * 1000, 'CheerioCrawler: handlePageFunction timed out.'),
]);
+
+ // rejectOnStopPromise rejects when .stop() is called or BasicCrawler throws.
+ // All running pages are therefore terminated with an error to be reclaimed and retried.
+ return Promise.race([pageHandledOrTimedOutPromise, this.rejectOnStopPromise]);
}
/**
@@ -210,11 +242,16 @@ export default class CheerioCrawler {
* @ignore
*/
async _defaultRequestFunction({ request }) {
- return rp({
- url: request.url,
- method: request.method,
- headers: request.headers,
- strictSSL: !this.ignoreSslErrors,
- });
+ if (!this.isRunning) throw new Error('CheerioCrawler is stopped.');
+
+ return Promise.race([
+ rp({
+ url: request.url,
+ method: request.method,
+ headers: request.headers,
+ strictSSL: !this.ignoreSslErrors,
+ }),
+ this.rejectOnStopPromise,
+ ]);
}
}
diff --git a/src/puppeteer_crawler.js b/src/puppeteer_crawler.js
index ce21c5299..0d623275b 100644
--- a/src/puppeteer_crawler.js
+++ b/src/puppeteer_crawler.js
@@ -1,14 +1,14 @@
import { checkParamOrThrow } from 'apify-client/build/utils';
import log from 'apify-shared/log';
import _ from 'underscore';
-import Promise from 'bluebird';
import BasicCrawler from './basic_crawler';
import PuppeteerPool from './puppeteer_pool';
-import { isPromise } from './utils';
+import { createTimeoutPromise } from './utils';
const DEFAULT_OPTIONS = {
- gotoFunction: ({ request, page }) => page.goto(request.url, { timeout: 60000 }),
+ gotoFunction: async ({ request, page }) => page.goto(request.url, { timeout: 60000 }),
pageOpsTimeoutMillis: 300000,
+ handlePageTimeoutSecs: 300,
};
const PAGE_CLOSE_TIMEOUT_MILLIS = 30000;
@@ -63,8 +63,8 @@ const PAGE_CLOSE_TIMEOUT_MILLIS = 30000;
* It is passed an object with the following fields:
* `request` is an instance of the `Request` object with details about the URL to open, HTTP method etc.
* `page` is an instance of the `Puppeteer.Page` class with `page.goto(request.url)` already called.
- * @param {Number} [options.pageOpsTimeoutMillis=300000]
- * Timeout in which the function passed as `options.handlePageFunction` needs to finish.
+ * @param {Number} [options.handlePageTimeoutSecs=300]
+ * Timeout in which the function passed as `options.handlePageFunction` needs to finish, in seconds.
* @param {Function} [options.gotoFunction=({ request, page }) => page.goto(request.url, { timeout: 60000 })]
* Overrides the function that opens the request in Puppeteer.
* The function should return a result of Puppeteer's
@@ -138,6 +138,7 @@ export default class PuppeteerCrawler {
handlePageFunction,
gotoFunction,
pageOpsTimeoutMillis,
+ handlePageTimeoutSecs,
// Autoscaled pool options
maxMemoryMbytes,
@@ -168,7 +169,7 @@ export default class PuppeteerCrawler {
this.handlePageFunction = handlePageFunction;
this.gotoFunction = gotoFunction;
- this.pageOpsTimeoutMillis = pageOpsTimeoutMillis;
+ this.handlePageTimeoutSecs = handlePageTimeoutSecs || Math.ceil(pageOpsTimeoutMillis / 1000);
this.puppeteerPoolOptions = {
maxOpenPagesPerInstance,
@@ -205,24 +206,31 @@ export default class PuppeteerCrawler {
*
* @return {Promise}
*/
- run() {
- if (this.isStopped) this.puppeteerPool = new PuppeteerPool(this.puppeteerPoolOptions);
- this.isStopped = false;
- this.isRunningPromise = new Promise((resolve, reject) => {
- this.isRunningReject = reject;
- });
- return this.basicCrawler.run()
- .finally(() => this.puppeteerPool.destroy());
+ async run() {
+ if (this.isRunning) return this.isRunningPromise;
+
+ this.puppeteerPool = new PuppeteerPool(this.puppeteerPoolOptions);
+ this.isRunning = true;
+ this.rejectOnStopPromise = new Promise((r, reject) => { this.rejectOnStop = reject; });
+ try {
+ this.isRunningPromise = this.basicCrawler.run();
+ await this.isRunningPromise;
+ this.isRunning = false;
+ } catch (err) {
+ this.isRunning = false; // Doing this before rejecting to make sure it's set when error handlers fire.
+ this.rejectOnStop(err);
+ } finally {
+ this.puppeteerPool.destroy();
+ }
}
/**
* Stops the crawler by preventing crawls of additional pages. Pages already running are terminated.
*/
stop() {
- this.isStopped = true;
- this.isRunningReject();
+ this.isRunning = false;
this.basicCrawler.stop();
- return this.puppeteerPool.destroy();
+ this.rejectOnStop(new Error('PuppeteerCrawler: .stop() function has been called. Stopping the crawler.'));
}
/**
@@ -230,35 +238,27 @@ export default class PuppeteerCrawler {
*
* @ignore
*/
- _handleRequestFunction({ request }) {
- if (this.isStopped) return Promise.reject(new Error('PuppeteerCrawler is stopped.')); // Pool will be destroyed.
- let page;
+ async _handleRequestFunction({ request }) {
+ if (!this.isRunning) throw new Error('PuppeteerCrawler is stopped.'); // Pool will be destroyed.
- const returnPromise = this.puppeteerPool
- .newPage()
- .then((newPage) => { page = newPage; })
- .then(() => this.gotoFunction({ page, request, puppeteerPool: this.puppeteerPool }))
- .then((response) => {
- const promise = this.handlePageFunction({
- page,
- request,
- puppeteerPool: this.puppeteerPool,
- response,
- });
+ const page = await this.puppeteerPool.newPage();
+ const response = await this.gotoFunction({ page, request, puppeteerPool: this.puppeteerPool });
- if (!isPromise(promise)) throw new Error('User provided handlePageFunction must return a Promise.');
+ const pageHandledOrTimedOutPromise = Promise.race([
+ this.handlePageFunction({ page, request, puppeteerPool: this.puppeteerPool, response }),
+ createTimeoutPromise(this.handlePageTimeoutSecs * 1000, 'PuppeteerCrawler: handlePageFunction timed out.'),
+ ]);
- return promise;
- })
- .timeout(this.pageOpsTimeoutMillis, 'PuppeteerCrawler: handlePageFunction timed out.')
- .finally(() => {
- return Promise.resolve()
- .then(() => {
- if (page) return page.close();
- })
- .timeout(PAGE_CLOSE_TIMEOUT_MILLIS, 'Operation timed out.')
- .catch(err => log.debug('PuppeteerCrawler: Page.close() failed.', { reason: err ? err.message : err }));
- });
- return Promise.race([returnPromise, this.isRunningPromise]); // Reject all requests immediately after stopping the crawler (to be reclaimed).
+ try {
+ // rejectOnStopPromise rejects when .stop() is called or BasicCrawler throws.
+ // All running pages are therefore terminated with an error to be reclaimed and retried.
+ return await Promise.race([pageHandledOrTimedOutPromise, this.rejectOnStopPromise]);
+ } finally {
+ try {
+ await Promise.race([page.close(), createTimeoutPromise(PAGE_CLOSE_TIMEOUT_MILLIS, 'Operation timed out.')]);
+ } catch (err) {
+ log.debug('PuppeteerCrawler: Page.close() failed.', { reason: err && err.message });
+ }
+ }
}
}
diff --git a/test/basic_crawler.js b/test/basic_crawler.js
index 8e7eeb682..86b3e6472 100644
--- a/test/basic_crawler.js
+++ b/test/basic_crawler.js
@@ -3,6 +3,7 @@ import chaiAsPromised from 'chai-as-promised';
import _ from 'underscore';
import 'babel-polyfill';
import sinon from 'sinon';
+import log from 'apify-shared/log';
import { delayPromise } from 'apify-shared/utilities';
import * as Apify from '../build/index';
import { RequestQueue, RequestQueueLocal } from '../build/request_queue';
@@ -11,6 +12,17 @@ import { LOCAL_EMULATION_DIR } from './_helper';
chai.use(chaiAsPromised);
describe('BasicCrawler', () => {
+ let logLevel;
+
+ before(() => {
+ logLevel = log.getLevel();
+ log.setLevel(log.LEVELS.ERROR);
+ });
+
+ after(() => {
+ log.setLevel(logLevel);
+ });
+
it('should run in parallel thru all the requests', async () => {
const startedAt = Date.now();
const sources = _.range(0, 500).map(index => ({ url: `https://example.com/${index}` }));
diff --git a/test/cheerio_crawler.js b/test/cheerio_crawler.js
index 960cbf825..7065ed34f 100644
--- a/test/cheerio_crawler.js
+++ b/test/cheerio_crawler.js
@@ -1,10 +1,22 @@
import chai, { expect } from 'chai';
import chaiAsPromised from 'chai-as-promised';
+import log from 'apify-shared/log';
import * as Apify from '../build/index';
chai.use(chaiAsPromised);
describe('CheerioCrawler', () => {
+ let logLevel;
+
+ before(() => {
+ logLevel = log.getLevel();
+ log.setLevel(log.LEVELS.ERROR);
+ });
+
+ after(() => {
+ log.setLevel(logLevel);
+ });
+
it('should work', async () => {
const sources = [
{ url: 'http://example.com/?q=1' },
diff --git a/test/puppeteer_crawler.js b/test/puppeteer_crawler.js
index e9be0011f..f0d9be506 100644
--- a/test/puppeteer_crawler.js
+++ b/test/puppeteer_crawler.js
@@ -2,6 +2,7 @@ import chai, { expect } from 'chai';
import chaiAsPromised from 'chai-as-promised';
import 'babel-polyfill';
import _ from 'underscore';
+import log from 'apify-shared/log';
import { ENV_VARS } from '../build/constants';
import * as Apify from '../build/index';
@@ -9,13 +10,17 @@ chai.use(chaiAsPromised);
describe('PuppeteerCrawler', () => {
let prevEnvHeadless;
+ let logLevel;
before(() => {
prevEnvHeadless = process.env[ENV_VARS.HEADLESS];
process.env[ENV_VARS.HEADLESS] = '1';
+ logLevel = log.getLevel();
+ log.setLevel(log.LEVELS.ERROR);
});
after(() => {
+ log.setLevel(logLevel);
process.env[ENV_VARS.HEADLESS] = prevEnvHeadless;
});