Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Refactor Crawlers to inherit from BasicCrawler (#870)
Ondra Urband6935c074ce5
diff --git a/.babelrc b/.babelrcindex 98ed8e4e2..b85d9ffef 100644--- a/.babelrc+++ b/.babelrc@@ -6,5 +6,6 @@"node": "10.17.0"}}]- ]+ ],+ "plugins": ["@babel/plugin-syntax-class-properties"]}diff --git a/.eslintrc.json b/.eslintrc.jsonindex a0c8140b4..ef6d79366 100644--- a/.eslintrc.json+++ b/.eslintrc.json@@ -1,3 +1,4 @@{- "extends": ["@apify"]+ "extends": ["@apify"],+ "parser": "@babel/eslint-parser"}diff --git a/package.json b/package.jsonindex 6943acc54..93153648d 100644--- a/package.json+++ b/package.json@@ -86,6 +86,8 @@"@babel/core": "^7.11.6","@babel/preset-env": "^7.11.5","@babel/register": "^7.11.5",+ "@babel/eslint-parser": "^7.12.1",+ "@babel/plugin-syntax-class-properties": "^7.12.1","@types/content-type": "^1.1.3","@types/fs-extra": "^9.0.2","@types/htmlparser2": "^3.10.2",diff --git a/src/crawlers/basic_crawler.js b/src/crawlers/basic_crawler.jsindex 1fcb098cd..7cc9ed1d5 100644--- a/src/crawlers/basic_crawler.js+++ b/src/crawlers/basic_crawler.js@@ -173,40 +173,42 @@ const SAFE_MIGRATION_WAIT_MILLIS = 20000;* or to abort it by calling {@link AutoscaledPool#abort}.*/class BasicCrawler {+ static optionsShape = {+ requestList: ow.optional.object.validate(validators.requestList),+ requestQueue: ow.optional.object.validate(validators.requestQueue),+ // Subclasses override this function instead of passing it+ // in constructor, so this validation needs to apply only+ // if the user creates an instance of BasicCrawler directly.+ handleRequestFunction: ow.function,+ handleRequestTimeoutSecs: ow.optional.number,+ handleFailedRequestFunction: ow.optional.function,+ maxRequestRetries: ow.optional.number,+ maxRequestsPerCrawl: ow.optional.number,+ autoscaledPoolOptions: ow.optional.object,+ sessionPoolOptions: ow.optional.object,+ useSessionPool: ow.optional.boolean,++ // AutoscaledPool shorthands+ minConcurrency: ow.optional.number,+ maxConcurrency: ow.optional.number,++ // internal+ log: ow.optional.object,+ };+/*** @param {BasicCrawlerOptions} options* All `BasicCrawler` parameters are passed via an options object.*/constructor(options) {- ow(options, ow.object.exactShape({- requestList: ow.optional.object.validate(validators.requestList),- requestQueue: ow.optional.object.validate(validators.requestQueue),- handleRequestFunction: ow.function,- handleRequestTimeoutSecs: ow.optional.number,- handleFailedRequestFunction: ow.optional.function,- maxRequestRetries: ow.optional.number,- maxRequestsPerCrawl: ow.optional.number,- autoscaledPoolOptions: ow.optional.object,- sessionPoolOptions: ow.optional.object,- useSessionPool: ow.optional.boolean,-- // AutoscaledPool shorthands- minConcurrency: ow.optional.number,- maxConcurrency: ow.optional.number,-- // internal- log: ow.optional.object,- }));+ ow(options, 'BasicCrawlerOptions', ow.object.exactShape(BasicCrawler.optionsShape));const {requestList,requestQueue,handleRequestFunction,handleRequestTimeoutSecs = 60,- handleFailedRequestFunction = ({ request }) => {- const details = _.pick(request, 'id', 'url', 'method', 'uniqueKey');- log.error('Request failed and reached maximum retries', details);- },+ handleFailedRequestFunction,maxRequestRetries = 3,maxRequestsPerCrawl,autoscaledPoolOptions = {},@@ -218,7 +220,7 @@ class BasicCrawler {maxConcurrency,// internal- log = defaultLog.child({ prefix: 'BasicCrawler' }),+ log = defaultLog.child({ prefix: this.constructor.name }),} = options;if (!requestList && !requestQueue) {@@ -229,7 +231,8 @@ class BasicCrawler {this.log = log;this.requestList = requestList;this.requestQueue = requestQueue;- this.handleRequestFunction = handleRequestFunction;+ this.userProvidedHandler = handleRequestFunction;+ this.failedContextHandler = handleFailedRequestFunction;this.handleRequestTimeoutMillis = handleRequestTimeoutSecs * 1000;this.handleFailedRequestFunction = handleFailedRequestFunction;this.maxRequestRetries = maxRequestRetries;@@ -302,17 +305,7 @@ class BasicCrawler {async run() {if (this.isRunningPromise) return this.isRunningPromise;- // Initialize AutoscaledPool before awaiting _loadHandledRequestCount(),- // so that the caller can get a reference to it before awaiting the promise returned from run()- // (otherwise there would be no way)- this.autoscaledPool = new AutoscaledPool(this.autoscaledPoolOptions);-- if (this.useSessionPool) {- this.sessionPool = await openSessionPool(this.sessionPoolOptions);- }-- await this._loadHandledRequestCount();-+ await this._init();this.isRunningPromise = this.autoscaledPool.run();await this.stats.startCapturing();@@ -335,6 +328,32 @@ class BasicCrawler {}}+ /**+ * @return {Promise<void>}+ * @private+ */+ async _init() {+ // Initialize AutoscaledPool before awaiting _loadHandledRequestCount(),+ // so that the caller can get a reference to it before awaiting the promise returned from run()+ // (otherwise there would be no way)+ this.autoscaledPool = new AutoscaledPool(this.autoscaledPoolOptions);++ if (this.useSessionPool) {+ this.sessionPool = await openSessionPool(this.sessionPoolOptions);+ }++ await this._loadHandledRequestCount();+ }++ /**+ * @param {object} crawlingContext+ * @return {Promise<void>}+ * @private+ */+ async _handleRequestFunction(crawlingContext) { // eslint-disable-line no-unused-vars+ await this.userProvidedHandler(crawlingContext);+ }+async _pauseOnMigration() {if (this.autoscaledPool) {// if run wasn't called, this is going to crash@@ -431,7 +450,7 @@ class BasicCrawler {try {await addTimeoutToPromise(- this.handleRequestFunction(crawlingContext),+ this._handleRequestFunction(crawlingContext),this.handleRequestTimeoutMillis,`handleRequestFunction timed out after ${this.handleRequestTimeoutMillis / 1000} seconds.`,);@@ -490,33 +509,54 @@ class BasicCrawler {* Handles errors thrown by user provided handleRequestFunction()* @param {Error} error* @param {object} crawlingContext+ * @param {Request} crawlingContext.request* @param {(RequestList|RequestQueue)} source- * @return {Promise<boolean|void|QueueOperationInfo>} willBeRetried+ * @return {Promise<void>}* @ignore*/async _requestFunctionErrorHandler(error, crawlingContext, source) {const { request } = crawlingContext;request.pushErrorMessage(error);- // Reclaim and retry request if flagged as retriable and retryCount is not exceeded.- if (!request.noRetry && request.retryCount < this.maxRequestRetries) {+ const shouldRetryRequest = !request.noRetry && request.retryCount < this.maxRequestRetries;+ if (shouldRetryRequest) {request.retryCount++;this.log.exception(error,'handleRequestFunction failed, reclaiming failed request back to the list or queue',_.pick(request, 'url', 'retryCount', 'id'),);- return source.reclaimRequest(request);+ await source.reclaimRequest(request);+ } else {+ // If we get here, the request is either not retryable+ // or failed more than retryCount times and will not be retried anymore.+ // Mark the request as failed and do not retry.+ this.handledRequestsCount++;+ await source.markRequestHandled(request);+ this.stats.failJob(request.id || request.url);+ crawlingContext.error = error;+ await this._handleFailedRequestFunction(crawlingContext); // This function prints an error message.}+ }- // If we get here, the request is either not retriable- // or failed more than retryCount times and will not be retried anymore.- // Mark the request as failed and do not retry.- this.handledRequestsCount++;- await source.markRequestHandled(request);- this.stats.failJob(request.id || request.url);- crawlingContext.error = error;- return this.handleFailedRequestFunction(crawlingContext); // This function prints an error message.+ /**+ * @param {object} crawlingContext+ * @param {Error} crawlingContext.error+ * @param {Request} crawlingContext.request+ * @return {Promise<void>}+ * @private+ */+ async _handleFailedRequestFunction(crawlingContext) {+ if (this.failedContextHandler) {+ await this.failedContextHandler(crawlingContext);+ } else {+ const { id, url, method, uniqueKey } = crawlingContext.request;+ this.log.exception(+ crawlingContext.error,+ 'Request failed and reached maximum retries',+ { id, url, method, uniqueKey },+ );+ }}/**diff --git a/src/crawlers/cheerio_crawler.js b/src/crawlers/cheerio_crawler.jsindex ae894681f..c9206d753 100644--- a/src/crawlers/cheerio_crawler.js+++ b/src/crawlers/cheerio_crawler.js@@ -6,14 +6,11 @@ import * as htmlparser from 'htmlparser2';import { WritableStream } from 'htmlparser2/lib/WritableStream';import * as iconv from 'iconv-lite';import ow from 'ow';-import * as _ from 'underscore';import * as util from 'util';-import { BASIC_CRAWLER_TIMEOUT_MULTIPLIER } from '../constants';import { TimeoutError } from '../errors';import { addTimeoutToPromise, parseContentTypeFromResponse } from '../utils';import * as utilsRequest from '../utils_request'; // eslint-disable-line import/no-duplicatesimport BasicCrawler from './basic_crawler'; // eslint-disable-line import/no-duplicates-import defaultLog from '../utils_log';import CrawlerExtension from './crawler_extension';// TYPE IMPORTS@@ -35,7 +32,7 @@ import { validators } from '../validators';*/const HTML_AND_XML_MIME_TYPES = ['text/html', 'text/xml', 'application/xhtml+xml', 'application/xml'];const APPLICATION_JSON_MIME_TYPE = 'application/json';-const DEFAULT_AUTOSCALED_POOL_OPTIONS = {+const CHEERIO_OPTIMIZED_AUTOSCALED_POOL_OPTIONS = {snapshotterOptions: {eventLoopSnapshotIntervalSecs: 2,maxBlockedMillis: 100,@@ -329,42 +326,34 @@ const DEFAULT_AUTOSCALED_POOL_OPTIONS = {* to pause the crawler by calling {@link AutoscaledPool#pause}* or to abort it by calling {@link AutoscaledPool#abort}.*/-class CheerioCrawler {+class CheerioCrawler extends BasicCrawler {+ static optionsShape = {+ ...BasicCrawler.optionsShape,+ // TODO temporary until the API is unified in V2+ handleRequestFunction: ow.undefined,++ handlePageFunction: ow.function,+ requestTimeoutSecs: ow.optional.number,+ handlePageTimeoutSecs: ow.optional.number,+ ignoreSslErrors: ow.optional.boolean,+ additionalMimeTypes: ow.optional.array.ofType(ow.string),+ suggestResponseEncoding: ow.optional.string,+ forceResponseEncoding: ow.optional.string,+ proxyConfiguration: ow.optional.object.validate(validators.proxyConfiguration),+ prepareRequestFunction: ow.optional.function,+ postResponseFunction: ow.optional.function,+ persistCookiesPerSession: ow.optional.boolean,++ // Deprecated+ requestOptions: ow.optional.object,+ }+/*** @param {CheerioCrawlerOptions} options* All `CheerioCrawler` parameters are passed via an options object.*/constructor(options) {- ow(options, ow.object.exactShape({- handlePageFunction: ow.function,- requestTimeoutSecs: ow.optional.number,- handlePageTimeoutSecs: ow.optional.number,- ignoreSslErrors: ow.optional.boolean,- additionalMimeTypes: ow.optional.array.ofType(ow.string),- suggestResponseEncoding: ow.optional.string,- forceResponseEncoding: ow.optional.string,- proxyConfiguration: ow.optional.object.validate(validators.proxyConfiguration),-- // Autoscaled pool shorthands- minConcurrency: ow.optional.number,- maxConcurrency: ow.optional.number,-- // Basic crawler options- requestList: ow.optional.object.validate(validators.requestList),- requestQueue: ow.optional.object.validate(validators.requestQueue),- maxRequestRetries: ow.optional.number,- maxRequestsPerCrawl: ow.optional.number,- handleFailedRequestFunction: ow.optional.function,- autoscaledPoolOptions: ow.optional.object,- prepareRequestFunction: ow.optional.function,- postResponseFunction: ow.optional.function,- useSessionPool: ow.optional.boolean,- sessionPoolOptions: ow.optional.object,- persistCookiesPerSession: ow.optional.boolean,-- // Deprecated- requestOptions: ow.optional.object,- }));+ ow(options, 'CheerioCrawlerOptions', ow.object.exactShape(CheerioCrawler.optionsShape));const {handlePageFunction,@@ -375,31 +364,29 @@ class CheerioCrawler {suggestResponseEncoding,forceResponseEncoding,proxyConfiguration,-- // Autoscaled pool shorthands- minConcurrency,- maxConcurrency,-- // Basic crawler options- requestList,- requestQueue,- maxRequestRetries,- maxRequestsPerCrawl,- handleFailedRequestFunction = this._defaultHandleFailedRequestFunction.bind(this),- autoscaledPoolOptions = DEFAULT_AUTOSCALED_POOL_OPTIONS,prepareRequestFunction,postResponseFunction,- useSessionPool = false,- sessionPoolOptions = {},persistCookiesPerSession = false,// DeprecatedrequestOptions,++ // BasicCrawler+ autoscaledPoolOptions = CHEERIO_OPTIMIZED_AUTOSCALED_POOL_OPTIONS,+ ...basicCrawlerOptions} = options;- this.log = defaultLog.child({ prefix: 'CheerioCrawler' });+ super({+ ...basicCrawlerOptions,+ // TODO temporary until the API is unified in V2+ handleRequestFunction: handlePageFunction,+ autoscaledPoolOptions,+ // We need to add some time for internal functions to finish,+ // but not too much so that we would stall the crawler.+ handleRequestTimeoutSecs: (requestTimeoutSecs + handlePageTimeoutSecs * 2) + 5,+ });- if (persistCookiesPerSession && !useSessionPool) {+ if (persistCookiesPerSession && !this.useSessionPool) {throw new Error('Cannot use "options.persistCookiesPerSession" without "options.useSessionPool"');}@@ -416,7 +403,6 @@ class CheerioCrawler {this.log.warning('Both forceResponseEncoding and suggestResponseEncoding options are set. Using forceResponseEncoding.');}- this.handlePageFunction = handlePageFunction;this.handlePageTimeoutMillis = handlePageTimeoutSecs * 1000;this.requestTimeoutMillis = requestTimeoutSecs * 1000;this.ignoreSslErrors = ignoreSslErrors;@@ -426,48 +412,6 @@ class CheerioCrawler {this.postResponseFunction = postResponseFunction;this.proxyConfiguration = proxyConfiguration;this.persistCookiesPerSession = persistCookiesPerSession;- this.useSessionPool = useSessionPool;- this.sessionPoolOptions = sessionPoolOptions;-- /** @ignore */- this.basicCrawler = new BasicCrawler({- // Basic crawler options.- requestList,- requestQueue,- maxRequestRetries,- maxRequestsPerCrawl,- handleRequestFunction: (...args) => this._handleRequestFunction(...args),- handleRequestTimeoutSecs: handlePageTimeoutSecs * BASIC_CRAWLER_TIMEOUT_MULTIPLIER,- handleFailedRequestFunction,-- // Autoscaled pool options.- minConcurrency,- maxConcurrency,- autoscaledPoolOptions,-- // Session pool options- sessionPoolOptions: this.sessionPoolOptions,- useSessionPool: this.useSessionPool,-- // log- log: this.log,- });-- this.isRunningPromise = null;- }-- /**- * Runs the crawler. Returns promise that gets resolved once all the requests got processed.- *- * @return {Promise<void>}- */- async run() {- if (this.isRunningPromise) return this.isRunningPromise;-- this.isRunningPromise = this.basicCrawler.run();- this.autoscaledPool = this.basicCrawler.autoscaledPool;-- await this.isRunningPromise;}/**@@ -479,6 +423,9 @@ class CheerioCrawler {ow(extension, ow.object.instanceOf(CrawlerExtension));const extensionOptions = extension.getCrawlerOptions();+ // TODO temporary until the API is unified in V2+ extensionOptions.userProvidedHandler = extensionOptions.handlePageFunction;+ delete extensionOptions.handlePageFunction;for (const [key, value] of Object.entries(extensionOptions)) {const isConfigurable = this.hasOwnProperty(key); // eslint-disable-line@@ -569,7 +516,7 @@ class CheerioCrawler {});return addTimeoutToPromise(- this.handlePageFunction(crawlingContext),+ this.userProvidedHandler(crawlingContext),this.handlePageTimeoutMillis,`handlePageFunction timed out after ${this.handlePageTimeoutMillis / 1000} seconds.`,);@@ -773,18 +720,6 @@ class CheerioCrawler {if (session) session.markBad();throw new Error(`request timed out after ${this.handlePageTimeoutMillis / 1000} seconds.`);}-- /**- * @param {Object} options- * @param {Error} options.error- * @param {Request} options.request- * @return {Promise<void>}- * @ignore- */- async _defaultHandleFailedRequestFunction({ error, request }) { // eslint-disable-line class-methods-use-this- const details = _.pick(request, 'id', 'url', 'method', 'uniqueKey');- this.log.exception(error, 'Request failed and reached maximum retries', details);- }}export default CheerioCrawler;diff --git a/src/crawlers/puppeteer_crawler.js b/src/crawlers/puppeteer_crawler.jsindex 6f563e0e9..2342878dd 100644--- a/src/crawlers/puppeteer_crawler.js+++ b/src/crawlers/puppeteer_crawler.js@@ -1,12 +1,8 @@import ow from 'ow';-import * as _ from 'underscore';import PuppeteerPool from '../puppeteer_pool'; // eslint-disable-line import/no-duplicates-import { BASIC_CRAWLER_TIMEOUT_MULTIPLIER } from '../constants';import { gotoExtended } from '../puppeteer_utils';-import { openSessionPool } from '../session_pool/session_pool'; // eslint-disable-line import/no-duplicatesimport { addTimeoutToPromise } from '../utils';import BasicCrawler from './basic_crawler'; // eslint-disable-line import/no-duplicates-import defaultLog from '../utils_log';// TYPE IMPORTS/* eslint-disable no-unused-vars,import/named,import/no-duplicates,import/order */@@ -214,78 +210,63 @@ import { validators } from '../validators';* or to abort it by calling {@link AutoscaledPool#abort}.**/-class PuppeteerCrawler {+class PuppeteerCrawler extends BasicCrawler {+ static optionsShape = {+ ...BasicCrawler.optionsShape,+ // TODO temporary until the API is unified in V2+ handleRequestFunction: ow.undefined,++ handlePageFunction: ow.function,+ gotoFunction: ow.optional.function,+ handlePageTimeoutSecs: ow.optional.number,+ gotoTimeoutSecs: ow.optional.number,+ proxyConfiguration: ow.optional.object.validate(validators.proxyConfiguration),+ persistCookiesPerSession: ow.optional.boolean,++ // PuppeteerPool options and shorthands+ puppeteerPoolOptions: ow.optional.object,+ launchPuppeteerFunction: ow.optional.function,+ launchPuppeteerOptions: ow.optional.object,+ }+/*** @param {PuppeteerCrawlerOptions} options* All `PuppeteerCrawler` parameters are passed via an options object.*/constructor(options) {- ow(options, ow.object.exactShape({- handlePageFunction: ow.function,- gotoFunction: ow.optional.function,- handlePageTimeoutSecs: ow.optional.number,- gotoTimeoutSecs: ow.optional.number,-- // AutoscaledPool shorthands- maxConcurrency: ow.optional.number,- minConcurrency: ow.optional.number,-- // BasicCrawler options- requestList: ow.optional.object.validate(validators.requestList),- requestQueue: ow.optional.object.validate(validators.requestQueue),- maxRequestRetries: ow.optional.number,- maxRequestsPerCrawl: ow.optional.number,- handleFailedRequestFunction: ow.optional.function,- autoscaledPoolOptions: ow.optional.object,-- // PuppeteerPool options and shorthands- puppeteerPoolOptions: ow.optional.object,- launchPuppeteerFunction: ow.optional.function,- launchPuppeteerOptions: ow.optional.object,-- sessionPoolOptions: ow.optional.object,- persistCookiesPerSession: ow.optional.boolean,- useSessionPool: ow.optional.boolean,- proxyConfiguration: ow.optional.object.validate(validators.proxyConfiguration),- }));+ ow(options, ow.object.exactShape(PuppeteerCrawler.optionsShape));const {handlePageFunction,- gotoFunction = this._defaultGotoFunction,+ gotoFunction,handlePageTimeoutSecs = 60,gotoTimeoutSecs = 60,-- // AutoscaledPool shorthands- maxConcurrency,- minConcurrency,-- // BasicCrawler options- requestList,- requestQueue,- maxRequestRetries,- maxRequestsPerCrawl,- handleFailedRequestFunction = this._defaultHandleFailedRequestFunction.bind(this),- autoscaledPoolOptions,+ proxyConfiguration,+ persistCookiesPerSession = false,// PuppeteerPool options and shorthandspuppeteerPoolOptions,launchPuppeteerFunction,launchPuppeteerOptions,- sessionPoolOptions = {},- persistCookiesPerSession = false,- useSessionPool = false,- proxyConfiguration,+ ...basicCrawlerOptions} = options;+ super({+ ...basicCrawlerOptions,+ // TODO temporary until the API is unified in V2+ handleRequestFunction: handlePageFunction,+ // We need to add some time for internal functions to finish,+ // but not too much so that we would stall the crawler.+ handleRequestTimeoutSecs: (gotoTimeoutSecs + handlePageTimeoutSecs * 2) + 5,+ });+if (proxyConfiguration && (launchPuppeteerOptions && launchPuppeteerOptions.proxyUrl)) {throw new Error('It is not possible to combine "options.proxyConfiguration" together with '+ 'custom "proxyUrl" option from "options.launchPuppeteerOptions".');}- this.log = defaultLog.child({ prefix: 'PuppeteerCrawler' });-- if (persistCookiesPerSession && !useSessionPool) {+ if (persistCookiesPerSession && !this.useSessionPool) {throw new Error('Cannot use "options.persistCookiesPerSession" without "options.useSessionPool"');}@@ -294,7 +275,6 @@ class PuppeteerCrawler {+ 'The timeout value will not be used. With a custom gotoFunction, you need to set the timeout in the function itself.');}- this.handlePageFunction = handlePageFunction;this.gotoFunction = gotoFunction;this.handlePageTimeoutMillis = handlePageTimeoutSecs * 1000;@@ -304,36 +284,13 @@ class PuppeteerCrawler {...puppeteerPoolOptions,launchPuppeteerFunction,launchPuppeteerOptions,+ proxyConfiguration,+ log: this.log,};this.puppeteerPool = null; // Constructed when .run()- this.useSessionPool = useSessionPool;- this.sessionPoolOptions = {- ...sessionPoolOptions,- log: this.log,- };this.persistCookiesPerSession = persistCookiesPerSession;this.proxyConfiguration = proxyConfiguration;-- /** @ignore */- this.basicCrawler = new BasicCrawler({- // Basic crawler options.- requestList,- requestQueue,- maxRequestRetries,- maxRequestsPerCrawl,- handleRequestFunction: (...args) => this._handleRequestFunction(...args),- handleRequestTimeoutSecs: handlePageTimeoutSecs * BASIC_CRAWLER_TIMEOUT_MULTIPLIER,- handleFailedRequestFunction,-- // Autoscaled pool options.- maxConcurrency,- minConcurrency,- autoscaledPoolOptions,-- // log- log: this.log,- });}/**@@ -344,26 +301,16 @@ class PuppeteerCrawler {async run() {if (this.isRunningPromise) return this.isRunningPromise;- if (this.useSessionPool) {- this.sessionPool = await openSessionPool(this.sessionPoolOptions);- this.puppeteerPoolOptions.sessionPool = this.sessionPool;- }+ await this._init();- if (this.proxyConfiguration) {- this.puppeteerPoolOptions.proxyConfiguration = this.proxyConfiguration;+ if (this.sessionPool) {+ this.puppeteerPoolOptions.sessionPool = this.sessionPool;}-- this.puppeteerPoolOptions.log = this.log;this.puppeteerPool = new PuppeteerPool(this.puppeteerPoolOptions);- try {- this.isRunningPromise = this.basicCrawler.run();- this.autoscaledPool = this.basicCrawler.autoscaledPool;- await this.isRunningPromise;+ try {+ await super.run();} finally {- if (this.useSessionPool) {- await this.sessionPool.teardown();- }await this.puppeteerPool.destroy();}}@@ -401,7 +348,7 @@ class PuppeteerCrawler {try {let response;try {- response = await this.gotoFunction(crawlingContext);+ response = await this._navigationHandler(crawlingContext);} catch (err) {// It would be better to compare the instances,// but we don't have access to puppeteer.errors here.@@ -431,7 +378,7 @@ class PuppeteerCrawler {crawlingContext.puppeteerPool = this.puppeteerPool;await addTimeoutToPromise(- this.handlePageFunction(crawlingContext),+ this.userProvidedHandler(crawlingContext),this.handlePageTimeoutMillis,`handlePageFunction timed out after ${this.handlePageTimeoutMillis / 1000} seconds.`,);@@ -443,9 +390,9 @@ class PuppeteerCrawler {}/**- * @param {Object} options- * @param {PuppeteerPage} options.page- * @param {Request} options.request+ * @param {Object} crawlingContext+ * @param {PuppeteerPage} crawlingContext.page+ * @param {Request} crawlingContext.request* @property {AutoscaledPool} autoscaledPool* @property {PuppeteerPool} puppeteerPool* @property {Session} [session]@@ -453,20 +400,9 @@ class PuppeteerCrawler {* @return {Promise<PuppeteerResponse>}* @ignore*/- async _defaultGotoFunction({ page, request }) {- return gotoExtended(page, request, { timeout: this.gotoTimeoutMillis });- }-- /**- * @param {Object} options- * @param {Error} options.error- * @param {Request} options.request- * @return {Promise<void>}- * @ignore- */- async _defaultHandleFailedRequestFunction({ error, request }) { // eslint-disable-line class-methods-use-this- const details = _.pick(request, 'id', 'url', 'method', 'uniqueKey');- this.log.exception(error, 'Request failed and reached maximum retries', details);+ async _navigationHandler(crawlingContext) {+ if (this.gotoFunction) return this.gotoFunction(crawlingContext);+ return gotoExtended(crawlingContext.page, crawlingContext.request, { timeout: this.gotoTimeoutMillis });}/**diff --git a/test/crawlers/cheerio_crawler.test.js b/test/crawlers/cheerio_crawler.test.jsindex d46630cc6..5943015d0 100644--- a/test/crawlers/cheerio_crawler.test.js+++ b/test/crawlers/cheerio_crawler.test.js@@ -282,7 +282,7 @@ describe('CheerioCrawler', () => {});// Override low value to prevent seeing timeouts from BasicCrawler- cheerioCrawler.basicCrawler.handleRequestTimeoutMillis = 10000;+ cheerioCrawler.handleRequestTimeoutMillis = 10000;await cheerioCrawler.run();@@ -987,6 +987,7 @@ describe('CheerioCrawler', () => {expect(cheerioCrawler.useSessionPool).toEqual(true);expect(cheerioCrawler.prepareRequestFunction).toEqual(prepareRequestFunction);expect(cheerioCrawler.handlePageFunction).toBeUndefined();+ expect(cheerioCrawler.userProvidedHandler).toBeUndefined();});});});diff --git a/test/crawlers/puppeteer_crawler.test.js b/test/crawlers/puppeteer_crawler.test.jsindex e21026172..91db373d3 100644--- a/test/crawlers/puppeteer_crawler.test.js+++ b/test/crawlers/puppeteer_crawler.test.js@@ -47,7 +47,7 @@ describe('PuppeteerCrawler', () => {const failed = [];const requestList = new Apify.RequestList({ sources });const handlePageFunction = async ({ page, request, response }) => {- await page.waitFor('title');+ await page.waitForSelector('title');expect(await response.status()).toBe(200);request.userData.title = await page.title();diff --git a/test/stealth/stealth.test.js b/test/stealth/stealth.test.jsindex 0e7368d81..41b89c30a 100644--- a/test/stealth/stealth.test.js+++ b/test/stealth/stealth.test.js@@ -191,32 +191,24 @@ describe('Stealth - testing headless chrome hiding tricks', () => {expect(message).toBeUndefined();});- describe('puppeteer crawler test', async () => {- let requestList;- let puppeteerCrawler;-- beforeEach(async () => {- requestList = await Apify.openRequestList('test', [testUrl]);- puppeteerCrawler = new Apify.PuppeteerCrawler({- requestList,- launchPuppeteerOptions: {- stealth: true,- useChrome: true,- headless: true,- },- handlePageFunction: () => {},- });- });- test('should work in crawler', async () => {- puppeteerCrawler.handlePageFunction = async ({ page: activePage }) => {+ test('should work in puppeteer crawler', async () => {+ const requestList = await Apify.openRequestList(null, [testUrl]);+ const puppeteerCrawler = new Apify.PuppeteerCrawler({+ requestList,+ launchPuppeteerOptions: {+ stealth: true,+ useChrome: true,+ headless: true,+ },+ handlePageFunction: async ({ page: activePage }) => {const fingerprint = await getFingerPrint(activePage);values.push(fingerprint);- };- const values = [];- await puppeteerCrawler.run();- const fingerprint = values[0];- expect(fingerprint.webDriver).toBe(false); // eslint-disable-line- expect(fingerprint.webDriverValue).toBeUndefined(); // eslint-disable-line+ },});+ const values = [];+ await puppeteerCrawler.run();+ const fingerprint = values[0];+ expect(fingerprint.webDriver).toBe(false);+ expect(fingerprint.webDriverValue).toBeUndefined();});});