Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Add gotoTimeoutSecs and remove deprecated fields

Ondra Urban8c246a667cea
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9b1cae4cf..2b37884ee 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,7 @@
+NEXT
+====================
+- Added `options.gotoTimeoutSecs` to `PuppeteerCrawler` to enable easier setting of navigation timeouts.
+
0.15.3 / 2019-07-29
====================
- `Apify.launchPuppeteer()` now accepts `proxyUrl` with the `https`, `socks4`
diff --git a/docs/api/PuppeteerCrawler.md b/docs/api/PuppeteerCrawler.md
index 6683e5a95..d37341214 100644
--- a/docs/api/PuppeteerCrawler.md
+++ b/docs/api/PuppeteerCrawler.md
@@ -138,7 +138,13 @@ await crawler.run();
<p> Note that a single page object is only used to process a single request and it is closed afterwards.</p>
<p> By default, the function invokes <a href="puppeteer#puppeteer.gotoExtended"><code>Apify.utils.puppeteer.gotoExtended()</code></a> with a timeout of 60 seconds.
For details, see source code on
- <a href="https://github.com/apifytech/apify-js/blob/master/src/crawlers/puppeteer_crawler.js#L9" target="_blank">GitHub</a>.</p>
+ <a href="https://github.com/apifytech/apify-js/blob/master/src/crawlers/puppeteer_crawler.js#L292" target="_blank">GitHub</a>.</p>
+</td></tr><tr>
+<td><code>[options.gotoTimeoutSecs]</code></td><td><code>Number</code></td><td><code>60</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Timeout in which page navigation needs to finish, in seconds. When <code>options.gotoFunction()</code> is used and thus the default
+ function is overridden, this timeout will not be used and needs to be configured in the new <code>gotoFunction()</code>.</p>
</td></tr><tr>
<td><code>[options.handleFailedRequestFunction]</code></td><td><code>function</code></td><td></td>
</tr>
@@ -151,7 +157,7 @@ await crawler.run();
}</code></pre><p> Where the <a href="request"><code>Request</code></a> instance corresponds to the failed request, and the <code>Error</code> instance
represents the last error thrown during processing of the request.</p>
<p> See
- <a href="https://github.com/apifytech/apify-js/blob/master/src/crawlers/puppeteer_crawler.js#L11" target="_blank">source code</a>
+ <a href="https://github.com/apifytech/apify-js/blob/master/src/crawlers/puppeteer_crawler.js#L301" target="_blank">source code</a>
for the default implementation of this function.</p>
</td></tr><tr>
<td><code>[options.maxRequestRetries]</code></td><td><code>Number</code></td><td><code>3</code></td>
diff --git a/src/crawlers/puppeteer_crawler.js b/src/crawlers/puppeteer_crawler.js
index 0561d77f9..2624e2ea9 100644
--- a/src/crawlers/puppeteer_crawler.js
+++ b/src/crawlers/puppeteer_crawler.js
@@ -7,15 +7,6 @@ import { addTimeoutToPromise } from '../utils';
import { BASIC_CRAWLER_TIMEOUT_MULTIPLIER } from '../constants';
import { gotoExtended } from '../puppeteer_utils';
-const DEFAULT_OPTIONS = {
- gotoFunction: async ({ request, page }) => gotoExtended(page, request, { timeout: 60000 }),
- handlePageTimeoutSecs: 60,
- handleFailedRequestFunction: ({ request }) => {
- const details = _.pick(request, 'id', 'url', 'method', 'uniqueKey');
- log.error('PuppeteerCrawler: Request failed and reached maximum retries', details);
- },
-};
-
/**
* Provides a simple framework for parallel crawling of web pages
* using headless Chrome with <a href="https://github.com/GoogleChrome/puppeteer" target="_blank">Puppeteer</a>.
@@ -131,7 +122,10 @@ const DEFAULT_OPTIONS = {
*
* By default, the function invokes [`Apify.utils.puppeteer.gotoExtended()`](puppeteer#puppeteer.gotoExtended) with a timeout of 60 seconds.
* For details, see source code on
- * <a href="https://github.com/apifytech/apify-js/blob/master/src/crawlers/puppeteer_crawler.js#L9" target="_blank">GitHub</a>.
+ * <a href="https://github.com/apifytech/apify-js/blob/master/src/crawlers/puppeteer_crawler.js#L292" target="_blank">GitHub</a>.
+ * @param {Number} [options.gotoTimeoutSecs=60]
+ * Timeout in which page navigation needs to finish, in seconds. When `options.gotoFunction()` is used and thus the default
+ * function is overridden, this timeout will not be used and needs to be configured in the new `gotoFunction()`.
* @param {Function} [options.handleFailedRequestFunction]
* A function to handle requests that failed more than `option.maxRequestRetries` times.
*
@@ -146,7 +140,7 @@ const DEFAULT_OPTIONS = {
* represents the last error thrown during processing of the request.
*
* See
- * <a href="https://github.com/apifytech/apify-js/blob/master/src/crawlers/puppeteer_crawler.js#L11" target="_blank">source code</a>
+ * <a href="https://github.com/apifytech/apify-js/blob/master/src/crawlers/puppeteer_crawler.js#L301" target="_blank">source code</a>
* for the default implementation of this function.
* @param {Number} [options.maxRequestRetries=3]
* Indicates how many times the request is retried if either `handlePageFunction()` or `gotoFunction()` fails.
@@ -182,8 +176,9 @@ class PuppeteerCrawler {
constructor(options) {
const {
handlePageFunction,
- gotoFunction,
- handlePageTimeoutSecs,
+ gotoFunction = this._defaultGotoFunction,
+ handlePageTimeoutSecs = 60,
+ gotoTimeoutSecs = 60,
// AutoscaledPool shorthands
maxConcurrency,
@@ -194,52 +189,38 @@ class PuppeteerCrawler {
requestQueue,
maxRequestRetries,
maxRequestsPerCrawl,
- handleFailedRequestFunction,
+ handleFailedRequestFunction = this._defaultHandleFailedRequestFunction,
autoscaledPoolOptions,
// PuppeteerPool options and shorthands
puppeteerPoolOptions,
launchPuppeteerFunction,
launchPuppeteerOptions,
-
- // TODO Deprecated PuppeteerPool options
- maxOpenPagesPerInstance,
- retireInstanceAfterRequestCount,
- instanceKillerIntervalMillis,
- killInstanceAfterMillis,
- proxyUrls,
-
- } = _.defaults({}, options, DEFAULT_OPTIONS);
+ } = options;
checkParamOrThrow(handlePageFunction, 'options.handlePageFunction', 'Function');
checkParamOrThrow(handlePageTimeoutSecs, 'options.handlePageTimeoutSecs', 'Number');
- checkParamOrThrow(handleFailedRequestFunction, 'options.handleFailedRequestFunction', 'Maybe Function');
+ checkParamOrThrow(handleFailedRequestFunction, 'options.handleFailedRequestFunction', 'Function');
checkParamOrThrow(gotoFunction, 'options.gotoFunction', 'Function');
+ checkParamOrThrow(gotoTimeoutSecs, 'options.gotoTimeoutSecs', 'Number');
checkParamOrThrow(puppeteerPoolOptions, 'options.puppeteerPoolOptions', 'Maybe Object');
+ if (options.gotoTimeoutSecs && options.gotoFunction) {
+ log.warning('PuppeteerCrawler: You are using gotoTimeoutSecs with a custom gotoFunction. '
+ + 'The timeout value will not be used. With a custom gotoFunction, you need to set the timeout in the function itself.');
+ }
+
this.handlePageFunction = handlePageFunction;
this.gotoFunction = gotoFunction;
this.handlePageTimeoutMillis = handlePageTimeoutSecs * 1000;
+ this.gotoTimeoutMillis = gotoTimeoutSecs * 1000;
- // TODO Deprecated in 3/2019
- const deprecatedPuppeteerPoolOptions = {
- maxOpenPagesPerInstance,
- retireInstanceAfterRequestCount,
- instanceKillerIntervalMillis,
- killInstanceAfterMillis,
- proxyUrls,
+ this.puppeteerPoolOptions = {
+ ...puppeteerPoolOptions,
+ launchPuppeteerFunction,
+ launchPuppeteerOptions,
};
- Object.entries(deprecatedPuppeteerPoolOptions).forEach(([key, value]) => {
- if (value) log.deprecated(`PuppeteerCrawler: "options.${key}" is deprecated, use "options.puppeteerPoolOptions" in PuppeteerCrawler() constructor instead.`); // eslint-disable-line max-len
- });
- // puppeteerPoolOptions can be null or undefined or Object, so we merge it this way, because null is not replaced by defaults above.
- this.puppeteerPoolOptions = _.defaults(
- {},
- { launchPuppeteerFunction, launchPuppeteerOptions },
- deprecatedPuppeteerPoolOptions,
- puppeteerPoolOptions,
- );
this.puppeteerPool = null; // Constructed when .run()
@@ -298,6 +279,26 @@ class PuppeteerCrawler {
await this.puppeteerPool.recyclePage(page);
}
}
+
+ /**
+ * @param {Page} page
+ * @param {Request} request
+ * @return {Promise<Response>}
+ * @ignore
+ */
+ async _defaultGotoFunction({ page, request }) {
+ return gotoExtended(page, request, { timeout: this.gotoTimeoutMillis });
+ }
+
+ /**
+ * @param {Request} request
+ * @return {Promise}
+ * @ignore
+ */
+ async _defaultHandleFailedRequestFunction({ request }) { // eslint-disable-line class-methods-use-this
+ const details = _.pick(request, 'id', 'url', 'method', 'uniqueKey');
+ log.error('PuppeteerCrawler: Request failed and reached maximum retries', details);
+ }
}