Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Implement PR comments

Ondra Urban07755c949f7e
diff --git a/src/cheerio_crawler.js b/src/cheerio_crawler.js
index 55cc5fd88..65e242eec 100644
--- a/src/cheerio_crawler.js
+++ b/src/cheerio_crawler.js
@@ -9,6 +9,7 @@ import { checkParamOrThrow } from 'apify-client/build/utils';
import BasicCrawler from './basic_crawler';
import { addTimeoutToPromise } from './utils';
import { getApifyProxyUrl } from './actor';
+import { BASIC_CRAWLER_TIMEOUT_MULTIPLIER } from './constants';
const DEFAULT_OPTIONS = {
requestTimeoutSecs: 30,
@@ -258,7 +259,7 @@ class CheerioCrawler {
maxRequestRetries,
maxRequestsPerCrawl,
handleRequestFunction: (...args) => this._handleRequestFunction(...args),
- handleRequestTimeoutSecs: handlePageTimeoutSecs * 2,
+ handleRequestTimeoutSecs: handlePageTimeoutSecs * BASIC_CRAWLER_TIMEOUT_MULTIPLIER,
handleFailedRequestFunction,
// Autoscaled pool options.
@@ -404,6 +405,9 @@ class CheerioCrawler {
strictSSL: !this.ignoreSslErrors,
proxy: this._getProxyUrl(),
};
+
+ if (/PATCH|POST|PUT/.test(request.method)) mandatoryRequestOptions.body = request.payload;
+
return Object.assign({}, this.requestOptions, mandatoryRequestOptions);
}
diff --git a/src/constants.js b/src/constants.js
index b51f9a94b..f2567975d 100644
--- a/src/constants.js
+++ b/src/constants.js
@@ -113,3 +113,11 @@ export const USER_AGENT_LIST = [
* @type {string}
*/
export const APIFY_API_BASE_URL = 'https://api.apify.com/v2';
+
+/**
+ * Multiplier used in CheerioCrawler and PuppeteerCrawler to set a reasonable
+ * handleRequestTimeoutSecs in BasicCrawler that would not impare functionality.
+ *
+ * @type {number}
+ */
+export const BASIC_CRAWLER_TIMEOUT_MULTIPLIER = 10;
diff --git a/src/puppeteer_crawler.js b/src/puppeteer_crawler.js
index 7855ef4e7..4271ebd52 100644
--- a/src/puppeteer_crawler.js
+++ b/src/puppeteer_crawler.js
@@ -4,6 +4,7 @@ import _ from 'underscore';
import BasicCrawler from './basic_crawler';
import PuppeteerPool from './puppeteer_pool';
import { addTimeoutToPromise } from './utils';
+import { BASIC_CRAWLER_TIMEOUT_MULTIPLIER } from './constants';
const DEFAULT_OPTIONS = {
gotoFunction: async ({ request, page }) => page.goto(request.url, { timeout: 60000 }),
@@ -192,6 +193,14 @@ class PuppeteerCrawler {
puppeteerPoolOptions,
launchPuppeteerFunction,
launchPuppeteerOptions,
+
+ // TODO Deprecated PuppeteerPool options
+ maxOpenPagesPerInstance,
+ retireInstanceAfterRequestCount,
+ instanceKillerIntervalMillis,
+ killInstanceAfterMillis,
+ proxyUrls,
+
} = _.defaults({}, options, DEFAULT_OPTIONS);
checkParamOrThrow(handlePageFunction, 'options.handlePageFunction', 'Function');
@@ -205,8 +214,24 @@ class PuppeteerCrawler {
this.handlePageTimeoutMillis = handlePageTimeoutSecs * 1000;
+ // TODO Deprecated in 3/2019
+ const deprecatedPuppeteerPoolOptions = {
+ maxOpenPagesPerInstance,
+ retireInstanceAfterRequestCount,
+ instanceKillerIntervalMillis,
+ killInstanceAfterMillis,
+ proxyUrls,
+ };
+ Object.entries(deprecatedPuppeteerPoolOptions).forEach(([key, value]) => {
+ if (value) log.deprecated(`PuppeteerCrawler: options.${key} is deprecated. Use options.puppeteerPoolOptions instead.`);
+ });
// puppeteerPoolOptions can be null or undefined or Object, so we merge it this way, because null is not replaced by defaults above.
- this.puppeteerPoolOptions = Object.assign({}, puppeteerPoolOptions, { launchPuppeteerFunction, launchPuppeteerOptions });
+ this.puppeteerPoolOptions = Object.assign(
+ {},
+ puppeteerPoolOptions,
+ { launchPuppeteerFunction, launchPuppeteerOptions },
+ deprecatedPuppeteerPoolOptions,
+ );
this.puppeteerPool = null; // Constructed when .run()
@@ -217,7 +242,7 @@ class PuppeteerCrawler {
maxRequestRetries,
maxRequestsPerCrawl,
handleRequestFunction: (...args) => this._handleRequestFunction(...args),
- handleRequestTimeoutSecs: handlePageTimeoutSecs * 2,
+ handleRequestTimeoutSecs: handlePageTimeoutSecs * BASIC_CRAWLER_TIMEOUT_MULTIPLIER,
handleFailedRequestFunction,
// Autoscaled pool options.
diff --git a/src/puppeteer_pool.js b/src/puppeteer_pool.js
index 1a34552fe..f32f411b0 100644
--- a/src/puppeteer_pool.js
+++ b/src/puppeteer_pool.js
@@ -13,7 +13,6 @@ import { addTimeoutToPromise } from './utils';
/* global process */
const PROCESS_KILL_TIMEOUT_MILLIS = 5000;
-const OPEN_NEW_TAB_RETRIES = 2;
const PAGE_CLOSE_KILL_TIMEOUT_MILLIS = 1000;
const DEFAULT_OPTIONS = {
@@ -257,17 +256,18 @@ class PuppeteerPool {
const id = this.browserCounter++;
log.debug('PuppeteerPool: Launching new browser', { id });
+ const errorMessageChunk = 'launchPuppeteerFunction timed out.';
const launchPromise = this.launchPuppeteerFunction();
const browserPromise = addTimeoutToPromise(
launchPromise,
this.puppeteerOperationTimeoutMillis,
- 'PuppeteerPool: launchPuppeteerFunction timed out.',
+ `PuppeteerPool: ${errorMessageChunk}`,
);
// If the browserPromise times out, the browser may still be started
// later and will not be managed by pool, so we need to get rid of it.
browserPromise.catch(async (err) => {
- if (err.stack.includes('launchPuppeteerFunction timed out')) {
+ if (err.stack.includes(errorMessageChunk)) {
try {
const browser = await launchPromise;
browser.disconnect();
@@ -479,7 +479,7 @@ class PuppeteerPool {
}
}
// If there are no live pages to be reused, we spawn a new tab.
- return this._openNewTab(OPEN_NEW_TAB_RETRIES);
+ return this._openNewTab();
}
/**
@@ -487,11 +487,10 @@ class PuppeteerPool {
* that resolves to an instance of a Puppeteer
* <a href="https://pptr.dev/#?product=Puppeteer&show=api-class-page" target="_blank"><code>Page</code></a>.
*
- * @param {number} retryCount
* @return {Promise<Page>}
* @ignore
*/
- async _openNewTab(retryCount) {
+ async _openNewTab() {
let instance = Object
.values(this.activeInstances)
.find(inst => inst.activePages < this.maxOpenPagesPerInstance);
@@ -512,13 +511,6 @@ class PuppeteerPool {
return this._decoratePage(page);
} catch (err) {
this._retireInstance(instance);
- if (retryCount) {
- return new Promise((resolve) => {
- // We can do this because _retireInstance is synchronous
- // so the page will open in a different browser.
- resolve(this._openNewTab(--retryCount));
- });
- }
const betterError = new Error(`PuppeteerPool: browser.newPage() failed: ${instance.id}.`);
betterError.stack = err.stack;
throw betterError;
diff --git a/test/utils.js b/test/utils.js
index 8cdc2c239..21653e457 100644
--- a/test/utils.js
+++ b/test/utils.js
@@ -842,3 +842,39 @@ describe('utils.snakeCaseToCamelCase()', () => {
});
});
});
+
+describe('utils.addTimeoutToPromise()', () => {
+ it('should timeout', async () => {
+ const clock = sinon.useFakeTimers();
+ try {
+ const p = utils.addTimeoutToPromise(
+ new Promise(r => setTimeout(r, 500)),
+ 100,
+ 'Timed out.',
+ );
+ clock.tick(101);
+ await p;
+ throw new Error('Wrong error.');
+ } catch (err) {
+ expect(err.message).to.be.eql('Timed out.');
+ } finally {
+ clock.restore();
+ }
+ });
+ it('should not timeout too soon', async () => {
+ const clock = sinon.useFakeTimers();
+ try {
+ const p = utils.addTimeoutToPromise(
+ new Promise(r => setTimeout(() => r('Done'), 100)),
+ 500,
+ 'Timed out.',
+ );
+ clock.tick(101);
+ expect(await p).to.be.eql('Done');
+ } catch (err) {
+ throw new Error('This should not fail.');
+ } finally {
+ clock.restore();
+ }
+ });
+});