Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Minor code improvements in puppetter_crawler.js and puppeteer_pool.js

Jan Curn7246e8c1adc9
diff --git a/src/puppeteer_crawler.js b/src/puppeteer_crawler.js
index 79ff83d3e..2d2d478dd 100644
--- a/src/puppeteer_crawler.js
+++ b/src/puppeteer_crawler.js
@@ -216,7 +216,7 @@ export default class PuppeteerCrawler {
_handleRequestFunction({ request }) {
let page;
- const handlePagePromise = this.puppeteerPool
+ return this.puppeteerPool
.newPage()
.then((newPage) => { page = newPage; })
.then(() => this.gotoFunction({ page, request, puppeteerPool: this.puppeteerPool }))
@@ -231,13 +231,13 @@ export default class PuppeteerCrawler {
if (!isPromise(promise)) throw new Error('User provided handlePageFunction must return a Promise.');
return promise;
- });
-
- return handlePagePromise
+ })
.timeout(this.pageOpsTimeoutMillis, 'PuppeteerCrawler: handlePageFunction timed out.')
.finally(() => {
return Promise
- .try(() => page.close())
+ .try(() => {
+ if (page) return page.close();
+ })
.timeout(PAGE_CLOSE_TIMEOUT_MILLIS, 'Operation timed out.')
.catch(err => log.debug('PuppeteerCrawler: Page.close() failed.', { reason: err ? err.message : err }));
});
diff --git a/src/puppeteer_pool.js b/src/puppeteer_pool.js
index 97650cbdd..89b3cf73c 100644
--- a/src/puppeteer_pool.js
+++ b/src/puppeteer_pool.js
@@ -4,6 +4,8 @@ import Promise from 'bluebird';
import { checkParamOrThrow } from 'apify-client/build/utils';
import { launchPuppeteer } from './puppeteer';
+/* global process */
+
const PROCESS_KILL_TIMEOUT_MILLIS = 5000;
const DEFAULT_OPTIONS = {
@@ -144,7 +146,7 @@ export default class PuppeteerPool {
if (!instance.killed) log.error('PuppeteerPool: Puppeteer sent "disconnect" event. Crashed???', { id });
this._retireInstance(instance);
});
- // This one is done manually in Puppeteerpool.newPage() to happen immediately.
+ // This one is done manually in Puppeteerpool.newPage() so that it happens immediately.
// browser.on('targetcreated', () => instance.activePages++);
browser.on('targetdestroyed', () => {
instance.activePages--;
@@ -279,7 +281,7 @@ export default class PuppeteerPool {
return instance.browserPromise
.then(browser => browser.newPage())
.then((page) => {
- page.on('error', (error) => {
+ page.once('error', (error) => {
log.exception(error, 'PuppeteerPool: page crashed');
// Ignore errors from Page.close()
page.close();