Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Write tests for proxyUrls, fix wrong error logging in Pool
Ondra Urban15f16773149c
diff --git a/src/puppeteer_pool.js b/src/puppeteer_pool.jsindex 353be464d..1ca78b0b3 100644--- a/src/puppeteer_pool.js+++ b/src/puppeteer_pool.js@@ -2,9 +2,9 @@ import _ from 'underscore';import fs from 'fs';import os from 'os';import path from 'path';+import util from 'util';import log from 'apify-shared/log';import LinkedList from 'apify-shared/linked_list';-import Promise from 'bluebird';import rimraf from 'rimraf';import { checkParamOrThrow } from 'apify-client/build/utils';import { launchPuppeteer } from './puppeteer';@@ -30,8 +30,8 @@ const DEFAULT_OPTIONS = {recycleDiskCache: false,};-const mkdtempAsync = Promise.promisify(fs.mkdtemp);-const rimrafAsync = Promise.promisify(rimraf);+const mkdtempAsync = util.promisify(fs.mkdtemp);+const rimrafAsync = util.promisify(rimraf);const DISK_CACHE_DIR = path.join(os.tmpdir(), 'puppeteer_disk_cache-');/**@@ -170,6 +170,8 @@ class PuppeteerPool {checkParamOrThrow(launchPuppeteerOptions, 'options.launchPuppeteerOptions', 'Maybe Object');checkParamOrThrow(recycleDiskCache, 'options.recycleDiskCache', 'Maybe Boolean');checkParamOrThrow(proxyUrls, 'options.proxyUrls', 'Maybe Array');+ // Enforce non-empty proxyUrls array+ if (proxyUrls && !proxyUrls.length) throw new Error('Parameter "options.proxyUrls" of type Array must not be empty');// The recycleDiskCache option is only supported in headful mode// See https://bugs.chromium.org/p/chromium/issues/detail?id=882431@@ -188,7 +190,7 @@ class PuppeteerPool {this.proxyUrls = _.shuffle(proxyUrls);this.launchPuppeteerFunction = async () => {// Do not modify passed launchPuppeteerOptions!- let opts = _.clone(launchPuppeteerOptions) || {};+ const opts = _.clone(launchPuppeteerOptions) || {};opts.args = _.clone(opts.args || []);// If requested, use recycled disk cache directory@@ -200,16 +202,10 @@ class PuppeteerPool {opts.args.push(`--disk-cache-dir=${diskCacheDir}`);}- // Select correct proxy options+ // Rotate custom proxyUrls.if (this.proxyUrls) {- if (this.proxyUrls.length === 0) {- log.warning('PuppeteerPool: Using "options.proxyUrls" parameter with 0 proxies. Proxy is disabled.');- } else if (opts.useApifyProxy) {- log.warning('PuppeteerPool: Using "options.proxyUrls" parameter in conjuction with '- + '"options.launchPuppeteerOptions.useApifyProxy" parameter. Apify Proxy is disabled.');- }- opts = _.omit(opts, ['useApifyProxy', 'apifyProxyGroups', 'apifyProxySession']);opts.proxyUrl = this.proxyUrls[this.lastUsedProxyUrlIndex % this.proxyUrls.length];+ this.lastUsedProxyUrlIndex++;}const browser = await launchPuppeteerFunction(opts);@@ -240,50 +236,62 @@ class PuppeteerPool {const browserPromise = this.launchPuppeteerFunction();const instance = new PuppeteerInstance(id, browserPromise);-- instance- .browserPromise- .then((browser) => {- browser.on('disconnected', () => {- // If instance.killed === true then we killed the instance so don't log it.- if (!instance.killed) log.error('PuppeteerPool: Puppeteer sent "disconnect" event. Maybe it crashed???', { id });- this._retireInstance(instance);- });- // This one is done manually in Puppeteerpool.newPage() so that it happens immediately.- // browser.on('targetcreated', () => instance.activePages++);- browser.on('targetdestroyed', (target) => {- // The event is also called for service workers and Chromium extensions, which must be ignored!- const type = target.type();- if (type !== 'page' && type !== 'other') return;-- instance.activePages--;-- if (instance.activePages === 0 && this.retiredInstances[id]) {- // Run this with a delay, otherwise page.close() that initiated this 'targetdestroyed' event- // might fail with "Protocol error (Target.closeTarget): Target closed."- // TODO: Alternatively we could close here the first about:blank tab, which will cause- // the browser to be closed immediately without waiting- setTimeout(() => {- log.debug('PuppeteerPool: killing retired browser because it has no active pages', { id });- this._killInstance(instance);- }, PAGE_CLOSE_KILL_TIMEOUT_MILLIS);- }- });-- instance.childProcess = browser.process();- instance.recycleDiskCacheDir = browser.recycleDiskCacheDir;- })- .catch((err) => {- log.exception(err, 'PuppeteerPool: browser launch failed', { id });-- return this._retireInstance(instance);- });-this.activeInstances[id] = instance;+ // Handle the async stuff elsewhere.+ this._initBrowser(browserPromise, instance);+return instance;}+ /**+ * Takes care of async processes in PuppeteerInstance construction with a Browser.+ * @param {Promise<Browser>} browserPromise+ * @param {PuppeteerInstance} instance+ * @returns {Promise}+ * @ignore+ */+ async _initBrowser(browserPromise, instance) {+ const { id } = instance;+ let browser;+ try {+ browser = await browserPromise;+ } catch (err) {+ log.exception(err, 'PuppeteerPool: browser launch failed', { id });+ delete this.activeInstances[id];+ return;+ }++ instance.childProcess = browser.process();+ instance.recycleDiskCacheDir = browser.recycleDiskCacheDir;++ browser.on('disconnected', () => {+ // If instance.killed === true then we killed the instance so don't log it.+ if (!instance.killed) log.error('PuppeteerPool: Puppeteer sent "disconnect" event. Maybe it crashed???', { id });+ this._retireInstance(instance);+ });+ // This one is done manually in Puppeteerpool.newPage() so that it happens immediately.+ // browser.on('targetcreated', () => instance.activePages++);+ browser.on('targetdestroyed', (target) => {+ // The event is also called for service workers and Chromium extensions, which must be ignored!+ const type = target.type();+ if (type !== 'page' && type !== 'other') return;++ instance.activePages--;++ if (instance.activePages === 0 && this.retiredInstances[id]) {+ // Run this with a delay, otherwise page.close() that initiated this 'targetdestroyed' event+ // might fail with "Protocol error (Target.closeTarget): Target closed."+ // TODO: Alternatively we could close here the first about:blank tab, which will cause+ // the browser to be closed immediately without waiting+ setTimeout(() => {+ log.debug('PuppeteerPool: killing retired browser because it has no active pages', { id });+ this._killInstance(instance);+ }, PAGE_CLOSE_KILL_TIMEOUT_MILLIS);+ }+ });+ }+/*** Retires some of the instances for example due to many uses.*@@ -310,7 +318,7 @@ class PuppeteerPool {_killRetiredInstances() {log.debug('PuppeteerPool: retired browsers count', { count: _.values(this.retiredInstances).length });- _.mapObject(this.retiredInstances, (instance) => {+ _.mapObject(this.retiredInstances, async (instance) => {// Kill instances that are more than this.killInstanceAfterMillis from last opened pageif (Date.now() - instance.lastPageOpenedAt > this.killInstanceAfterMillis) {log.debug('PuppeteerPool: killing retired browser after period of inactivity', { id: instance.id, killInstanceAfterMillis: this.killInstanceAfterMillis }); // eslint-disable-line max-len@@ -318,19 +326,18 @@ class PuppeteerPool {return;}- instance- .browserPromise- .then(browser => browser.pages())- .then((pages) => {- // NOTE: we are killing instance when number of pages below 1 because there is always about:blank page.- if (pages.length <= 1) {- log.debug('PuppeteerPool: killing retired browser because it has no open tabs', { id: instance.id });- this._killInstance(instance);- }- }, (err) => {- log.exception(err, 'PuppeteerPool: browser.pages() failed', { id: instance.id });+ const browser = await instance.browserPromise;+ try {+ const pages = await browser.pages();+ // NOTE: we are killing instance when the number of pages is less or equal to 1 because there is always about:blank page.+ if (pages.length <= 1) {+ log.debug('PuppeteerPool: killing retired browser because it has no open tabs', { id: instance.id });this._killInstance(instance);- });+ }+ } catch (err) {+ log.exception(err, 'PuppeteerPool: browser.pages() failed', { id: instance.id });+ this._killInstance(instance);+ }});}@@ -339,8 +346,9 @@ class PuppeteerPool {** @ignore*/- _killInstance(instance) {- const { id, childProcess } = instance;+ async _killInstance(instance) {+ const { id, childProcess, killed, browserPromise } = instance;+ if (killed) return;log.debug('PuppeteerPool: killing browser', { id });@@ -363,19 +371,14 @@ class PuppeteerPool {recycleDiskCache();}, PROCESS_KILL_TIMEOUT_MILLIS);- instance- .browserPromise- .then((browser) => {- if (instance.killed) return;-- instance.killed = true;-- return browser.close();- })- .then(() => {- recycleDiskCache();- })- .catch(err => log.exception(err, 'PuppeteerPool: cannot close the browser instance', { id }));+ try {+ const browser = await browserPromise;+ instance.killed = true;+ await browser.close();+ recycleDiskCache();+ } catch (err) {+ log.exception(err, 'PuppeteerPool: cannot close the browser instance', { id });+ }}/**@@ -445,7 +448,7 @@ class PuppeteerPool {* Closes all open browsers.* @return {Promise}*/- destroy() {+ async destroy() {clearInterval(this.instanceKillerInterval);process.removeListener('SIGINT', this.sigintListener);@@ -461,28 +464,23 @@ class PuppeteerPool {return instance.browserPromise;});- const closePromises = browserPromises.map((browserPromise) => {- return browserPromise- .then((browser) => {- browser.close();- return browser;- })- .then((browser) => {- if (browser.recycleDiskCacheDir) return deleteDiskCacheDir(browser.recycleDiskCacheDir);- });+ const closePromises = browserPromises.map(async (browserPromise) => {+ const browser = await browserPromise;+ await browser.close();+ if (browser.recycleDiskCacheDir) await deleteDiskCacheDir(browser.recycleDiskCacheDir);});- return Promise- .all(closePromises)- .then(() => {- // Delete all cache directories- const promises = [];- while (this.recycledDiskCacheDirs && this.recycledDiskCacheDirs.length > 0) {- promises.push(deleteDiskCacheDir(this.recycledDiskCacheDirs.removeFirst()));- }- return Promise.all(promises);- })- .catch(err => log.exception(err, 'PuppeteerPool: cannot close the browsers'));+ try {+ await Promise.all(closePromises);+ // Delete all cache directories+ const dirDeletionPromises = [];+ while (this.recycledDiskCacheDirs && this.recycledDiskCacheDirs.length > 0) {+ dirDeletionPromises.push(deleteDiskCacheDir(this.recycledDiskCacheDirs.removeFirst()));+ }+ await Promise.all(dirDeletionPromises);+ } catch (err) {+ log.exception(err, 'PuppeteerPool: cannot close the browsers');+ }}/**@@ -491,19 +489,21 @@ class PuppeteerPool {* @return {Promise}* @ignore*/- _findInstanceByBrowser(browser) {+ async _findInstanceByBrowser(browser) {const instances = Object.values(this.activeInstances);- return Promise.filter(instances, instance => instance.browserPromise.then(savedBrowser => browser === savedBrowser))- .then((results) => {- switch (results.length) {- case 0:- return null;- case 1:- return results[0];- default:- throw new Error('PuppeteerPool: Multiple instances of PuppeteerPool found using a single browser instance.');- }- });+ const resultPromises = instances.map(async (instance) => {+ const savedBrowser = await instance.browserPromise;+ return browser === savedBrowser ? instance : null;+ });+ const results = (await Promise.all(resultPromises)).filter(i => i);+ switch (results.length) {+ case 0:+ return null;+ case 1:+ return results[0];+ default:+ throw new Error('PuppeteerPool: Multiple instances of PuppeteerPool found using a single browser instance.');+ }}/**@@ -514,12 +514,10 @@ class PuppeteerPool {* @param {Browser} browser* @return {Promise}*/- retire(browser) {- return this._findInstanceByBrowser(browser)- .then((instance) => {- if (instance) return this._retireInstance(instance);- log.debug('PuppeteerPool: browser is retired already');- });+ async retire(browser) {+ const instance = await this._findInstanceByBrowser(browser);+ if (instance) return this._retireInstance(instance);+ log.debug('PuppeteerPool: browser is retired already');}}diff --git a/src/puppeteer_utils.js b/src/puppeteer_utils.jsindex 79be66199..c059f9460 100644--- a/src/puppeteer_utils.js+++ b/src/puppeteer_utils.js@@ -5,7 +5,7 @@ import _ from 'underscore';import log from 'apify-shared/log';import { checkParamOrThrow } from 'apify-client/build/utils';import { checkParamPrototypeOrThrow } from 'apify-shared/utilities';-import { RequestQueue, RequestQueueLocal, openRequestQueue } from './request_queue';+import { RequestQueue, RequestQueueLocal } from './request_queue';import Request from './request';import PseudoUrl from './pseudo_url';diff --git a/test/puppeteer_pool.js b/test/puppeteer_pool.jsindex 270a9c0a5..e957fa802 100644--- a/test/puppeteer_pool.js+++ b/test/puppeteer_pool.js@@ -6,6 +6,7 @@ import log from 'apify-shared/log';import 'babel-polyfill';import { ENV_VARS } from 'apify-shared/consts';import * as Apify from '../build/index';+import { launchPuppeteer } from '../build/puppeteer';chai.use(chaiAsPromised);@@ -314,4 +315,72 @@ describe('PuppeteerPool', () => {expect(fs.existsSync(dir1)).to.be.eql(false);expect(fs.existsSync(dir3)).to.be.eql(false);});++ describe('the proxyUrls parameter', () => {+ it('supports rotation of custom proxies', async () => {+ const optionsLog = [];+ const pool = new Apify.PuppeteerPool({+ maxOpenPagesPerInstance: 1,+ proxyUrls: ['http://proxy.com:1111', 'http://proxy.com:2222', 'http://proxy.com:3333'],+ launchPuppeteerFunction: async (launchOpts) => {+ optionsLog.push(launchOpts);+ return launchPuppeteer(launchOpts);+ },+ });+ const proxies = [...pool.proxyUrls];+ // Open 4 browsers to do full rotation cycle+ await pool.newPage();+ await pool.newPage();+ await pool.newPage();+ await pool.newPage();+ await pool.destroy();++ expect(optionsLog).to.have.lengthOf(4);+ expect(optionsLog[0].proxyUrl).to.be.eql(proxies[0]);+ expect(optionsLog[1].proxyUrl).to.be.eql(proxies[1]);+ expect(optionsLog[2].proxyUrl).to.be.eql(proxies[2]);+ expect(optionsLog[3].proxyUrl).to.be.eql(proxies[0]);+ });++ describe('throws', () => {+ let pool;+ beforeEach(() => {+ log.setLevel(log.LEVELS.OFF);+ });+ afterEach(async () => {+ log.setLevel(log.LEVELS.ERROR);+ if (pool) await pool.destroy();+ });++ it('when used with useApifyProxy', async () => {+ pool = new Apify.PuppeteerPool({+ maxOpenPagesPerInstance: 1,+ proxyUrls: ['http://proxy.com:1111', 'http://proxy.com:2222', 'http://proxy.com:3333'],+ launchPuppeteerOptions: {+ useApifyProxy: true,+ apifyProxyGroups: ['G1', 'G2'],+ },+ });++ try {+ await pool.newPage();+ throw new Error('Invalid error.');+ } catch (err) {+ expect(err.message).to.include('useApifyProxy');+ }+ });++ it('when empty', async () => {+ try {+ pool = new Apify.PuppeteerPool({+ maxOpenPagesPerInstance: 1,+ proxyUrls: [],+ });+ throw new Error('Invalid error.');+ } catch (err) {+ expect(err.message).to.include('must not be empty');+ }+ });+ });+ });});