Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Add puppeteerPool.retire() function to manually retire browsers
Ondra Urbanfa13de7578bf
diff --git a/src/puppeteer_pool.js b/src/puppeteer_pool.jsindex bbe51eb0e..ad73ec0af 100644--- a/src/puppeteer_pool.js+++ b/src/puppeteer_pool.js@@ -303,4 +303,38 @@ export default class PuppeteerPool {.all(closePromises).catch(err => log.exception(err, 'PuppeteerPool: cannot close the browsers'));}++ /**+ * Finds a PuppeteerInstance given a Puppeteer Browser running in the instance.+ * @param {Puppeteer.Browser} browser+ * @ignore+ */+ _findInstanceByBrowser(browser) {+ const instances = Object.values(this.activeInstances);+ return Promise.filter(instances, instance => instance.browserPromise.then(savedBrowser => browser === savedBrowser))+ .then((results) => {+ switch (results.length) {+ case 0:+ return null;+ case 1:+ return results[0];+ default:+ throw new Error('PuppeteerPool: Multiple instances of PuppeteerPool found using a single browser instance.');+ }+ });+ }++ /**+ * Manually retires a Puppeteer Browser instance from the pool. The browser will continue+ * to process open pages so that they may gracefully finish. This is unlike browser.close()+ * which will forcibly terminate the browser and all open pages will be closed.+ * @param {Puppeteer.Browser} browser+ */+ retire(browser) {+ return this._findInstanceByBrowser(browser)+ .then((instance) => {+ if (instance) return this._retireInstance(instance);+ log.debug('PuppeteerPool: browser is retired already');+ });+ }}diff --git a/test/puppeteer_pool.js b/test/puppeteer_pool.jsindex 226847387..e723226e4 100644--- a/test/puppeteer_pool.js+++ b/test/puppeteer_pool.js@@ -148,4 +148,47 @@ describe('PuppeteerPool', () => {delete process.env[ENV_VARS.PROXY_HOSTNAME];delete process.env[ENV_VARS.PROXY_PORT];});++ it('retire manually', async () => {+ process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';+ process.env[ENV_VARS.PROXY_HOSTNAME] = 'my.host.com';+ process.env[ENV_VARS.PROXY_PORT] = 123;++ const pool = new Apify.PuppeteerPool({+ maxOpenPagesPerInstance: 1,+ abortInstanceAfterRequestCount: 5,+ instanceKillerIntervalMillis: 1000,+ killInstanceAfterMillis: 500,+ });+ const pages = [];++ // Open 3 pages.+ pages.push(pool.newPage());+ pages.push(pool.newPage());+ pages.push(pool.newPage());+ await Promise.all(pages);+ expect(_.values(pool.activeInstances).length).to.be.eql(3);+ expect(_.values(pool.retiredInstances).length).to.be.eql(0);++ // Retire 1.+ await pool.retire((await pages[0]).browser());+ expect(_.values(pool.activeInstances).length).to.be.eql(2);+ expect(_.values(pool.retiredInstances).length).to.be.eql(1);++ // Retire 2.+ await pool.retire((await pages[1]).browser());+ expect(_.values(pool.activeInstances).length).to.be.eql(1);+ expect(_.values(pool.retiredInstances).length).to.be.eql(2);++ // Sleep and check that two have been killed.+ await shortSleep(2100);+ expect(_.values(pool.activeInstances).length).to.be.eql(1);+ expect(_.values(pool.retiredInstances).length).to.be.eql(0);++ // Cleanup everything.+ await pool.destroy();+ delete process.env[ENV_VARS.PROXY_PASSWORD];+ delete process.env[ENV_VARS.PROXY_HOSTNAME];+ delete process.env[ENV_VARS.PROXY_PORT];+ });});