Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Kill missed browsers

Ondra Urbane797bed2f3b9
diff --git a/src/puppeteer_pool.js b/src/puppeteer_pool.js
index 57e53de06..1a34552fe 100644
--- a/src/puppeteer_pool.js
+++ b/src/puppeteer_pool.js
@@ -257,11 +257,25 @@ class PuppeteerPool {
const id = this.browserCounter++;
log.debug('PuppeteerPool: Launching new browser', { id });
+ const launchPromise = this.launchPuppeteerFunction();
const browserPromise = addTimeoutToPromise(
- this.launchPuppeteerFunction(),
+ launchPromise,
this.puppeteerOperationTimeoutMillis,
'PuppeteerPool: launchPuppeteerFunction timed out.',
);
+
+ // If the browserPromise times out, the browser may still be started
+ // later and will not be managed by pool, so we need to get rid of it.
+ browserPromise.catch(async (err) => {
+ if (err.stack.includes('launchPuppeteerFunction timed out')) {
+ try {
+ const browser = await launchPromise;
+ browser.disconnect();
+ browser.process().kill('SIGKILL');
+ } catch (e) { /* do nothing, will be handled elsewhere */ }
+ }
+ });
+
const instance = new PuppeteerInstance(id, browserPromise);
this.activeInstances[id] = instance;
diff --git a/test/puppeteer_pool.js b/test/puppeteer_pool.js
index b68bcd939..57b6faa0e 100644
--- a/test/puppeteer_pool.js
+++ b/test/puppeteer_pool.js
@@ -557,19 +557,20 @@ describe('PuppeteerPool', () => {
describe('prevents hanging of puppeteer operations', () => {
let pool;
beforeEach(() => {
+ log.setLevel(log.LEVELS.OFF);
pool = new Apify.PuppeteerPool({
- puppeteerOperationTimeoutSecs: 0.5,
+ puppeteerOperationTimeoutSecs: 0.05,
launchPuppeteerOptions: {
headless: true,
},
});
});
- afterEach(() => pool.destroy());
+ afterEach(async () => {
+ log.setLevel(log.LEVELS.ERROR);
+ await pool.destroy();
+ });
it('should work', async () => {
- const page = await pool.newPage();
- const browser = page.browser();
- browser.newPage = () => shortSleep(2000);
try {
await pool._openNewTab(0); // eslint-disable-line no-underscore-dangle
throw new Error('invalid error');