Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Adding timeout that ensures that chrome will be really killed in PuppeteerPool

Marek Trunkat5497adbff39c
diff --git a/src/puppeteer_pool.js b/src/puppeteer_pool.js
index 899ec8876..f08fb9d53 100644
--- a/src/puppeteer_pool.js
+++ b/src/puppeteer_pool.js
@@ -12,6 +12,8 @@ const DEFAULT_PUPPETEER_CONFIG = {
args: [],
};
+const PROCESS_KILL_TIMEOUT_MILLIS = 5000;
+
const DEFAULT_OPTIONS = {
maxOpenPagesPerInstance: 100,
abortInstanceAfterRequestCount: 150,
@@ -59,6 +61,7 @@ class PuppeteerInstance {
this.browserPromise = browserPromise;
this.lastPageOpenedAt = Date.now();
this.killed = false;
+ this.childProcess = null;
}
}
@@ -155,6 +158,8 @@ export default class PuppeteerPool {
if (instance.activePages === 0 && this.retiredInstances[id]) this._killInstance(instance);
});
+
+ instance.childProcess = browser.process();
})
.catch((err) => {
log.exception(err, 'PuppeteerPool: Browser start failed', { id });
@@ -215,12 +220,15 @@ export default class PuppeteerPool {
* @ignore
*/
_killInstance(instance) {
- const { id } = instance;
+ const { id, childProcess } = instance;
log.info('PuppeteerPool: killing browser', { id });
delete this.retiredInstances[id];
+ // Ensure that Chrome process will be really killed.
+ setTimeout(() => childProcess.kill('SIGKILL'), PROCESS_KILL_TIMEOUT_MILLIS);
+
instance
.browserPromise
.then((browser) => {