Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Resolve merge conflicts

Ondra Urban6ca73afc3526
diff --cc src/puppeteer_pool.js
index be2bb5f51,ad73ec0af..97650cbdd
--- a/src/puppeteer_pool.js
+++ b/src/puppeteer_pool.js
@@@ -120,9 -120,6 +120,10 @@@ export default class PuppeteerPool
this.activeInstances = {};
this.retiredInstances = {};
this.instanceKillerInterval = setInterval(() => this._killRetiredInstances(), instanceKillerIntervalMillis);
+
+ // ensure termination on SIGINT
- this._terminateInstancesOnSigint();
++ this.sigintListener = () => this._killAllInstances();
++ process.on('SIGINT', this.sigintListener);
}
/**
@@@ -238,6 -235,6 +239,21 @@@
.catch(err => log.exception(err, 'PuppeteerPool: cannot close the browser instance', { id }));
}
++ /**
++ * Kills all running PuppeteerInstances.
++ * @ignore
++ */
++ _killAllInstances() {
++ const allInstances = Object.values(this.activeInstances).concat(Object.values(this.retiredInstances));
++ allInstances.forEach((instance) => {
++ try {
++ instance.childProcess.kill('SIGINT');
++ } catch (e) {
++ // do nothing, it's dead
++ }
++ });
++ }
++
/**
* Opens new tab in one of the browsers and returns promise that resolves to its Puppeteer.Page.
*
@@@ -287,6 -284,6 +303,7 @@@
*/
destroy() {
clearInterval(this.instanceKillerInterval);
++ process.removeListener('SIGINT', this.sigintListener);
const browserPromises = _
.values(this.activeInstances)