Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Resolve merge conflicts
Ondra Urban6ca73afc3526
diff --cc src/puppeteer_pool.jsindex be2bb5f51,ad73ec0af..97650cbdd--- a/src/puppeteer_pool.js+++ b/src/puppeteer_pool.js@@@ -120,9 -120,6 +120,10 @@@ export default class PuppeteerPoolthis.activeInstances = {};this.retiredInstances = {};this.instanceKillerInterval = setInterval(() => this._killRetiredInstances(), instanceKillerIntervalMillis);++ // ensure termination on SIGINT- this._terminateInstancesOnSigint();++ this.sigintListener = () => this._killAllInstances();++ process.on('SIGINT', this.sigintListener);}/**@@@ -238,6 -235,6 +239,21 @@@.catch(err => log.exception(err, 'PuppeteerPool: cannot close the browser instance', { id }));}++ /**++ * Kills all running PuppeteerInstances.++ * @ignore++ */++ _killAllInstances() {++ const allInstances = Object.values(this.activeInstances).concat(Object.values(this.retiredInstances));++ allInstances.forEach((instance) => {++ try {++ instance.childProcess.kill('SIGINT');++ } catch (e) {++ // do nothing, it's dead++ }++ });++ }++/*** Opens new tab in one of the browsers and returns promise that resolves to its Puppeteer.Page.*@@@ -287,6 -284,6 +303,7 @@@*/destroy() {clearInterval(this.instanceKillerInterval);++ process.removeListener('SIGINT', this.sigintListener);const browserPromises = _.values(this.activeInstances)