Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Bump puppeteer to 1.19.0; Use pipes again
Ondra Urbanc6a4d4dc47e8
diff --git a/package.json b/package.jsonindex 1908b40d3..0287ffa50 100644--- a/package.json+++ b/package.json@@ -104,6 +104,6 @@"why-is-node-running": "^2.0.3"},"optionalDependencies": {- "puppeteer": "1.18.1"+ "puppeteer": "1.19.0"}}diff --git a/src/puppeteer.js b/src/puppeteer.jsindex 0419dca64..3f8ae5446 100644--- a/src/puppeteer.js+++ b/src/puppeteer.js@@ -263,8 +263,7 @@ export const launchPuppeteer = async (options = {}) => {optsCopy.args.push(`--user-agent=${userAgent}`);}- // TODO Monitor https://github.com/GoogleChrome/puppeteer/issues/4374- // if (optsCopy.pipe !== false) optsCopy.pipe = true;+ if (optsCopy.pipe !== false) optsCopy.pipe = true;let browser;if (optsCopy.proxyUrl) {diff --git a/src/puppeteer_pool.js b/src/puppeteer_pool.jsindex 415b511f9..1368b79da 100644--- a/src/puppeteer_pool.js+++ b/src/puppeteer_pool.js@@ -16,10 +16,6 @@ import LiveViewServer from './live_view/live_view_server';const PROCESS_KILL_TIMEOUT_MILLIS = 5000;const PAGE_CLOSE_KILL_TIMEOUT_MILLIS = 1000;-// See https://github.com/GoogleChrome/puppeteer/blob/2b68c104aaf309488fe710a0dd7127369373d5fb/lib/Launcher.js#L174-const PUPPETEER_PIPE_TRANSPORT_WRITE = 3;-const PUPPETEER_PIPE_TRANSPORT_READ = 4;-const DEFAULT_OPTIONS = {reusePages: false,// Don't make these too large, otherwise Puppeteer might start crashing weirdly,@@ -307,8 +303,6 @@ class PuppeteerPool {instance.childProcess = browser.process();instance.recycleDiskCacheDir = browser.recycleDiskCacheDir;- this._attachPipeErrorHandlers(instance.childProcess);-browser.on('disconnected', () => {// If instance.killed === true then we killed the instance so don't log it.if (!instance.killed) log.error('PuppeteerPool: Puppeteer sent "disconnect" event. Maybe it crashed???', { id });@@ -334,28 +328,6 @@ class PuppeteerPool {});}- /**- * When using "pipe: true" launch option, the pipe can break for various reasons,- * typically when the browser gets killed by PuppeteerPool. We swallow those errors,- * because there's nothing we can do anyway and we don't want them to crash Node.- * @param chromeProcess- * @ignore- */- _attachPipeErrorHandlers(chromeProcess) { // eslint-disable-line class-methods-use-this- const writeSocket = chromeProcess.stdio[PUPPETEER_PIPE_TRANSPORT_WRITE];- const readSocket = chromeProcess.stdio[PUPPETEER_PIPE_TRANSPORT_READ];-- if (!(writeSocket && readSocket)) return; // We're not using pipes.-- const errorHandler = (err) => {- log.debug('Puppeteer Pool: Browser connection failed. Browser will shut down.', { message: err.message });- if (chromeProcess) chromeProcess.kill('SIGKILL');- };-- writeSocket.on('error', errorHandler);- readSocket.on('error', errorHandler);- }-/*** Retires some of the instances for example due to many uses.*