Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Use pipes instead of web sockets
Ondra Urban6345aeecbf51
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 21d103e96..a00866e78 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -1,3 +1,8 @@+xxx+===================+- Use pipes instead of web sockets in Puppeteer.++0.14.10 / 2019-05-24===================- Update `Puppeteer` to 1.17.0.diff --git a/src/puppeteer.js b/src/puppeteer.jsindex c3406d9c4..c31b7a705 100644--- a/src/puppeteer.js+++ b/src/puppeteer.js@@ -242,6 +242,8 @@ export const launchPuppeteer = async (options = {}) => {optsCopy.args.push(`--user-agent=${userAgent}`);}+ if (optsCopy.pipe !== false) optsCopy.pipe = true;+let browser;if (optsCopy.proxyUrl) {browser = await launchPuppeteerWithProxy(puppeteer, optsCopy);diff --git a/src/puppeteer_pool.js b/src/puppeteer_pool.jsindex 3353ef3f8..674927d5f 100644--- a/src/puppeteer_pool.js+++ b/src/puppeteer_pool.js@@ -16,6 +16,10 @@ import LiveViewServer from './live_view/live_view_server';const PROCESS_KILL_TIMEOUT_MILLIS = 5000;const PAGE_CLOSE_KILL_TIMEOUT_MILLIS = 1000;+// See https://github.com/GoogleChrome/puppeteer/blob/2b68c104aaf309488fe710a0dd7127369373d5fb/lib/Launcher.js#L174+const PUPPETEER_PIPE_TRANSPORT_WRITE = 3;+const PUPPETEER_PIPE_TRANSPORT_READ = 4;+const DEFAULT_OPTIONS = {reusePages: false,// Don't make these too large, otherwise Puppeteer might start crashing weirdly,@@ -226,6 +230,9 @@ class PuppeteerPool {opts.proxyUrl = this.proxyUrls[this.lastUsedProxyUrlIndex++ % this.proxyUrls.length];}+ // Set timeout for browser launch.+ if (opts.timeout == null) opts.timeout = this.puppeteerOperationTimeoutMillis;+const browser = await launchPuppeteerFunction(opts);if (!browser || typeof browser.newPage !== 'function') {// eslint-disable-next-line max-len@@ -266,25 +273,7 @@ class PuppeteerPool {const id = this.browserCounter++;log.debug('PuppeteerPool: Launching new browser', { id });- const errorMessageChunk = 'launchPuppeteerFunction timed out.';- const launchPromise = this.launchPuppeteerFunction();- const browserPromise = addTimeoutToPromise(- launchPromise,- this.puppeteerOperationTimeoutMillis,- `PuppeteerPool: ${errorMessageChunk}`,- );-- // If the browserPromise times out, the browser may still be started- // later and will not be managed by pool, so we need to get rid of it.- browserPromise.catch(async (err) => {- if (err.stack.includes(errorMessageChunk)) {- try {- const browser = await launchPromise;- browser.disconnect();- browser.process().kill('SIGKILL');- } catch (e) { /* do nothing, will be handled elsewhere */ }- }- });+ const browserPromise = this.launchPuppeteerFunction();const instance = new PuppeteerInstance(id, browserPromise);this.activeInstances[id] = instance;@@ -316,6 +305,8 @@ class PuppeteerPool {instance.childProcess = browser.process();instance.recycleDiskCacheDir = browser.recycleDiskCacheDir;+ this._attachPipeErrorHandlers(instance.childProcess);+browser.on('disconnected', () => {// If instance.killed === true then we killed the instance so don't log it.if (!instance.killed) log.error('PuppeteerPool: Puppeteer sent "disconnect" event. Maybe it crashed???', { id });@@ -341,6 +332,26 @@ class PuppeteerPool {});}+ /**+ * When using "pipe: true" launch option, the pipe can break for various reasons,+ * typically when the browser gets killed by PuppeteerPool. We swallow those errors,+ * because there's nothing we can do anyway and we don't want them to crash Node.+ * @param browserProcess+ * @ignore+ */+ _attachPipeErrorHandlers(browserProcess) { // eslint-disable-line class-methods-use-this+ const writeSocket = browserProcess.stdio[PUPPETEER_PIPE_TRANSPORT_WRITE];+ const readSocket = browserProcess.stdio[PUPPETEER_PIPE_TRANSPORT_READ];++ const errorHandler = (err) => {+ if (err.code === 'EPIPE') log.debug('Puppeteer Pool: Browser connection failed with an EPIPE error.');+ else throw err;+ };++ writeSocket.on('error', errorHandler);+ readSocket.on('error', errorHandler);+ }+/*** Retires some of the instances for example due to many uses.*diff --git a/test/puppeteer_pool.js b/test/puppeteer_pool.jsindex 616f16991..30a24d34a 100644--- a/test/puppeteer_pool.js+++ b/test/puppeteer_pool.js@@ -89,10 +89,9 @@ describe('PuppeteerPool', () => {await (await browsers[2]).close();await (await browsers[6]).close();- // @TODO for some reason it fails here:- // await (await browsers[7]).close();- // await shortSleep();- // expect(_.values(pool.retiredInstances).length).to.be.eql(0);+ await (await browsers[7]).close();+ await shortSleep(2000);+ expect(_.values(pool.retiredInstances).length).to.be.eql(0);// Cleanup everything.await pool.destroy();@@ -476,10 +475,6 @@ describe('PuppeteerPool', () => {await pool.destroy();});-- xit('should work together with request interception', async () => {- // TODO- });});describe('the proxyUrls parameter', () => {