Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Switch back to default Web Sockets for Puppeteer

Ondra Urban860771aa356e
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2e6bf17d1..d39f8957d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,7 @@
0.15.2 / 2019-07-11
====================
- Fix error where Puppeteer would fail to launch when pipes are turned off.
+- Switch back to default Web Socket transport for Puppeteer due to upstream issues.
0.15.1 / 2019-07-09
====================
diff --git a/src/puppeteer.js b/src/puppeteer.js
index c31b7a705..7b779371a 100644
--- a/src/puppeteer.js
+++ b/src/puppeteer.js
@@ -242,7 +242,8 @@ export const launchPuppeteer = async (options = {}) => {
optsCopy.args.push(`--user-agent=${userAgent}`);
}
- if (optsCopy.pipe !== false) optsCopy.pipe = true;
+ // TODO Monitor https://github.com/GoogleChrome/puppeteer/issues/4374
+ // if (optsCopy.pipe !== false) optsCopy.pipe = true;
let browser;
if (optsCopy.proxyUrl) {
diff --git a/test/puppeteer_pool.js b/test/puppeteer_pool.js
index 86b7728cd..3ab190699 100644
--- a/test/puppeteer_pool.js
+++ b/test/puppeteer_pool.js
@@ -550,7 +550,6 @@ describe('PuppeteerPool', () => {
beforeEach(() => {
log.setLevel(log.LEVELS.OFF);
pool = new Apify.PuppeteerPool({
- puppeteerOperationTimeoutSecs: 0.005,
launchPuppeteerOptions: {
headless: true,
},
@@ -562,11 +561,14 @@ describe('PuppeteerPool', () => {
});
it('should work', async () => {
+ // Start browser;
+ await pool._openNewTab(); // eslint-disable-line no-underscore-dangle
+ pool.puppeteerOperationTimeoutMillis = 0.005;
try {
await pool._openNewTab(); // eslint-disable-line no-underscore-dangle
throw new Error('invalid error');
} catch (err) {
- expect(err.stack).to.include('timed out');
+ expect(err.stack).to.include('PuppeteerPool: browser.newPage() timed out.');
}
});
});