Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

chore: upgrade `browser-pool` (#1210)

Szymon Marczakb16e0e089bf0
diff --git a/package.json b/package.json
index b00e220d0..627488881 100644
--- a/package.json
+++ b/package.json
@@ -56,7 +56,7 @@
"@apify/storage-local": "^2.0.1",
"@apify/utilities": "^1.1.2",
"apify-client": "^2.0.2",
- "browser-pool": "^2.0.2",
+ "browser-pool": "^3.0.0",
"cheerio": "1.0.0-rc.10",
"content-type": "^1.0.4",
"got-scraping": "^3.2.5",
diff --git a/src/browser_launchers/browser_launcher.js b/src/browser_launchers/browser_launcher.js
index a247cbffc..3a0a1998b 100644
--- a/src/browser_launchers/browser_launcher.js
+++ b/src/browser_launchers/browser_launcher.js
@@ -159,5 +159,13 @@ export default class BrowserLauncher {
if (!/^(http|https|socks4|socks5)/i.test(proxyUrl)) {
throw new Error(`Invalid "proxyUrl". Unsupported protocol: ${proxyUrl}.`);
}
+
+ const url = new URL(proxyUrl);
+
+ if (url.username || url.password) {
+ if (url.protocol !== 'http:' && url.protocol !== 'https:') {
+ throw new Error('Invalid "proxyUrl" option: authentication is only supported for HTTP proxy type.');
+ }
+ }
}
}
diff --git a/test/browser_launchers/playwright_launcher.test.js b/test/browser_launchers/playwright_launcher.test.js
index 32d8e17af..0d8d5343c 100644
--- a/test/browser_launchers/playwright_launcher.test.js
+++ b/test/browser_launchers/playwright_launcher.test.js
@@ -72,7 +72,6 @@ describe('Apify.launchPlaywright()', () => {
await expect(Apify.launchPlaywright({ proxyUrl: {} })).rejects.toThrow(Error);
await expect(Apify.launchPlaywright({ proxyUrl: 'invalidurl' })).rejects.toThrow(Error);
await expect(Apify.launchPlaywright({ proxyUrl: 'invalid://somehost:1234' })).rejects.toThrow(Error);
- await expect(Apify.launchPlaywright({ proxyUrl: 'https://user:[email protected]:1234' })).rejects.toThrow(Error);
await expect(Apify.launchPlaywright({ proxyUrl: 'socks4://user:[email protected]:1234' })).rejects.toThrow(Error);
await expect(Apify.launchPlaywright({ proxyUrl: 'socks5://user:[email protected]:1234' })).rejects.toThrow(Error);
await expect(Apify.launchPlaywright({ proxyUrl: ' something really bad' })).rejects.toThrow(Error);
diff --git a/test/browser_launchers/puppeteer_launcher.test.js b/test/browser_launchers/puppeteer_launcher.test.js
index 01af94fe3..f04433a2c 100644
--- a/test/browser_launchers/puppeteer_launcher.test.js
+++ b/test/browser_launchers/puppeteer_launcher.test.js
@@ -98,7 +98,6 @@ describe('Apify.launchPuppeteer()', () => {
await expect(Apify.launchPuppeteer({ proxyUrl: {} })).rejects.toThrow(Error);
await expect(Apify.launchPuppeteer({ proxyUrl: 'invalidurl' })).rejects.toThrow(Error);
await expect(Apify.launchPuppeteer({ proxyUrl: 'invalid://somehost:1234' })).rejects.toThrow(Error);
- await expect(Apify.launchPuppeteer({ proxyUrl: 'https://user:[email protected]:1234' })).rejects.toThrow(Error);
await expect(Apify.launchPuppeteer({ proxyUrl: 'socks4://user:[email protected]:1234' })).rejects.toThrow(Error);
await expect(Apify.launchPuppeteer({ proxyUrl: 'socks5://user:[email protected]:1234' })).rejects.toThrow(Error);
await expect(Apify.launchPuppeteer({ proxyUrl: ' something really bad' })).rejects.toThrow(Error);
@@ -243,6 +242,7 @@ describe('Apify.launchPuppeteer()', () => {
return {
on() {},
close() {},
+ newPage() {},
};
},
someProps,