Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
chore: upgrade `browser-pool` (#1210)
Szymon Marczakb16e0e089bf0
diff --git a/package.json b/package.jsonindex b00e220d0..627488881 100644--- a/package.json+++ b/package.json@@ -56,7 +56,7 @@"@apify/storage-local": "^2.0.1","@apify/utilities": "^1.1.2","apify-client": "^2.0.2",- "browser-pool": "^2.0.2",+ "browser-pool": "^3.0.0","cheerio": "1.0.0-rc.10","content-type": "^1.0.4","got-scraping": "^3.2.5",diff --git a/src/browser_launchers/browser_launcher.js b/src/browser_launchers/browser_launcher.jsindex a247cbffc..3a0a1998b 100644--- a/src/browser_launchers/browser_launcher.js+++ b/src/browser_launchers/browser_launcher.js@@ -159,5 +159,13 @@ export default class BrowserLauncher {if (!/^(http|https|socks4|socks5)/i.test(proxyUrl)) {throw new Error(`Invalid "proxyUrl". Unsupported protocol: ${proxyUrl}.`);}++ const url = new URL(proxyUrl);++ if (url.username || url.password) {+ if (url.protocol !== 'http:' && url.protocol !== 'https:') {+ throw new Error('Invalid "proxyUrl" option: authentication is only supported for HTTP proxy type.');+ }+ }}}diff --git a/test/browser_launchers/playwright_launcher.test.js b/test/browser_launchers/playwright_launcher.test.jsindex 32d8e17af..0d8d5343c 100644--- a/test/browser_launchers/playwright_launcher.test.js+++ b/test/browser_launchers/playwright_launcher.test.js@@ -72,7 +72,6 @@ describe('Apify.launchPlaywright()', () => {await expect(Apify.launchPlaywright({ proxyUrl: {} })).rejects.toThrow(Error);await expect(Apify.launchPlaywright({ proxyUrl: 'invalidurl' })).rejects.toThrow(Error);await expect(Apify.launchPlaywright({ proxyUrl: 'invalid://somehost:1234' })).rejects.toThrow(Error);- await expect(Apify.launchPlaywright({ proxyUrl: 'https://user:[email protected]:1234' })).rejects.toThrow(Error);await expect(Apify.launchPlaywright({ proxyUrl: 'socks4://user:[email protected]:1234' })).rejects.toThrow(Error);await expect(Apify.launchPlaywright({ proxyUrl: 'socks5://user:[email protected]:1234' })).rejects.toThrow(Error);await expect(Apify.launchPlaywright({ proxyUrl: ' something really bad' })).rejects.toThrow(Error);diff --git a/test/browser_launchers/puppeteer_launcher.test.js b/test/browser_launchers/puppeteer_launcher.test.jsindex 01af94fe3..f04433a2c 100644--- a/test/browser_launchers/puppeteer_launcher.test.js+++ b/test/browser_launchers/puppeteer_launcher.test.js@@ -98,7 +98,6 @@ describe('Apify.launchPuppeteer()', () => {await expect(Apify.launchPuppeteer({ proxyUrl: {} })).rejects.toThrow(Error);await expect(Apify.launchPuppeteer({ proxyUrl: 'invalidurl' })).rejects.toThrow(Error);await expect(Apify.launchPuppeteer({ proxyUrl: 'invalid://somehost:1234' })).rejects.toThrow(Error);- await expect(Apify.launchPuppeteer({ proxyUrl: 'https://user:[email protected]:1234' })).rejects.toThrow(Error);await expect(Apify.launchPuppeteer({ proxyUrl: 'socks4://user:[email protected]:1234' })).rejects.toThrow(Error);await expect(Apify.launchPuppeteer({ proxyUrl: 'socks5://user:[email protected]:1234' })).rejects.toThrow(Error);await expect(Apify.launchPuppeteer({ proxyUrl: ' something really bad' })).rejects.toThrow(Error);@@ -243,6 +242,7 @@ describe('Apify.launchPuppeteer()', () => {return {on() {},close() {},+ newPage() {},};},someProps,