Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: parse proxy URLs correctly when using `useIncognitoPages` (#3433)
Copilotdb2bb6817e0d
diff --git a/packages/browser-pool/src/playwright/playwright-controller.ts b/packages/browser-pool/src/playwright/playwright-controller.tsindex a04d007ca..06416aa5a 100644--- a/packages/browser-pool/src/playwright/playwright-controller.ts+++ b/packages/browser-pool/src/playwright/playwright-controller.ts@@ -27,7 +27,7 @@ export class PlaywrightController extends BrowserController<return {proxy: {- server: url.origin,+ server: `${url.protocol}//${url.host}`,username,password,bypass: pageOptions?.proxy?.bypass,diff --git a/packages/browser-pool/src/puppeteer/puppeteer-controller.ts b/packages/browser-pool/src/puppeteer/puppeteer-controller.tsindex 1e825f4e6..56aa038d9 100644--- a/packages/browser-pool/src/puppeteer/puppeteer-controller.ts+++ b/packages/browser-pool/src/puppeteer/puppeteer-controller.ts@@ -31,7 +31,7 @@ export class PuppeteerController extends BrowserController<const password = decodeURIComponent(url.password);return {- proxyServer: url.origin,+ proxyServer: `${url.protocol}//${url.host}`,proxyUsername: username,proxyPassword: password,proxyBypassList: pageOptions?.proxyBypassList,diff --git a/packages/stagehand-crawler/src/internals/stagehand-controller.ts b/packages/stagehand-crawler/src/internals/stagehand-controller.tsindex a870bdc69..6f344298b 100644--- a/packages/stagehand-crawler/src/internals/stagehand-controller.ts+++ b/packages/stagehand-crawler/src/internals/stagehand-controller.ts@@ -84,7 +84,7 @@ export class StagehandController extends BrowserController<BrowserType, LaunchOpreturn {proxy: {- server: url.origin,+ server: `${url.protocol}//${url.host}`,username,password,bypass: pageOptions?.proxy?.bypass,diff --git a/test/browser-pool/browser-plugins/plugins.test.ts b/test/browser-pool/browser-plugins/plugins.test.tsindex 54ff3cd41..d64c2a296 100644--- a/test/browser-pool/browser-plugins/plugins.test.ts+++ b/test/browser-pool/browser-plugins/plugins.test.ts@@ -356,6 +356,38 @@ describe('Plugins', () => {runPluginTest(PuppeteerPlugin, PuppeteerController, puppeteer);+ describe('normalizeProxyOptions', () => {+ test.each([+ ['socks5://user:[email protected]:1080', 'socks5://proxy.example.com:1080'],+ ['http://user:[email protected]:8080', 'http://proxy.example.com:8080'],+ ])('PlaywrightController should handle %s', (proxyUrl, expectedServer) => {+ const plugin = new PlaywrightPlugin(playwright.chromium);+ const browserController = new PlaywrightController(plugin);++ const result = browserController.normalizeProxyOptions(proxyUrl, {});++ expect(result).toMatchObject({+ proxy: { server: expectedServer, username: 'user', password: 'pass' },+ });+ });++ test.each([+ ['socks5://user:[email protected]:1080', 'socks5://proxy.example.com:1080'],+ ['http://user:[email protected]:8080', 'http://proxy.example.com:8080'],+ ])('PuppeteerController should handle %s', (proxyUrl, expectedServer) => {+ const plugin = new PuppeteerPlugin(puppeteer);+ const browserController = new PuppeteerController(plugin);++ const result = browserController.normalizeProxyOptions(proxyUrl, {});++ expect(result).toMatchObject({+ proxyServer: expectedServer,+ proxyUsername: 'user',+ proxyPassword: 'pass',+ });+ });+ });+describe('Playwright specifics', () => {let browser: playwright.Browser;