Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: parse proxy URLs correctly when using `useIncognitoPages` (#3433)

Copilotdb2bb6817e0d
diff --git a/packages/browser-pool/src/playwright/playwright-controller.ts b/packages/browser-pool/src/playwright/playwright-controller.ts
index a04d007ca..06416aa5a 100644
--- a/packages/browser-pool/src/playwright/playwright-controller.ts
+++ b/packages/browser-pool/src/playwright/playwright-controller.ts
@@ -27,7 +27,7 @@ export class PlaywrightController extends BrowserController<
return {
proxy: {
- server: url.origin,
+ server: `${url.protocol}//${url.host}`,
username,
password,
bypass: pageOptions?.proxy?.bypass,
diff --git a/packages/browser-pool/src/puppeteer/puppeteer-controller.ts b/packages/browser-pool/src/puppeteer/puppeteer-controller.ts
index 1e825f4e6..56aa038d9 100644
--- a/packages/browser-pool/src/puppeteer/puppeteer-controller.ts
+++ b/packages/browser-pool/src/puppeteer/puppeteer-controller.ts
@@ -31,7 +31,7 @@ export class PuppeteerController extends BrowserController<
const password = decodeURIComponent(url.password);
return {
- proxyServer: url.origin,
+ proxyServer: `${url.protocol}//${url.host}`,
proxyUsername: username,
proxyPassword: password,
proxyBypassList: pageOptions?.proxyBypassList,
diff --git a/packages/stagehand-crawler/src/internals/stagehand-controller.ts b/packages/stagehand-crawler/src/internals/stagehand-controller.ts
index a870bdc69..6f344298b 100644
--- a/packages/stagehand-crawler/src/internals/stagehand-controller.ts
+++ b/packages/stagehand-crawler/src/internals/stagehand-controller.ts
@@ -84,7 +84,7 @@ export class StagehandController extends BrowserController<BrowserType, LaunchOp
return {
proxy: {
- server: url.origin,
+ server: `${url.protocol}//${url.host}`,
username,
password,
bypass: pageOptions?.proxy?.bypass,
diff --git a/test/browser-pool/browser-plugins/plugins.test.ts b/test/browser-pool/browser-plugins/plugins.test.ts
index 54ff3cd41..d64c2a296 100644
--- a/test/browser-pool/browser-plugins/plugins.test.ts
+++ b/test/browser-pool/browser-plugins/plugins.test.ts
@@ -356,6 +356,38 @@ describe('Plugins', () => {
runPluginTest(PuppeteerPlugin, PuppeteerController, puppeteer);
+ describe('normalizeProxyOptions', () => {
+ test.each([
+ ['socks5://user:[email protected]:1080', 'socks5://proxy.example.com:1080'],
+ ['http://user:[email protected]:8080', 'http://proxy.example.com:8080'],
+ ])('PlaywrightController should handle %s', (proxyUrl, expectedServer) => {
+ const plugin = new PlaywrightPlugin(playwright.chromium);
+ const browserController = new PlaywrightController(plugin);
+
+ const result = browserController.normalizeProxyOptions(proxyUrl, {});
+
+ expect(result).toMatchObject({
+ proxy: { server: expectedServer, username: 'user', password: 'pass' },
+ });
+ });
+
+ test.each([
+ ['socks5://user:[email protected]:1080', 'socks5://proxy.example.com:1080'],
+ ['http://user:[email protected]:8080', 'http://proxy.example.com:8080'],
+ ])('PuppeteerController should handle %s', (proxyUrl, expectedServer) => {
+ const plugin = new PuppeteerPlugin(puppeteer);
+ const browserController = new PuppeteerController(plugin);
+
+ const result = browserController.normalizeProxyOptions(proxyUrl, {});
+
+ expect(result).toMatchObject({
+ proxyServer: expectedServer,
+ proxyUsername: 'user',
+ proxyPassword: 'pass',
+ });
+ });
+ });
+
describe('Playwright specifics', () => {
let browser: playwright.Browser;