Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: remove trailing slash for proxyUrl (#1405)

Andrey Bykovb57ad763c8db
diff --git a/packages/browser-pool/src/abstract-classes/browser-plugin.ts b/packages/browser-pool/src/abstract-classes/browser-plugin.ts
index 8e3708ca3..18774e36c 100644
--- a/packages/browser-pool/src/abstract-classes/browser-plugin.ts
+++ b/packages/browser-pool/src/abstract-classes/browser-plugin.ts
@@ -109,7 +109,7 @@ export abstract class BrowserPlugin<
this.library = library;
this.launchOptions = launchOptions;
- this.proxyUrl = proxyUrl && new URL(proxyUrl).href;
+ this.proxyUrl = proxyUrl && new URL(proxyUrl).href.slice(0, -1);
this.userDataDir = userDataDir;
this.useIncognitoPages = useIncognitoPages;
}
diff --git a/packages/browser-pool/src/anonymize-proxy.ts b/packages/browser-pool/src/anonymize-proxy.ts
index 3a3cde58f..73c80dec9 100644
--- a/packages/browser-pool/src/anonymize-proxy.ts
+++ b/packages/browser-pool/src/anonymize-proxy.ts
@@ -16,7 +16,7 @@ export const anonymizeProxySugar = async (
}
if (url.username || url.password) {
- const anonymized = await anonymizeProxy(url.href);
+ const anonymized = await anonymizeProxy(url.href.slice(0, -1));
return [
anonymized,
diff --git a/test/browser-pool/browser-plugins/plugins.test.ts b/test/browser-pool/browser-plugins/plugins.test.ts
index e61a14de7..329addac6 100644
--- a/test/browser-pool/browser-plugins/plugins.test.ts
+++ b/test/browser-pool/browser-plugins/plugins.test.ts
@@ -85,7 +85,7 @@ const runPluginTest = <
const context = plugin.createLaunchContext();
- expect(context.proxyUrl).toEqual(proxyUrl);
+ expect(context.proxyUrl).toEqual(proxyUrl.slice(0, -1));
expect(context.useIncognitoPages).toBeTruthy();
expect(context.userDataDir).toEqual('test');
});