Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: Fix trailing slash removal in BrowserPool (#2921)

Jan Bucharc1fc439e8e9c
diff --git a/packages/browser-pool/src/anonymize-proxy.ts b/packages/browser-pool/src/anonymize-proxy.ts
index 8c98d3a01..31e87baff 100644
--- a/packages/browser-pool/src/anonymize-proxy.ts
+++ b/packages/browser-pool/src/anonymize-proxy.ts
@@ -16,7 +16,9 @@ export const anonymizeProxySugar = async (
}
if (url.username || url.password) {
- const anonymized = await anonymizeProxy(url.href.slice(0, -1));
+ // trim off trailing slash if it's present
+ const proxyUrlString = url.href.endsWith('/') ? url.href.slice(0, -1) : url.href;
+ const anonymized = await anonymizeProxy(proxyUrlString);
return [
anonymized,
diff --git a/packages/browser-pool/test/tsconfig.json b/packages/browser-pool/test/tsconfig.json
index 52d7cd5d4..64962b1ce 100644
--- a/packages/browser-pool/test/tsconfig.json
+++ b/packages/browser-pool/test/tsconfig.json
@@ -4,6 +4,5 @@
"noEmit": true,
"incremental": false,
"types": ["vitest/globals"]
- },
- "include": ["./"]
+ }
}
diff --git a/test/browser-pool/anonymize-proxy-sugar.test.ts b/test/browser-pool/anonymize-proxy-sugar.test.ts
new file mode 100644
index 000000000..c4a08cd96
--- /dev/null
+++ b/test/browser-pool/anonymize-proxy-sugar.test.ts
@@ -0,0 +1,32 @@
+import { anonymizeProxy } from 'proxy-chain';
+import { vi } from 'vitest';
+
+import { anonymizeProxySugar } from '../../packages/browser-pool/src/anonymize-proxy';
+
+describe('anonymizeProxySugar', () => {
+ // Mock the anonymizeProxy function from proxy-chain
+ beforeEach(() => {
+ vi.mock('proxy-chain', () => ({
+ anonymizeProxy: vi.fn((url) => Promise.resolve(`anonymized-${url}`)),
+ }));
+ });
+
+ afterEach(() => {
+ vi.clearAllMocks();
+ });
+
+ test.each([
+ ['http://username:password@proxy:1000', 'http://username:password@proxy:1000'],
+ ['http://username:password@proxy:1000/', 'http://username:password@proxy:1000'],
+ ['socks://username:password@proxy:1000', 'socks://username:password@proxy:1000'],
+ ['socks://username:password@proxy:1000/', 'socks://username:password@proxy:1000'],
+ ])(
+ 'should call anonymizeProxy from proxy-chain with correctly pre-processed URL: %s',
+ async (input, expectedOutput) => {
+ const [anonymized] = await anonymizeProxySugar(input);
+
+ expect(anonymizeProxy).toHaveBeenCalledWith(expectedOutput);
+ expect(anonymized).toBeTypeOf('string');
+ },
+ );
+});