Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: Fix trailing slash removal in BrowserPool (#2921)
Jan Bucharc1fc439e8e9c
diff --git a/packages/browser-pool/src/anonymize-proxy.ts b/packages/browser-pool/src/anonymize-proxy.tsindex 8c98d3a01..31e87baff 100644--- a/packages/browser-pool/src/anonymize-proxy.ts+++ b/packages/browser-pool/src/anonymize-proxy.ts@@ -16,7 +16,9 @@ export const anonymizeProxySugar = async (}if (url.username || url.password) {- const anonymized = await anonymizeProxy(url.href.slice(0, -1));+ // trim off trailing slash if it's present+ const proxyUrlString = url.href.endsWith('/') ? url.href.slice(0, -1) : url.href;+ const anonymized = await anonymizeProxy(proxyUrlString);return [anonymized,diff --git a/packages/browser-pool/test/tsconfig.json b/packages/browser-pool/test/tsconfig.jsonindex 52d7cd5d4..64962b1ce 100644--- a/packages/browser-pool/test/tsconfig.json+++ b/packages/browser-pool/test/tsconfig.json@@ -4,6 +4,5 @@"noEmit": true,"incremental": false,"types": ["vitest/globals"]- },- "include": ["./"]+ }}diff --git a/test/browser-pool/anonymize-proxy-sugar.test.ts b/test/browser-pool/anonymize-proxy-sugar.test.tsnew file mode 100644index 000000000..c4a08cd96--- /dev/null+++ b/test/browser-pool/anonymize-proxy-sugar.test.ts@@ -0,0 +1,32 @@+import { anonymizeProxy } from 'proxy-chain';+import { vi } from 'vitest';++import { anonymizeProxySugar } from '../../packages/browser-pool/src/anonymize-proxy';++describe('anonymizeProxySugar', () => {+ // Mock the anonymizeProxy function from proxy-chain+ beforeEach(() => {+ vi.mock('proxy-chain', () => ({+ anonymizeProxy: vi.fn((url) => Promise.resolve(`anonymized-${url}`)),+ }));+ });++ afterEach(() => {+ vi.clearAllMocks();+ });++ test.each([+ ['http://username:password@proxy:1000', 'http://username:password@proxy:1000'],+ ['http://username:password@proxy:1000/', 'http://username:password@proxy:1000'],+ ['socks://username:password@proxy:1000', 'socks://username:password@proxy:1000'],+ ['socks://username:password@proxy:1000/', 'socks://username:password@proxy:1000'],+ ])(+ 'should call anonymizeProxy from proxy-chain with correctly pre-processed URL: %s',+ async (input, expectedOutput) => {+ const [anonymized] = await anonymizeProxySugar(input);++ expect(anonymizeProxy).toHaveBeenCalledWith(expectedOutput);+ expect(anonymized).toBeTypeOf('string');+ },+ );+});