Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: shallow clone browserPoolOptions before normalization (#1665)
Szymon Marczak22467ca81ad9
diff --git a/packages/playwright-crawler/src/internals/playwright-crawler.ts b/packages/playwright-crawler/src/internals/playwright-crawler.tsindex d1d152534..aaa676993 100644--- a/packages/playwright-crawler/src/internals/playwright-crawler.ts+++ b/packages/playwright-crawler/src/internals/playwright-crawler.ts@@ -192,10 +192,13 @@ export class PlaywrightCrawler extends BrowserCrawler<{ browserPlugins: [Playwriconst {launchContext = {},headless,- browserPoolOptions = {} as BrowserPoolOptions<PlaywrightPlugin>,...browserCrawlerOptions} = options;+ const browserPoolOptions = {+ ...options.browserPoolOptions,+ } as BrowserPoolOptions;+if (launchContext.proxyUrl) {throw new Error('PlaywrightCrawlerOptions.launchContext.proxyUrl is not allowed in PlaywrightCrawler.'+ 'Use PlaywrightCrawlerOptions.proxyConfiguration');diff --git a/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts b/packages/puppeteer-crawler/src/internals/puppeteer-crawler.tsindex 1edac4a73..c861afdad 100644--- a/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts+++ b/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts@@ -141,11 +141,14 @@ export class PuppeteerCrawler extends BrowserCrawler<{ browserPlugins: [Puppeteeconst {launchContext = {},headless,- browserPoolOptions = {} as BrowserPoolOptions,proxyConfiguration,...browserCrawlerOptions} = options;+ const browserPoolOptions = {+ ...options.browserPoolOptions,+ } as BrowserPoolOptions;+if (launchContext.proxyUrl) {throw new Error('PuppeteerCrawlerOptions.launchContext.proxyUrl is not allowed in PuppeteerCrawler.'+ 'Use PuppeteerCrawlerOptions.proxyConfiguration');diff --git a/test/core/crawlers/playwright_crawler.test.ts b/test/core/crawlers/playwright_crawler.test.tsindex 8e2efc754..e323f4339 100644--- a/test/core/crawlers/playwright_crawler.test.ts+++ b/test/core/crawlers/playwright_crawler.test.ts@@ -133,4 +133,16 @@ describe('PlaywrightCrawler', () => {await playwrightCrawler.run();expect(options.timeout).toEqual(timeoutSecs * 1000);});++ test('shallow clones browserPoolOptions before normalization', () => {+ const options = {+ browserPoolOptions: {},+ requestHandler: async () => {},+ };++ void new PlaywrightCrawler(options);+ void new PlaywrightCrawler(options);++ expect(Object.keys(options.browserPoolOptions).length).toBe(0);+ });});diff --git a/test/core/crawlers/puppeteer_crawler.test.ts b/test/core/crawlers/puppeteer_crawler.test.tsindex ef963d0cd..693867964 100644--- a/test/core/crawlers/puppeteer_crawler.test.ts+++ b/test/core/crawlers/puppeteer_crawler.test.ts@@ -399,6 +399,18 @@ describe('PuppeteerCrawler', () => {expect(sessions.size).toBe(3); // 3 different sessions used});+ test('shallow clones browserPoolOptions before normalization', () => {+ const options = {+ browserPoolOptions: {},+ requestHandler: async () => {},+ };++ void new PuppeteerCrawler(options);+ void new PuppeteerCrawler(options);++ expect(Object.keys(options.browserPoolOptions).length).toBe(0);+ });+if (os.platform() !== 'darwin') {test('proxy per page', async () => {const requestListLarge = await RequestList.open({