Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: shallow clone browserPoolOptions before normalization (#1665)

Szymon Marczak22467ca81ad9
diff --git a/packages/playwright-crawler/src/internals/playwright-crawler.ts b/packages/playwright-crawler/src/internals/playwright-crawler.ts
index d1d152534..aaa676993 100644
--- a/packages/playwright-crawler/src/internals/playwright-crawler.ts
+++ b/packages/playwright-crawler/src/internals/playwright-crawler.ts
@@ -192,10 +192,13 @@ export class PlaywrightCrawler extends BrowserCrawler<{ browserPlugins: [Playwri
const {
launchContext = {},
headless,
- browserPoolOptions = {} as BrowserPoolOptions<PlaywrightPlugin>,
...browserCrawlerOptions
} = options;
+ const browserPoolOptions = {
+ ...options.browserPoolOptions,
+ } as BrowserPoolOptions;
+
if (launchContext.proxyUrl) {
throw new Error('PlaywrightCrawlerOptions.launchContext.proxyUrl is not allowed in PlaywrightCrawler.'
+ 'Use PlaywrightCrawlerOptions.proxyConfiguration');
diff --git a/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts b/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts
index 1edac4a73..c861afdad 100644
--- a/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts
+++ b/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts
@@ -141,11 +141,14 @@ export class PuppeteerCrawler extends BrowserCrawler<{ browserPlugins: [Puppetee
const {
launchContext = {},
headless,
- browserPoolOptions = {} as BrowserPoolOptions,
proxyConfiguration,
...browserCrawlerOptions
} = options;
+ const browserPoolOptions = {
+ ...options.browserPoolOptions,
+ } as BrowserPoolOptions;
+
if (launchContext.proxyUrl) {
throw new Error('PuppeteerCrawlerOptions.launchContext.proxyUrl is not allowed in PuppeteerCrawler.'
+ 'Use PuppeteerCrawlerOptions.proxyConfiguration');
diff --git a/test/core/crawlers/playwright_crawler.test.ts b/test/core/crawlers/playwright_crawler.test.ts
index 8e2efc754..e323f4339 100644
--- a/test/core/crawlers/playwright_crawler.test.ts
+++ b/test/core/crawlers/playwright_crawler.test.ts
@@ -133,4 +133,16 @@ describe('PlaywrightCrawler', () => {
await playwrightCrawler.run();
expect(options.timeout).toEqual(timeoutSecs * 1000);
});
+
+ test('shallow clones browserPoolOptions before normalization', () => {
+ const options = {
+ browserPoolOptions: {},
+ requestHandler: async () => {},
+ };
+
+ void new PlaywrightCrawler(options);
+ void new PlaywrightCrawler(options);
+
+ expect(Object.keys(options.browserPoolOptions).length).toBe(0);
+ });
});
diff --git a/test/core/crawlers/puppeteer_crawler.test.ts b/test/core/crawlers/puppeteer_crawler.test.ts
index ef963d0cd..693867964 100644
--- a/test/core/crawlers/puppeteer_crawler.test.ts
+++ b/test/core/crawlers/puppeteer_crawler.test.ts
@@ -399,6 +399,18 @@ describe('PuppeteerCrawler', () => {
expect(sessions.size).toBe(3); // 3 different sessions used
});
+ test('shallow clones browserPoolOptions before normalization', () => {
+ const options = {
+ browserPoolOptions: {},
+ requestHandler: async () => {},
+ };
+
+ void new PuppeteerCrawler(options);
+ void new PuppeteerCrawler(options);
+
+ expect(Object.keys(options.browserPoolOptions).length).toBe(0);
+ });
+
if (os.platform() !== 'darwin') {
test('proxy per page', async () => {
const requestListLarge = await RequestList.open({