Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: correctly apply `launchOptions` with `useIncognitoPages` (#3181)

Jindřich Bär84a4b709ee59
diff --git a/packages/browser-pool/src/playwright/playwright-controller.ts b/packages/browser-pool/src/playwright/playwright-controller.ts
index 283968a7e..7dc27f7c4 100644
--- a/packages/browser-pool/src/playwright/playwright-controller.ts
+++ b/packages/browser-pool/src/playwright/playwright-controller.ts
@@ -48,21 +48,29 @@ export class PlaywrightController extends BrowserController<
let close = async () => {};
- if (this.launchContext.useIncognitoPages && contextOptions?.proxy) {
- const [anonymizedProxyUrl, closeProxy] = await anonymizeProxySugar(
- contextOptions.proxy.server,
- contextOptions.proxy.username,
- contextOptions.proxy.password,
- );
+ if (this.launchContext.useIncognitoPages) {
+ // Each page requires to have all the context options applied
+ contextOptions = {
+ ...this.launchContext.launchOptions,
+ ...contextOptions,
+ };
+
+ if (contextOptions?.proxy) {
+ const [anonymizedProxyUrl, closeProxy] = await anonymizeProxySugar(
+ contextOptions.proxy.server,
+ contextOptions.proxy.username,
+ contextOptions.proxy.password,
+ );
- if (anonymizedProxyUrl) {
- contextOptions.proxy = {
- server: anonymizedProxyUrl,
- bypass: contextOptions.proxy.bypass,
- };
- }
+ if (anonymizedProxyUrl) {
+ contextOptions.proxy = {
+ server: anonymizedProxyUrl,
+ bypass: contextOptions.proxy.bypass,
+ };
+ }
- close = closeProxy;
+ close = closeProxy;
+ }
}
try {
diff --git a/test/core/crawlers/playwright_crawler.test.ts b/test/core/crawlers/playwright_crawler.test.ts
index 789dd4995..d4da700cb 100644
--- a/test/core/crawlers/playwright_crawler.test.ts
+++ b/test/core/crawlers/playwright_crawler.test.ts
@@ -153,4 +153,48 @@ describe('PlaywrightCrawler', () => {
expect(Object.keys(options.browserPoolOptions).length).toBe(0);
});
+
+ test.each([{ useIncognitoPages: true }, { useIncognitoPages: false }])(
+ 'should apply launchOptions with useIncognitoPages: $useIncognitoPages',
+ async ({ useIncognitoPages }) => {
+ // Some launch options apply to the browser, while some apply to the context.
+ // Here we use some context options to verify that those are actually applied.
+ const launchOptions = {
+ locale: 'cz-CZ',
+ reducedMotion: 'reduce' as const,
+ timezoneId: 'Pacific/Tahiti',
+ };
+
+ let [timezone, locale, reducedMotion] = ['', '', ''];
+
+ const playwrightCrawler = new PlaywrightCrawler({
+ maxConcurrency: 1,
+ launchContext: {
+ useIncognitoPages,
+ launchOptions,
+ },
+ browserPoolOptions: {
+ // don't overwrite locale with fingerprint's locale
+ useFingerprints: false,
+ },
+ requestHandler: async ({ page }) => {
+ [timezone, locale, reducedMotion] = await Promise.all([
+ page.evaluate(() => Intl.DateTimeFormat().resolvedOptions().timeZone),
+ page.evaluate(() => navigator.language),
+ page.evaluate(() => {
+ return window.matchMedia('(prefers-reduced-motion: reduce)').matches
+ ? 'reduce'
+ : 'no-preference';
+ }),
+ ]);
+ },
+ });
+
+ await playwrightCrawler.run([`http://${HOSTNAME}:${port}/`]);
+
+ expect(timezone).toBe(launchOptions.timezoneId);
+ expect(locale).toBe(launchOptions.locale);
+ expect(reducedMotion).toBe(launchOptions.reducedMotion);
+ },
+ );
});