Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: correctly apply `launchOptions` with `useIncognitoPages` (#3181)
Jindřich Bär84a4b709ee59
diff --git a/packages/browser-pool/src/playwright/playwright-controller.ts b/packages/browser-pool/src/playwright/playwright-controller.tsindex 283968a7e..7dc27f7c4 100644--- a/packages/browser-pool/src/playwright/playwright-controller.ts+++ b/packages/browser-pool/src/playwright/playwright-controller.ts@@ -48,21 +48,29 @@ export class PlaywrightController extends BrowserController<let close = async () => {};- if (this.launchContext.useIncognitoPages && contextOptions?.proxy) {- const [anonymizedProxyUrl, closeProxy] = await anonymizeProxySugar(- contextOptions.proxy.server,- contextOptions.proxy.username,- contextOptions.proxy.password,- );+ if (this.launchContext.useIncognitoPages) {+ // Each page requires to have all the context options applied+ contextOptions = {+ ...this.launchContext.launchOptions,+ ...contextOptions,+ };++ if (contextOptions?.proxy) {+ const [anonymizedProxyUrl, closeProxy] = await anonymizeProxySugar(+ contextOptions.proxy.server,+ contextOptions.proxy.username,+ contextOptions.proxy.password,+ );- if (anonymizedProxyUrl) {- contextOptions.proxy = {- server: anonymizedProxyUrl,- bypass: contextOptions.proxy.bypass,- };- }+ if (anonymizedProxyUrl) {+ contextOptions.proxy = {+ server: anonymizedProxyUrl,+ bypass: contextOptions.proxy.bypass,+ };+ }- close = closeProxy;+ close = closeProxy;+ }}try {diff --git a/test/core/crawlers/playwright_crawler.test.ts b/test/core/crawlers/playwright_crawler.test.tsindex 789dd4995..d4da700cb 100644--- a/test/core/crawlers/playwright_crawler.test.ts+++ b/test/core/crawlers/playwright_crawler.test.ts@@ -153,4 +153,48 @@ describe('PlaywrightCrawler', () => {expect(Object.keys(options.browserPoolOptions).length).toBe(0);});++ test.each([{ useIncognitoPages: true }, { useIncognitoPages: false }])(+ 'should apply launchOptions with useIncognitoPages: $useIncognitoPages',+ async ({ useIncognitoPages }) => {+ // Some launch options apply to the browser, while some apply to the context.+ // Here we use some context options to verify that those are actually applied.+ const launchOptions = {+ locale: 'cz-CZ',+ reducedMotion: 'reduce' as const,+ timezoneId: 'Pacific/Tahiti',+ };++ let [timezone, locale, reducedMotion] = ['', '', ''];++ const playwrightCrawler = new PlaywrightCrawler({+ maxConcurrency: 1,+ launchContext: {+ useIncognitoPages,+ launchOptions,+ },+ browserPoolOptions: {+ // don't overwrite locale with fingerprint's locale+ useFingerprints: false,+ },+ requestHandler: async ({ page }) => {+ [timezone, locale, reducedMotion] = await Promise.all([+ page.evaluate(() => Intl.DateTimeFormat().resolvedOptions().timeZone),+ page.evaluate(() => navigator.language),+ page.evaluate(() => {+ return window.matchMedia('(prefers-reduced-motion: reduce)').matches+ ? 'reduce'+ : 'no-preference';+ }),+ ]);+ },+ });++ await playwrightCrawler.run([`http://${HOSTNAME}:${port}/`]);++ expect(timezone).toBe(launchOptions.timezoneId);+ expect(locale).toBe(launchOptions.locale);+ expect(reducedMotion).toBe(launchOptions.reducedMotion);+ },+ );});