Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix(playwright): wire browser options through AdaptivePlaywrightCrawler (#4088)
Jindřich Bär6cc242028e66
diff --git a/docs/public-api/crawlee-playwright.api.md b/docs/public-api/crawlee-playwright.api.mdindex 8aac6b5d7..d1e3ed444 100644--- a/docs/public-api/crawlee-playwright.api.md+++ b/docs/public-api/crawlee-playwright.api.md@@ -97,7 +97,7 @@ export interface AdaptivePlaywrightCrawlerContext<UserData extends Dictionary =}// @public (undocumented)-export interface AdaptivePlaywrightCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest_2<AdaptivePlaywrightCrawlerContext['request']>>, StatisticStateExtension extends AdaptivePlaywrightCrawlerStatisticState = AdaptivePlaywrightCrawlerStatisticState> extends Omit<BasicCrawlerOptions<AdaptivePlaywrightCrawlerContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>, 'preNavigationHooks' | 'postNavigationHooks'> {+export interface AdaptivePlaywrightCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest_2<AdaptivePlaywrightCrawlerContext['request']>>, StatisticStateExtension extends AdaptivePlaywrightCrawlerStatisticState = AdaptivePlaywrightCrawlerStatisticState> extends Omit<BasicCrawlerOptions<AdaptivePlaywrightCrawlerContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>, 'preNavigationHooks' | 'postNavigationHooks'>, Pick<PlaywrightCrawlerOptions, 'launchContext' | 'headless' | 'browserPool' | 'remoteBrowser'> {postNavigationHooks?: AdaptivePostNavigationHook<ContextExtension>[];preNavigationHooks?: AdaptiveHook<ContextExtension>[];renderingTypeDetectionRatio?: number;diff --git a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.tsindex 6d8d56d0f..10f17c677 100644--- a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts+++ b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts@@ -45,7 +45,12 @@ import { z } from 'zod';import { addTimeoutToPromise } from '@apify/timeout';-import type { PlaywrightCrawlingContext, PlaywrightGotoOptions, PlaywrightHook } from './playwright-crawler.js';+import type {+ PlaywrightCrawlerOptions,+ PlaywrightCrawlingContext,+ PlaywrightGotoOptions,+ PlaywrightHook,+} from './playwright-crawler.js';import { PlaywrightCrawler } from './playwright-crawler.js';import {type IRenderingTypePredictor,@@ -163,16 +168,19 @@ export interface AdaptivePlaywrightCrawlerOptions<GetUserDataFromRequest<AdaptivePlaywrightCrawlerContext['request']>>,StatisticStateExtension extends AdaptivePlaywrightCrawlerStatisticState = AdaptivePlaywrightCrawlerStatisticState,-> extends Omit<- BasicCrawlerOptions<- AdaptivePlaywrightCrawlerContext,- ContextExtension,- ExtendedContext,- Routes,- StatisticStateExtension- >,- 'preNavigationHooks' | 'postNavigationHooks'-> {+>+ extends+ Omit<+ BasicCrawlerOptions<+ AdaptivePlaywrightCrawlerContext,+ ContextExtension,+ ExtendedContext,+ Routes,+ StatisticStateExtension+ >,+ 'preNavigationHooks' | 'postNavigationHooks'+ >,+ Pick<PlaywrightCrawlerOptions, 'launchContext' | 'headless' | 'browserPool' | 'remoteBrowser'> {/*** Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies.* The function accepts a subset of the crawling context. If you attempt to access the `page` property during HTTP-only crawling,@@ -340,6 +348,10 @@ export class AdaptivePlaywrightCrawler<extendContext,contextPipelineBuilder,transactionalStorage,+ launchContext,+ headless,+ browserPool,+ remoteBrowser,...rest} = options;@@ -452,6 +464,10 @@ export class AdaptivePlaywrightCrawler<preNavigationHooks: preNavigationHooks as unknown as PlaywrightHook[],postNavigationHooks: postNavigationHooks as unknown as PlaywrightHook[],extendContext,+ launchContext,+ headless,+ browserPool,+ remoteBrowser,});this.#teardownHooks.push(browserCrawler.teardown.bind(browserCrawler));diff --git a/test/core/crawlers/adaptive_playwright_crawler.test.ts b/test/core/crawlers/adaptive_playwright_crawler.test.tsindex bc12ffd7d..d5cd99677 100644--- a/test/core/crawlers/adaptive_playwright_crawler.test.ts+++ b/test/core/crawlers/adaptive_playwright_crawler.test.ts@@ -24,6 +24,7 @@ import {BasicCrawler,createAdaptivePlaywrightRouter,fullResultComparator,+ playwrightBrowserPool,RenderingTypePredictor,RequestList,RequestQueue,@@ -57,6 +58,7 @@ describe('AdaptivePlaywrightCrawler', () => {const HOSTNAME = '127.0.0.1';let port: number;let server: Server;+ let lastDynamicRequestUserAgent: string | undefined;beforeAll(async () => {const app = express();@@ -82,7 +84,8 @@ describe('AdaptivePlaywrightCrawler', () => {`);});- app.get('/dynamic', (_req, res) => {+ app.get('/dynamic', (req, res) => {+ lastDynamicRequestUserAgent = req.headers['user-agent'];res.status(200);res.send(`<html>@@ -145,6 +148,7 @@ describe('AdaptivePlaywrightCrawler', () => {// reset storage, and the crawler restores a stale handled-request count and processes nothing.// @ts-expect-error Reset private static instance counter for test isolationBasicCrawler.instanceCount = 0;+ lastDynamicRequestUserAgent = undefined;});// Test setup helpers@@ -989,4 +993,34 @@ describe('AdaptivePlaywrightCrawler', () => {await crawler.run();expect(warningCalled).toBe(true);});++ test('forwards browser options to the inner PlaywrightCrawler', () => {+ // `headless` can only collide with `browserPool` in the strict-object validation of the inner+ // `PlaywrightCrawler` if both options actually reached it - proving the options are wired through.+ expect(() => new AdaptivePlaywrightCrawler({ browserPool: playwrightBrowserPool(), headless: false })).toThrow(+ 'PlaywrightCrawler: `headless` cannot be combined with `browserPool`',+ );+ });++ test('launchContext reaches the browser launched for the browser-rendering path', async () => {+ const renderingTypePredictor = makeRiggedRenderingTypePredictor({+ renderingType: 'clientOnly',+ detectionProbabilityRecommendation: 0,+ });++ const distinctiveUserAgent = 'CrawleeAdaptiveBrowserOptionsTest/1.0';++ const crawler = await makeOneshotCrawler(+ {+ requestHandler: async () => {},+ renderingTypePredictor,+ launchContext: { userAgent: distinctiveUserAgent },+ },+ [`http://${HOSTNAME}:${port}/dynamic`],+ );++ await crawler.run();++ expect(lastDynamicRequestUserAgent).toBe(distinctiveUserAgent);+ });});