Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: let the adaptive crawler's navigation window be set and extended (#4219)
Muhammad Haadhee Sheeraz Mian7038553c009b
diff --git a/docs/public-api/crawlee-playwright.api.md b/docs/public-api/crawlee-playwright.api.mdindex a6ac56bae..c9151dcdb 100644--- a/docs/public-api/crawlee-playwright.api.md+++ b/docs/public-api/crawlee-playwright.api.md@@ -56,7 +56,7 @@ type AdaptiveHook<ContextExtension = Dictionary<never>> = BrowserHook<AdaptiveHo// Not exported by the entry point; reachable only as a referenced type.// @public (undocumented)-interface AdaptiveHookContext extends Pick<AdaptivePlaywrightCrawlerContext, 'id' | 'session' | 'proxyInfo' | 'log'> {+interface AdaptiveHookContext extends Pick<AdaptivePlaywrightCrawlerContext, 'id' | 'session' | 'proxyInfo' | 'log' | 'extendTimeout'> {// (undocumented)gotoOptions?: PlaywrightGotoOptions;// (undocumented)@@ -97,6 +97,7 @@ export interface AdaptivePlaywrightCrawlerContext<UserData extends Dictionary =// @public (undocumented)export interface AdaptivePlaywrightCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest_2<AdaptivePlaywrightCrawlerContext['request']>>, StatisticStateExtension extends AdaptivePlaywrightCrawlerStatisticState = AdaptivePlaywrightCrawlerStatisticState> extends Omit<BasicCrawlerOptions<AdaptivePlaywrightCrawlerContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>, 'preNavigationHooks' | 'postNavigationHooks' | 'contextPipelineBuilder'>, Pick<PlaywrightCrawlerOptions, 'launchContext' | 'headless' | 'browserPool' | 'remoteBrowser'> {+ navigationTimeoutSecs?: number;postNavigationHooks?: AdaptivePostNavigationHook<ContextExtension>[];preNavigationHooks?: AdaptiveHook<ContextExtension>[];renderingTypeDetectionRatio?: number;diff --git a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.tsindex 5b4723edd..44fd519c9 100644--- a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts+++ b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts@@ -23,6 +23,7 @@ import {ContextPipeline,ContextPipelineInitializationError,ContextPipelineInterruptedError,+ navigationDeadlineKey,RequestHandlerError,resolveBaseUrlForEnqueueLinksFiltering,Router,@@ -152,7 +153,10 @@ export interface AdaptivePlaywrightCrawlerContext<enqueueLinks(options?: EnqueueLinksOptions): Promise<unknown>;}-interface AdaptiveHookContext extends Pick<AdaptivePlaywrightCrawlerContext, 'id' | 'session' | 'proxyInfo' | 'log'> {+interface AdaptiveHookContext extends Pick<+ AdaptivePlaywrightCrawlerContext,+ 'id' | 'session' | 'proxyInfo' | 'log' | 'extendTimeout'+> {page?: Page;request: CrawlingRequest;gotoOptions?: PlaywrightGotoOptions;@@ -192,6 +196,13 @@ export interface AdaptivePlaywrightCrawlerOptions<'preNavigationHooks' | 'postNavigationHooks' | 'contextPipelineBuilder'>,Pick<PlaywrightCrawlerOptions, 'launchContext' | 'headless' | 'browserPool' | 'remoteBrowser'> {+ /**+ * Timeout for the navigation phase of each attempt, in seconds, shared by the `preNavigationHooks`, the+ * navigation and the `postNavigationHooks`. Applies to both the HTTP-only and the browser attempt. When not+ * set, each attempt keeps its own default: 30 seconds for HTTP-only and 60 seconds for the browser.+ */+ navigationTimeoutSecs?: number;+/*** Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies.* The function accepts a subset of the crawling context. If you attempt to access the `page` property during HTTP-only crawling,@@ -377,6 +388,7 @@ export class AdaptivePlaywrightCrawler<postNavigationHooks = [],extendContext,transactionalStorage,+ navigationTimeoutSecs,launchContext,headless,browserPool,@@ -486,6 +498,7 @@ export class AdaptivePlaywrightCrawler<preNavigationHooks,postNavigationHooks,extendContext,+ navigationTimeoutSecs,});const browserCrawler = new PlaywrightCrawler({@@ -494,6 +507,7 @@ export class AdaptivePlaywrightCrawler<preNavigationHooks: preNavigationHooks as unknown as PlaywrightHook[],postNavigationHooks: postNavigationHooks as unknown as PlaywrightHook[],extendContext,+ navigationTimeoutSecs,launchContext,headless,browserPool,@@ -624,17 +638,25 @@ export class AdaptivePlaywrightCrawler<const deferredCleanup: (() => Promise<unknown>)[] = [];+ const subCrawlerContext = Object.defineProperties(+ {},+ Object.getOwnPropertyDescriptors(context),+ ) as typeof context & { [navigationDeadlineKey]?: number };+const attemptBoundContextHelpers = {useState: useStateFunction,log: this.#createLogProxy(context.log, logs),registerDeferredCleanup: (cleanup: () => Promise<unknown>) => deferredCleanup.push(cleanup),+ // The sub-crawler pipeline keeps this attempt's navigation deadline on `subCrawlerContext`, but the+ // outer `extendTimeout` only pushes the deadline it finds on `context`, so push this one as well.+ extendTimeout: (secs: number) => {+ context.extendTimeout(secs);+ if (subCrawlerContext[navigationDeadlineKey] !== undefined) {+ subCrawlerContext[navigationDeadlineKey] += secs * 1000;+ }+ },};- const subCrawlerContext = Object.defineProperties(- {},- Object.getOwnPropertyDescriptors(context),- ) as typeof context;-// Mark attempt-bound helpers as non-configurable so they survive the sub-crawler context pipeline// (which would otherwise override them with the sub-crawler's own versions, losing the binding).for (const [key, descriptor] of Object.entries(Object.getOwnPropertyDescriptors(attemptBoundContextHelpers))) {diff --git a/test/core/crawlers/adaptive-playwright-crawler.test.ts b/test/core/crawlers/adaptive-playwright-crawler.test.tsindex 842ce36f8..b76a0cc48 100644--- a/test/core/crawlers/adaptive-playwright-crawler.test.ts+++ b/test/core/crawlers/adaptive-playwright-crawler.test.ts@@ -87,6 +87,10 @@ describe('AdaptivePlaywrightCrawler', () => {`);});+ app.get('/slow', (_req, res) => {+ setTimeout(() => res.status(200).send('<html><body><h1>Heading</h1></body></html>'), 2000);+ });+app.get('/dynamic', (req, res) => {lastDynamicRequestUserAgent = req.headers['user-agent'];res.status(200);@@ -1161,6 +1165,45 @@ describe('AdaptivePlaywrightCrawler', () => {expect(warningCalled).toBe(true);});+ describe('navigation timeout of the HTTP-only attempt', () => {+ const runSlowStaticRequest = async (options: Partial<AdaptivePlaywrightCrawlerOptions>) => {+ const outcomes: string[] = [];+ const crawler = await makeOneshotCrawler(+ {+ navigationTimeoutSecs: 1,+ renderingTypePredictor: makeRiggedRenderingTypePredictor({+ renderingType: 'static',+ detectionProbabilityRecommendation: 0,+ }),+ // Surface the HTTP-only attempt's error instead of falling back to a browser.+ shouldPropagateError: () => true,+ requestHandler: async () => {+ outcomes.push('handled');+ },+ failedRequestHandler: async (_context, error) => {+ outcomes.push(error.message);+ },+ ...options,+ },+ [`http://${HOSTNAME}:${port}/slow`],+ );+ await crawler.run();+ return outcomes;+ };++ test('honors navigationTimeoutSecs', async () => {+ expect(await runSlowStaticRequest({})).toEqual(['Navigation timed out after 1 seconds.']);+ });++ test('can be extended from a pre-navigation hook', async () => {+ const outcomes = await runSlowStaticRequest({+ preNavigationHooks: [async ({ extendTimeout }) => extendTimeout(5)],+ });++ expect(outcomes).toEqual(['handled']);+ });+ });+test('forwards browser options to the inner PlaywrightCrawler', () => {// `headless` can only collide with `browserPool` in the strict-object validation of the inner// `PlaywrightCrawler` if both options actually reached it - proving the options are wired through.