Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: Run extendContext before navigation hooks (#3908)
Jan Bucharaeee91d60855
diff --git a/docs/public-api/crawlee-basic.api.md b/docs/public-api/crawlee-basic.api.mdindex 4273d6438..1c4e81882 100644--- a/docs/public-api/crawlee-basic.api.md+++ b/docs/public-api/crawlee-basic.api.md@@ -21,7 +21,7 @@ import { CheerioRoot } from '@crawlee/utils';import type { Configuration } from '@crawlee/core';import { ContextPipeline } from '@crawlee/core';import type { CrawleeLogger } from '@crawlee/core';-import type { CrawlingContext } from '@crawlee/core';+import { CrawlingContext } from '@crawlee/core';import { Dataset } from '@crawlee/core';import type { DatasetExportOptions } from '@crawlee/core';import type { Dictionary } from '@crawlee/types';@@ -186,7 +186,7 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingCcontextPipelineBuilder?: () => ContextPipeline<CrawlingContext, Context>;errorHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;eventManager?: EventManager;- extendContext?: (context: Context) => Awaitable<ContextExtension>;+ extendContext?: (context: CrawlingContext) => Awaitable<ContextExtension>;failedRequestHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;httpClient?: BaseHttpClient;id?: string;diff --git a/docs/public-api/crawlee-http.api.md b/docs/public-api/crawlee-http.api.mdindex 90843c2cd..2917e5cff 100644--- a/docs/public-api/crawlee-http.api.md+++ b/docs/public-api/crawlee-http.api.md@@ -151,8 +151,8 @@ export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContextforceResponseEncoding?: string;ignoreSslErrors?: boolean;navigationTimeoutSecs?: number;- postNavigationHooks?: ((crawlingContext: CrawlingContextWithResponse) => Awaitable<void | Partial<CrawlingContextWithResponse>>)[];- preNavigationHooks?: InternalHttpHook<CrawlingContext>[];+ postNavigationHooks?: ((crawlingContext: CrawlingContextWithResponse & ContextExtension) => Awaitable<void | Partial<CrawlingContextWithResponse>>)[];+ preNavigationHooks?: InternalHttpHook<CrawlingContext, ContextExtension>[];saveResponseCookies?: boolean;suggestResponseEncoding?: string;}diff --git a/docs/public-api/crawlee-playwright.api.md b/docs/public-api/crawlee-playwright.api.mdindex 5f1a3c4fa..26d1d019c 100644--- a/docs/public-api/crawlee-playwright.api.md+++ b/docs/public-api/crawlee-playwright.api.md@@ -65,8 +65,8 @@ import type { StatisticState } from '@crawlee/core';import { StringPredicate } from 'ow';// @public-export class AdaptivePlaywrightCrawler<ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext> extends BasicCrawler<AdaptivePlaywrightCrawlerContext, ExtendedContext> {- constructor(options?: AdaptivePlaywrightCrawlerOptions<ExtendedContext>);+export class AdaptivePlaywrightCrawler<ContextExtension = Dictionary_2<never>, ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext & ContextExtension> extends BasicCrawler<AdaptivePlaywrightCrawlerContext, ContextExtension, ExtendedContext> {+ constructor(options?: AdaptivePlaywrightCrawlerOptions<ContextExtension, ExtendedContext>);// (undocumented)protected buildContextPipeline(): ContextPipeline_2<CrawlingContext_2<Dictionary_2>, CrawlingContext_2<Dictionary_2> & {readonly request: LoadedRequest<Request_3<Dictionary_2>>;@@ -103,9 +103,9 @@ export interface AdaptivePlaywrightCrawlerContext<UserData extends Dictionary_2}// @public (undocumented)-export interface AdaptivePlaywrightCrawlerOptions<ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext> extends Omit<BasicCrawlerOptions<AdaptivePlaywrightCrawlerContext, ExtendedContext>, 'preNavigationHooks' | 'postNavigationHooks'> {- postNavigationHooks?: AdaptivePostNavigationHook[];- preNavigationHooks?: AdaptiveHook[];+export interface AdaptivePlaywrightCrawlerOptions<ContextExtension = Dictionary_2<never>, ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext & ContextExtension> extends Omit<BasicCrawlerOptions<AdaptivePlaywrightCrawlerContext, ContextExtension, ExtendedContext>, 'preNavigationHooks' | 'postNavigationHooks'> {+ postNavigationHooks?: AdaptivePostNavigationHook<ContextExtension>[];+ preNavigationHooks?: AdaptiveHook<ContextExtension>[];preventDirectStorageAccess?: boolean;renderingTypeDetectionRatio?: number;renderingTypePredictor?: Pick<RenderingTypePredictor, 'predict' | 'storeResult' | 'initialize'>;@@ -259,7 +259,7 @@ declare namespace playwrightClickElements {export class PlaywrightCrawler<ContextExtension = Dictionary_2<never>, ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension> extends BrowserCrawler<Page, Response_2, {browserPlugins: [PlaywrightPlugin];}, LaunchOptions, PlaywrightCrawlingContext, ContextExtension, ExtendedContext> {- constructor(options?: PlaywrightCrawlerOptions<ExtendedContext>);+ constructor(options?: PlaywrightCrawlerOptions<ContextExtension, ExtendedContext>);// (undocumented)protected buildContextPipeline(): ContextPipeline<CrawlingContext<Dictionary_2>, BrowserCrawlingContext<Page, Response_2, Dictionary_2, Dictionary_2> & {injectFile: (filePath: string, options?: InjectFileOptions) => Promise<unknown>;@@ -333,8 +333,8 @@ export interface PlaywrightCrawlerOptions<ContextExtension = Dictionary_2<never>browserPlugins: [PlaywrightPlugin];}> {launchContext?: PlaywrightLaunchContext;- postNavigationHooks?: PlaywrightHook[];- preNavigationHooks?: PlaywrightHook[];+ postNavigationHooks?: BrowserHook<PlaywrightCrawlingContext, ContextExtension>[];+ preNavigationHooks?: BrowserHook<PlaywrightCrawlingContext, ContextExtension>[];requestHandler?: RequestHandler<ExtendedContext>;}diff --git a/docs/public-api/crawlee-puppeteer.api.md b/docs/public-api/crawlee-puppeteer.api.mdindex 8b5463033..e0e1df842 100644--- a/docs/public-api/crawlee-puppeteer.api.md+++ b/docs/public-api/crawlee-puppeteer.api.md@@ -250,8 +250,8 @@ export interface PuppeteerCrawlerOptions<ContextExtension = Dictionary<never>, EbrowserPlugins: [PuppeteerPlugin];}> {launchContext?: PuppeteerLaunchContext;- postNavigationHooks?: PuppeteerHook[];- preNavigationHooks?: PuppeteerHook[];+ postNavigationHooks?: BrowserHook<PuppeteerCrawlingContext, ContextExtension>[];+ preNavigationHooks?: BrowserHook<PuppeteerCrawlingContext, ContextExtension>[];}// @public (undocumented)diff --git a/docs/upgrading/upgrading_v4.md b/docs/upgrading/upgrading_v4.mdindex 6b9d24969..c7475d9c4 100644--- a/docs/upgrading/upgrading_v4.md+++ b/docs/upgrading/upgrading_v4.md@@ -124,6 +124,8 @@ const crawler = new CheerioCrawler({})```+`extendContext` runs **before navigation**, so the members it returns are visible to the `preNavigationHooks`, `postNavigationHooks`, and the `requestHandler` alike. As a consequence, the `context` passed to `extendContext` is the pre-navigation context and does **not** include navigation-dependent members (e.g. `page`, `response`, `$`, `body`). If your extension needs to read those, do it in a `postNavigationHook` or the `requestHandler` instead.+## Crawling context is strictly typedPreviously, the crawling context extended a `Record` type, allowing to access any property. This was changed to a strict type, which means that you can only access properties that are defined in the context.diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex 726c743a7..5c4283ac9 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -186,7 +186,13 @@ export interface BasicCrawlerOptions<requestHandler?: RequestHandler<ExtendedContext>;/**- * Allows the user to extend the crawling context passed to the request handler with custom functionality.+ * Allows the user to extend the crawling context with custom functionality (helpers, references, etc.).+ *+ * `extendContext` runs *before* navigation, so the returned members are visible to the+ * `preNavigationHooks`, `postNavigationHooks`, and the `requestHandler` alike. As a consequence,+ * the `context` passed to `extendContext` is the pre-navigation {@apilink CrawlingContext} and does+ * **not** include navigation-dependent members (e.g. `page`, `response`, `$`, `body`). If you need+ * those, use a `postNavigationHook` or the `requestHandler` instead.** **Example usage:***@@ -206,7 +212,7 @@ export interface BasicCrawlerOptions<* });* ```*/- extendContext?: (context: Context) => Awaitable<ContextExtension>;+ extendContext?: (context: CrawlingContext) => Awaitable<ContextExtension>;/*** *Intended for BasicCrawler subclasses*. Prepares a context pipeline that transforms the initial crawling context into the shape given by the `Context` type parameter.@@ -708,7 +714,7 @@ export class BasicCrawler<protected readonly identity: CrawlerIdentity;private readonly contextPipelineOptions: {contextPipelineBuilder?: () => ContextPipeline<CrawlingContext, Context>;- extendContext?: (context: Context) => Awaitable<ContextExtension>;+ extendContext?: (context: CrawlingContext) => Awaitable<ContextExtension>;};protected static optionsShape = {@@ -1209,14 +1215,32 @@ export class BasicCrawler<}private buildFinalContextPipeline(): ContextPipeline<CrawlingContext, ExtendedContext> {- let contextPipeline = (this.contextPipelineOptions.contextPipelineBuilder?.() ??+ const subclassPipeline = (this.contextPipelineOptions.contextPipelineBuilder?.() ??this.buildContextPipeline()) as ContextPipeline<CrawlingContext, Context>;+ // `extendContext` runs *before* the subclass navigation pipeline (which includes the+ // pre/post-navigation hooks). This makes the extension visible to those hooks and to the+ // request handler alike. The trade-off is that `extendContext` cannot access+ // navigation-dependent context members (e.g. `page`, `response`, `$`, `body`), as those+ // don't exist yet at this point in the pipeline.+ // The `extendContext` output (`ContextExtension`) is carried through the subclass pipeline at+ // runtime (the pipeline copies each middleware's returned members onto the shared context), but+ // TypeScript cannot express that `Context` transitively includes `ContextExtension` here. The+ // casts below are sound because `buildFinalContextPipeline` is declared to return the fully+ // resolved `ExtendedContext` (= `Context & ContextExtension`).const { extendContext } = this.contextPipelineOptions;+ let contextPipeline: ContextPipeline<CrawlingContext, Context>;if (extendContext !== undefined) {- contextPipeline = contextPipeline.compose({- action: async (context) => await extendContext(context),- });+ contextPipeline = ContextPipeline.create<CrawlingContext>()+ .compose({ action: async (context) => await extendContext(context) })+ .chain(+ subclassPipeline as unknown as ContextPipeline<+ CrawlingContext & ContextExtension,+ CrawlingContext & ContextExtension+ >,+ ) as unknown as ContextPipeline<CrawlingContext, Context>;+ } else {+ contextPipeline = subclassPipeline;}contextPipeline = contextPipeline.compose({diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.tsindex 82a1450a7..da8e3e17d 100644--- a/packages/browser-crawler/src/internals/browser-crawler.ts+++ b/packages/browser-crawler/src/internals/browser-crawler.ts@@ -88,8 +88,8 @@ export interface BrowserCrawlingContext<enqueueLinks: (options?: EnqueueLinksOptions) => Promise<BatchAddRequestsResult>;}-export type BrowserHook<Context = BrowserCrawlingContext> = (- crawlingContext: Context,+export type BrowserHook<Context = BrowserCrawlingContext, ContextExtension = {}> = (+ crawlingContext: Context & ContextExtension,) => Awaitable<void | Partial<Context>>;const COOKIES_BEFORE_HOOKS = Symbol('cookiesBeforeHooks');@@ -111,7 +111,7 @@ export interface BrowserCrawlerOptions<__BrowserControllerReturn extends BrowserController = ReturnType<__BrowserPlugins[number]['createController']>,__LaunchContextReturn extends LaunchContext = ReturnType<__BrowserPlugins[number]['createLaunchContext']>,> extends Omit<- BasicCrawlerOptions<Context, ExtendedContext>,+ BasicCrawlerOptions<Context, ContextExtension, ExtendedContext>,// Overridden with browser context'requestHandler' | 'failedRequestHandler' | 'errorHandler'> {@@ -214,8 +214,13 @@ export interface BrowserCrawlerOptions<** A hook may optionally return a partial object whose properties are merged into the crawling context,* allowing the hook to override context members for subsequent hooks and pipeline stages.+ *+ * The context is built up in the following order: base context (`request`, `session`, helpers, ...) ->+ * `extendContext` -> `preNavigationHooks` -> navigation -> `postNavigationHooks` -> `requestHandler`.+ * This means the members added by `extendContext` are already available here, but navigation-dependent+ * members (e.g. `page`, `response`) are not.*/- preNavigationHooks?: BrowserHook<Context>[];+ preNavigationHooks?: BrowserHook<Context, ContextExtension>[];/*** Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.@@ -242,7 +247,7 @@ export interface BrowserCrawlerOptions<* ]* ```*/- postNavigationHooks?: BrowserHook<Context>[];+ postNavigationHooks?: BrowserHook<Context, ContextExtension>[];/*** Timeout in which page navigation needs to finish, in seconds.@@ -420,13 +425,16 @@ export abstract class BrowserCrawler<.compose({ action: this.handleBlockedRequestByContent.bind(this) }).compose({ action: this.restoreRequestState.bind(this) });},- extendContext: extendContext as (context: Context) => Awaitable<ContextExtension>,+ extendContext,});this.launchContext = launchContext;this.navigationTimeoutMillis = navigationTimeoutSecs * 1000;- this.preNavigationHooks = preNavigationHooks;- this.postNavigationHooks = postNavigationHooks;+ // The public option hooks are extension-aware; internal storage uses the base context type+ // (the pipeline composes hooks against the concrete context, which does not statically carry+ // `ContextExtension`). The extension members are present at runtime regardless.+ this.preNavigationHooks = preNavigationHooks as BrowserHook<Context>[];+ this.postNavigationHooks = postNavigationHooks as BrowserHook<Context>[];this.ignoreIframes = ignoreIframes;this.ignoreShadowRoots = ignoreShadowRoots;diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.tsindex 499c7524c..c25e64716 100644--- a/packages/http-crawler/src/internals/http-crawler.ts+++ b/packages/http-crawler/src/internals/http-crawler.ts@@ -82,6 +82,11 @@ export interface HttpCrawlerOptions<** A hook may optionally return a partial object whose properties are merged into the crawling context,* allowing the hook to override context members for subsequent hooks and pipeline stages.+ *+ * The context is built up in the following order: base context (`request`, `session`, helpers, ...) ->+ * `extendContext` -> `preNavigationHooks` -> navigation -> `postNavigationHooks` -> `requestHandler`.+ * This means the members added by `extendContext` are already available here, but navigation-dependent+ * members (e.g. `response`, `body`, `$`) are not.* Example:* ```* preNavigationHooks: [@@ -91,7 +96,7 @@ export interface HttpCrawlerOptions<* ]* ```*/- preNavigationHooks?: InternalHttpHook<CrawlingContext>[];+ preNavigationHooks?: InternalHttpHook<CrawlingContext, ContextExtension>[];/*** Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.@@ -111,7 +116,7 @@ export interface HttpCrawlerOptions<* ```*/postNavigationHooks?: ((- crawlingContext: CrawlingContextWithResponse,+ crawlingContext: CrawlingContextWithResponse & ContextExtension,) => Awaitable<void | Partial<CrawlingContextWithResponse>>)[];/**@@ -157,7 +162,9 @@ export interface HttpCrawlerOptions</*** @internal*/-export type InternalHttpHook<Context> = (crawlingContext: Context) => Awaitable<void | Partial<Context>>;+export type InternalHttpHook<Context, ContextExtension = {}> = (+ crawlingContext: Context & ContextExtension,+) => Awaitable<void | Partial<Context>>;export type HttpHook<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler@@ -316,6 +323,10 @@ export class HttpCrawler<ContextExtension = Dictionary<never>,ExtendedContext extends Context = Context & ContextExtension,> extends BasicCrawler<Context, ContextExtension, ExtendedContext> {+ // Internal storage uses the base (non-extended) context types. The public option types are+ // extension-aware for consumer DX, but internally the pipeline composes hooks against the+ // concrete crawling context, which does not statically carry `ContextExtension`. The members+ // added by `extendContext` are present at runtime regardless.private preNavigationHooks: InternalHttpHook<CrawlingContext>[];private postNavigationHooks: ((crawlingContext: CrawlingContextWithResponse,@@ -387,10 +398,13 @@ export class HttpCrawler<this.ignoreSslErrors = ignoreSslErrors;this.suggestResponseEncoding = suggestResponseEncoding;this.forceResponseEncoding = forceResponseEncoding;- this.preNavigationHooks = preNavigationHooks;+ // Cast away the extension-aware option types to the base internal storage types (see the field+ // declarations above). This is sound - the hooks only ever receive the base context plus the+ // members `extendContext` added at runtime.+ this.preNavigationHooks = preNavigationHooks as InternalHttpHook<CrawlingContext>[];this.postNavigationHooks = [({ request, response }) => this._abortDownloadOfBody(request, response!),- ...postNavigationHooks,+ ...(postNavigationHooks as typeof this.postNavigationHooks),];this.saveResponseCookies = saveResponseCookies;diff --git a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.tsindex 5e2617df7..4b118ef1c 100644--- a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts+++ b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts@@ -174,16 +174,18 @@ interface AdaptiveHookContext extends Pick<AdaptivePlaywrightCrawlerContext, 'idgotoOptions?: PlaywrightGotoOptions;}-interface AdaptiveHook extends BrowserHook<AdaptiveHookContext> {}+type AdaptiveHook<ContextExtension = Dictionary<never>> = BrowserHook<AdaptiveHookContext, ContextExtension>;-interface AdaptivePostNavigationHook extends BrowserHook<- Omit<AdaptiveHookContext, 'request'> & { request: LoadedRequest<Request> }-> {}+type AdaptivePostNavigationHook<ContextExtension = Dictionary<never>> = BrowserHook<+ Omit<AdaptiveHookContext, 'request'> & { request: LoadedRequest<Request> },+ ContextExtension+>;export interface AdaptivePlaywrightCrawlerOptions<- ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext,+ ContextExtension = Dictionary<never>,+ ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext & ContextExtension,> extends Omit<- BasicCrawlerOptions<AdaptivePlaywrightCrawlerContext, ExtendedContext>,+ BasicCrawlerOptions<AdaptivePlaywrightCrawlerContext, ContextExtension, ExtendedContext>,'preNavigationHooks' | 'postNavigationHooks'> {/**@@ -194,7 +196,7 @@ export interface AdaptivePlaywrightCrawlerOptions<* A hook may optionally return a partial object whose properties are merged into the crawling context,* allowing the hook to override context members for subsequent hooks and pipeline stages.*/- preNavigationHooks?: AdaptiveHook[];+ preNavigationHooks?: AdaptiveHook<ContextExtension>[];/*** Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.@@ -204,7 +206,7 @@ export interface AdaptivePlaywrightCrawlerOptions<* A hook may optionally return a partial object whose properties are merged into the crawling context* (e.g. to override `response` after solving a challenge).*/- postNavigationHooks?: AdaptivePostNavigationHook[];+ postNavigationHooks?: AdaptivePostNavigationHook<ContextExtension>[];/*** Specifies the frequency of rendering type detection checks - 0.1 means roughly 10% of requests.@@ -302,8 +304,9 @@ type LogProxyCall = [log: CrawleeLogger, method: (typeof proxyLogMethods)[number* @experimental*/export class AdaptivePlaywrightCrawler<- ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext,-> extends BasicCrawler<AdaptivePlaywrightCrawlerContext, ExtendedContext> {+ ContextExtension = Dictionary<never>,+ ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext & ContextExtension,+> extends BasicCrawler<AdaptivePlaywrightCrawlerContext, ContextExtension, ExtendedContext> {private renderingTypePredictor: NonNullable<AdaptivePlaywrightCrawlerOptions['renderingTypePredictor']>;private resultChecker: NonNullable<AdaptivePlaywrightCrawlerOptions['resultChecker']>;private shouldPropagateError: NonNullable<AdaptivePlaywrightCrawlerOptions['shouldPropagateError']>;@@ -318,7 +321,7 @@ export class AdaptivePlaywrightCrawler<private teardownHooks: (() => Promise<unknown>)[] = [];- constructor(options: AdaptivePlaywrightCrawlerOptions<ExtendedContext> = {}) {+ constructor(options: AdaptivePlaywrightCrawlerOptions<ContextExtension, ExtendedContext> = {}) {const {requestHandler,renderingTypeDetectionRatio = 0.1,@@ -367,10 +370,14 @@ export class AdaptivePlaywrightCrawler<);};}- // Each adaptive hook is registered as its own static/browser hook so the underlying- // `ContextPipeline` handles override merging between hooks for free. The hook signatures- // are structurally compatible with the underlying crawlers' contexts (subset of fields);- // the casts just relax the nominal type difference.+ // `extendContext` is forwarded to the inner crawlers, which run it *before* navigation (see+ // `BasicCrawler`), keeping the behavior consistent with the non-adaptive crawlers: the+ // extension is visible to the pre/post-navigation hooks and the request handler, but cannot+ // access navigation-dependent members (`page`, `response`, `$`, ...).+ //+ // The adaptive hooks target a subset context (`AdaptiveHookContext`); the casts to the inner+ // crawlers' `PlaywrightHook` type relax that nominal difference. The `ContextPipeline` merges+ // each hook's overrides at runtime regardless of the static type.const staticCrawler = new CheerioCrawler({...rest,statisticsOptions: {@@ -378,6 +385,7 @@ export class AdaptivePlaywrightCrawler<},preNavigationHooks,postNavigationHooks,+ extendContext,});const browserCrawler = new PlaywrightCrawler({@@ -387,27 +395,18 @@ export class AdaptivePlaywrightCrawler<},preNavigationHooks: preNavigationHooks as unknown as PlaywrightHook[],postNavigationHooks: postNavigationHooks as unknown as PlaywrightHook[],+ extendContext,});this.teardownHooks.push(browserCrawler.teardown.bind(browserCrawler));- this.staticContextPipeline = staticCrawler.contextPipeline- .compose({- action: this.adaptCheerioContext.bind(this),- })- .compose({- action: async (context) =>- extendContext ? await extendContext(context) : (context as unknown as ExtendedContext),- });-- this.browserContextPipeline = browserCrawler.contextPipeline- .compose({- action: this.adaptPlaywrightContext.bind(this),- })- .compose({- action: async (context) =>- extendContext ? await extendContext(context) : (context as unknown as ExtendedContext),- });+ this.staticContextPipeline = staticCrawler.contextPipeline.compose({+ action: this.adaptCheerioContext.bind(this),+ }) as unknown as ContextPipeline<CrawlingContext, ExtendedContext>;++ this.browserContextPipeline = browserCrawler.contextPipeline.compose({+ action: this.adaptPlaywrightContext.bind(this),+ }) as unknown as ContextPipeline<CrawlingContext, ExtendedContext>;this.stats = new AdaptivePlaywrightCrawlerStatistics({logMessage: `${this.log.getOptions().prefix} request statistics:`,diff --git a/packages/playwright-crawler/src/internals/playwright-crawler.ts b/packages/playwright-crawler/src/internals/playwright-crawler.tsindex 8c1f1bfe9..6afe9a397 100644--- a/packages/playwright-crawler/src/internals/playwright-crawler.ts+++ b/packages/playwright-crawler/src/internals/playwright-crawler.ts@@ -91,7 +91,7 @@ export interface PlaywrightCrawlerOptions<* ]* ```*/- preNavigationHooks?: PlaywrightHook[];+ preNavigationHooks?: BrowserHook<PlaywrightCrawlingContext, ContextExtension>[];/*** Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.@@ -109,7 +109,7 @@ export interface PlaywrightCrawlerOptions<* ]* ```*/- postNavigationHooks?: PlaywrightHook[];+ postNavigationHooks?: BrowserHook<PlaywrightCrawlingContext, ContextExtension>[];}/**@@ -200,7 +200,7 @@ export class PlaywrightCrawler</*** All `PlaywrightCrawler` parameters are passed via an options object.*/- constructor(options: PlaywrightCrawlerOptions<ExtendedContext> = {}) {+ constructor(options: PlaywrightCrawlerOptions<ContextExtension, ExtendedContext> = {}) {ow(options, 'PlaywrightCrawlerOptions', ow.object.exactShape(PlaywrightCrawler.optionsShape));const { launchContext = {}, headless, contextPipelineBuilder, ...browserCrawlerOptions } = options;@@ -232,7 +232,7 @@ export class PlaywrightCrawler<browserPoolOptions.browserPlugins = [playwrightLauncher.createBrowserPlugin()];super({- ...(browserCrawlerOptions as PlaywrightCrawlerOptions<ExtendedContext>),+ ...(browserCrawlerOptions as unknown as PlaywrightCrawlerOptions<ContextExtension, ExtendedContext>),launchContext,browserPoolOptions,contextPipelineBuilder: contextPipelineBuilder ?? (() => this.buildContextPipeline()),diff --git a/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts b/packages/puppeteer-crawler/src/internals/puppeteer-crawler.tsindex f846b0cbf..b9538ce78 100644--- a/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts+++ b/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts@@ -68,7 +68,7 @@ export interface PuppeteerCrawlerOptions<* ]* ```*/- preNavigationHooks?: PuppeteerHook[];+ preNavigationHooks?: BrowserHook<PuppeteerCrawlingContext, ContextExtension>[];/*** Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.@@ -86,7 +86,7 @@ export interface PuppeteerCrawlerOptions<* ]* ```*/- postNavigationHooks?: PuppeteerHook[];+ postNavigationHooks?: BrowserHook<PuppeteerCrawlingContext, ContextExtension>[];}/**diff --git a/test/core/crawlers/adaptive_playwright_crawler.test.ts b/test/core/crawlers/adaptive_playwright_crawler.test.tsindex 1ed884a90..e9988f392 100644--- a/test/core/crawlers/adaptive_playwright_crawler.test.ts+++ b/test/core/crawlers/adaptive_playwright_crawler.test.ts@@ -167,6 +167,53 @@ describe('AdaptivePlaywrightCrawler', () => {storeResult: vi.fn((_request: Request, _renderingType: string) => {}),});+ test.each([+ ['/static', 'static'],+ ['/dynamic', 'clientOnly'],+ ] as const)(+ 'extendContext is visible to pre/post-navigation hooks and the request handler (%s)',+ async (path, renderingType) => {+ const renderingTypePredictor = makeRiggedRenderingTypePredictor({+ detectionProbabilityRecommendation: 0,+ renderingType,+ });+ const url = new URL(`http://${HOSTNAME}:${port}${path}`);++ const seenIn: Record<string, unknown> = {};++ // Instantiated directly (rather than via `makeOneshotCrawler`) so that the `ContextExtension`+ // generic is inferred from `extendContext` and the hooks/handler see it without casts.+ const crawler = new AdaptivePlaywrightCrawler({+ renderingTypeDetectionRatio: 0.1,+ maxConcurrency: 1,+ maxRequestRetries: 0,+ maxRequestsPerCrawl: 1,+ requestList: await RequestList.open({ sources: [url.toString()] }),+ renderingTypePredictor,+ extendContext: () => ({ injected: 'from-extend-context' }),+ preNavigationHooks: [+ async (context) => {+ seenIn.preNavigation = context.injected;+ },+ ],+ postNavigationHooks: [+ async (context) => {+ seenIn.postNavigation = context.injected;+ },+ ],+ requestHandler: async (context) => {+ seenIn.requestHandler = context.injected;+ },+ });++ await crawler.run();++ expect(seenIn.preNavigation).toBe('from-extend-context');+ expect(seenIn.postNavigation).toBe('from-extend-context');+ expect(seenIn.requestHandler).toBe('from-extend-context');+ },+ );+describe('should detect page rendering type', () => {test.each([['/static', 'static'],diff --git a/test/core/crawlers/http_crawler.test.ts b/test/core/crawlers/http_crawler.test.tsindex cba6ab62b..fb7f69b83 100644--- a/test/core/crawlers/http_crawler.test.ts+++ b/test/core/crawlers/http_crawler.test.ts@@ -474,6 +474,34 @@ test('navigation hooks can override context members via return value', async ()expect(observedBody).toContain('overridden body');});+test('extendContext is visible to pre/post-navigation hooks and the request handler', async () => {+ const seenIn: Record<string, unknown> = {};++ const crawler = new HttpCrawler({+ maxRequestRetries: 0,+ extendContext: () => ({ injected: 'from-extend-context' }),+ preNavigationHooks: [+ async (context) => {+ seenIn.preNavigation = context.injected;+ },+ ],+ postNavigationHooks: [+ async (context) => {+ seenIn.postNavigation = context.injected;+ },+ ],+ requestHandler: async (context) => {+ seenIn.requestHandler = context.injected;+ },+ });++ await crawler.run([url]);++ expect(seenIn.preNavigation).toBe('from-extend-context');+ expect(seenIn.postNavigation).toBe('from-extend-context');+ expect(seenIn.requestHandler).toBe('from-extend-context');+});+test('works with a custom HttpClient', async () => {const results: string[] = [];