Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

feat: allow navigation hooks to override crawling context members (#3672)

Jindřich Bära54e3087f1fa
diff --git a/docs/guides/avoid_blocking.mdx b/docs/guides/avoid_blocking.mdx
index ed10846f5..bc84bf772 100644
--- a/docs/guides/avoid_blocking.mdx
+++ b/docs/guides/avoid_blocking.mdx
@@ -58,7 +58,7 @@ On the contrary, sometimes we want to entirely disable the usage of browser fing
## Camoufox
-For some protections, using our integrated solutions is not enough, one example could be the Cloudflare challenge. For such pages, you can try [Camoufox](https://camoufox.com/), a custom stealthy build of Firefox for web scraping. It might not get you through the challenge automatically, but with our `handleCloudflareChallenge` helper, it should be able to successfully mimic the required user action and get you through it.
+For some protections, using our integrated solutions is not enough, one example could be the Cloudflare challenge. For such pages, you can try [Camoufox](https://camoufox.com/), a custom stealthy build of Firefox for web scraping. It might not get you through the challenge automatically, but with our `handleCloudflareChallengeHook` post-navigation hook, it should be able to successfully mimic the required user action and get you through it. The hook also reloads the page after the challenge clears and propagates the fresh response back into the crawling context.
<CodeBlock language="ts">
{PlaywrightCamoufox}
diff --git a/docs/guides/avoid_blocking_camoufox.ts b/docs/guides/avoid_blocking_camoufox.ts
index 131234578..01b8f8ca4 100644
--- a/docs/guides/avoid_blocking_camoufox.ts
+++ b/docs/guides/avoid_blocking_camoufox.ts
@@ -1,13 +1,9 @@
-import { PlaywrightCrawler } from 'crawlee';
+import { PlaywrightCrawler, handleCloudflareChallengeHook } from 'crawlee';
import { launchOptions } from 'camoufox-js';
import { firefox } from 'playwright';
const crawler = new PlaywrightCrawler({
- postNavigationHooks: [
- async ({ handleCloudflareChallenge }) => {
- await handleCloudflareChallenge();
- },
- ],
+ postNavigationHooks: [handleCloudflareChallengeHook()],
browserPoolOptions: {
// Disable the default fingerprint spoofing to avoid conflicts with Camoufox.
useFingerprints: false,
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
index 40ed11e49..510d521df 100644
--- a/packages/basic-crawler/src/internals/basic-crawler.ts
+++ b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -2316,17 +2316,6 @@ export class BasicCrawler<
}
}
- protected async _executeHooks<HookLike extends (...args: any[]) => Awaitable<void>>(
- hooks: HookLike[],
- ...args: Parameters<HookLike>
- ) {
- if (Array.isArray(hooks) && hooks.length) {
- for (const hook of hooks) {
- await hook(...args);
- }
- }
- }
-
/**
* Stops the crawler immediately.
*
diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.ts
index 6440a862f..2a49d1ea4 100644
--- a/packages/browser-crawler/src/internals/browser-crawler.ts
+++ b/packages/browser-crawler/src/internals/browser-crawler.ts
@@ -2,6 +2,7 @@ import type {
Awaitable,
BasicCrawlerOptions,
BasicCrawlingContext,
+ ContextMiddleware,
CrawlingContext,
Dictionary,
EnqueueLinksOptions,
@@ -60,6 +61,7 @@ export interface BrowserCrawlingContext<
Response extends BaseResponse = BaseResponse,
ProvidedController = BrowserController,
UserData extends Dictionary = Dictionary,
+ GoToOptions extends Dictionary = Dictionary,
> extends CrawlingContext<UserData> {
/**
* An instance of the {@apilink BrowserController} that manages the browser instance and provides access to its API.
@@ -81,16 +83,25 @@ export interface BrowserCrawlingContext<
*/
response: Response;
+ /**
+ * Options object passed to the underlying `page.goto()` call. `preNavigationHooks` can mutate this
+ * object (or return `{ gotoOptions: ... }`) to influence the navigation.
+ */
+ gotoOptions: GoToOptions;
+
/**
* Helper function for extracting URLs from the current page and adding them to the request queue.
*/
enqueueLinks: (options?: EnqueueLinksOptions) => Promise<BatchAddRequestsResult>;
}
-export type BrowserHook<Context = BrowserCrawlingContext, GoToOptions extends Dictionary | undefined = Dictionary> = (
+export type BrowserHook<Context = BrowserCrawlingContext> = (
crawlingContext: Context,
- gotoOptions: GoToOptions,
-) => Awaitable<void>;
+) => Awaitable<void | Partial<Context>>;
+
+const COOKIES_BEFORE_HOOKS = Symbol('cookiesBeforeHooks');
+
+const readContextField = <T>(ctx: object, key: symbol): T => (ctx as Record<symbol, unknown>)[key] as T;
export interface BrowserCrawlerOptions<
Page extends CommonPage = CommonPage,
@@ -176,15 +187,14 @@ export interface BrowserCrawlerOptions<
/**
* Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies
- * or browser properties before navigation. The function accepts two parameters, `crawlingContext` and `gotoOptions`,
- * which are passed to the `page.goto()` function the crawler calls to navigate.
+ * or browser properties before navigation. The function receives the `crawlingContext`; the options object
+ * forwarded to `page.goto()` is available as `crawlingContext.gotoOptions` and can be mutated in place.
*
* **Example:**
*
* ```js
* preNavigationHooks: [
- * async (crawlingContext, gotoOptions) => {
- * const { page } = crawlingContext;
+ * async ({ page, gotoOptions }) => {
* await page.evaluate((attr) => { window.foo = attr; }, 'bar');
* gotoOptions.timeout = 60_000;
* gotoOptions.waitUntil = 'domcontentloaded';
@@ -192,8 +202,8 @@ export interface BrowserCrawlerOptions<
* ]
* ```
*
- * Modyfing `pageOptions` is supported only in Playwright incognito.
- * See {@apilink PrePageCreateHook}
+ * A hook may optionally return a partial object whose properties are merged into the crawling context,
+ * allowing the hook to override context members for subsequent hooks and pipeline stages.
*/
preNavigationHooks?: BrowserHook<Context>[];
@@ -201,6 +211,9 @@ export interface BrowserCrawlerOptions<
* Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
* The function accepts `crawlingContext` as the only parameter.
*
+ * A hook may optionally return a partial object whose properties are merged into the crawling context.
+ * This is useful for overriding context members (e.g. `response`) after solving a challenge.
+ *
* **Example:**
*
* ```js
@@ -211,6 +224,11 @@ export interface BrowserCrawlerOptions<
* await solveCaptcha(page);
* }
* },
+ * async (crawlingContext) => {
+ * if (await needsRevalidation(crawlingContext)) {
+ * return { response: await crawlingContext.page.reload() };
+ * }
+ * },
* ]
* ```
*/
@@ -360,13 +378,32 @@ export abstract class BrowserCrawler<
...basicCrawlerOptions
} = options;
+ const skipGuard = <Ctx extends Context>(
+ action: (ctx: Ctx) => Awaitable<void | Partial<Ctx>>,
+ ): ContextMiddleware<Ctx, Partial<Ctx>> => ({
+ action: async (ctx) => (ctx.request.skipNavigation ? {} : ((await action(ctx)) ?? {})),
+ });
+
super({
...basicCrawlerOptions,
- contextPipelineBuilder: () =>
- contextPipelineBuilder()
- .compose({ action: this.performNavigation.bind(this) })
+ contextPipelineBuilder: () => {
+ let pipeline = contextPipelineBuilder().compose({ action: this.prepareNavigation.bind(this) });
+
+ for (const hook of this.preNavigationHooks) {
+ pipeline = pipeline.compose(skipGuard(hook));
+ }
+
+ pipeline = pipeline.compose(skipGuard(this.navigate.bind(this)));
+
+ for (const hook of this.postNavigationHooks) {
+ pipeline = pipeline.compose(skipGuard(hook));
+ }
+
+ return pipeline
+ .compose(skipGuard(this.finalizeNavigation.bind(this)))
.compose({ action: this.handleBlockedRequestByContent.bind(this) })
- .compose({ action: this.restoreRequestState.bind(this) }),
+ .compose({ action: this.restoreRequestState.bind(this) });
+ },
extendContext: extendContext as (context: Context) => Awaitable<ContextExtension>,
});
@@ -503,6 +540,9 @@ export abstract class BrowserCrawler<
"The `response` property is not available. This might mean that you're trying to access it before navigation or that navigation resulted in `null` (this should only happen with `about:` URLs)",
);
},
+ get gotoOptions(): Dictionary {
+ throw new Error('The `gotoOptions` property is not available until `prepareNavigation` runs.');
+ },
browserController: browserControllerInstance,
enqueueLinks: async (enqueueOptions: EnqueueLinksOptions = {}) => {
return (await browserCrawlerEnqueueLinks({
@@ -521,10 +561,7 @@ export abstract class BrowserCrawler<
};
}
- private async performNavigation(crawlingContext: Context): Promise<{
- request: LoadedRequest<Request>;
- response?: Response;
- }> {
+ private async prepareNavigation(crawlingContext: Context): Promise<Partial<Context>> {
if (crawlingContext.request.skipNavigation) {
return {
request: new Proxy(crawlingContext.request, {
@@ -542,42 +579,56 @@ export abstract class BrowserCrawler<
'The `response` property is not available - `skipNavigation` was used',
);
},
- };
+ } as Partial<Context>;
}
- const gotoOptions = { timeout: this.navigationTimeoutMillis } as unknown as GoToOptions;
+ crawlingContext.request.state = RequestState.BEFORE_NAV;
- const preNavigationHooksCookies = this._getCookieHeaderFromRequest(crawlingContext.request);
+ return {
+ gotoOptions: { timeout: this.navigationTimeoutMillis } as unknown as GoToOptions,
+ [COOKIES_BEFORE_HOOKS]: this._getCookieHeaderFromRequest(crawlingContext.request),
+ } as unknown as Partial<Context>;
+ }
- crawlingContext.request.state = RequestState.BEFORE_NAV;
- await this._executeHooks(this.preNavigationHooks, crawlingContext, gotoOptions);
+ private async navigate(crawlingContext: Context): Promise<Partial<Context>> {
tryCancel();
- const postNavigationHooksCookies = this._getCookieHeaderFromRequest(crawlingContext.request);
+ const gotoOptions = crawlingContext.gotoOptions as GoToOptions;
+ const cookiesBeforeHooks = readContextField<string>(crawlingContext, COOKIES_BEFORE_HOOKS);
+ const cookiesAfterHooks = this._getCookieHeaderFromRequest(crawlingContext.request);
- await this._applyCookies(crawlingContext, preNavigationHooksCookies, postNavigationHooksCookies);
+ await this._applyCookies(crawlingContext, cookiesBeforeHooks, cookiesAfterHooks);
let response: Response | undefined;
-
try {
response = (await this._navigationHandler(crawlingContext, gotoOptions)) ?? undefined;
} catch (error) {
await this._handleNavigationTimeout(crawlingContext, error as Error);
-
crawlingContext.request.state = RequestState.ERROR;
-
this._throwIfProxyError(error as Error);
throw error;
}
tryCancel();
crawlingContext.request.state = RequestState.AFTER_NAV;
- await this._executeHooks(this.postNavigationHooks, crawlingContext, gotoOptions);
+
+ return { response } as Partial<Context>;
+ }
+
+ private async finalizeNavigation(crawlingContext: Context): Promise<Partial<Context>> {
+ tryCancel();
+
+ let response: Response | undefined;
+ try {
+ response = crawlingContext.response;
+ } catch {
+ // `preparePage` installs a throwing getter for `response`; reaching this branch means
+ // navigation produced no response and no hook overrode it. Treat as undefined.
+ }
await this.processResponse(response, crawlingContext);
tryCancel();
- // save cookies
// TODO: Should we save the cookies also after/only the handle page?
if (this.saveResponseCookies && crawlingContext.session) {
const cookies = await crawlingContext.browserController.getCookies(crawlingContext.page);
@@ -594,16 +645,7 @@ export abstract class BrowserCrawler<
}
}
- if (response !== undefined) {
- return {
- request: crawlingContext.request as LoadedRequest<Request>,
- response,
- };
- }
-
- return {
- request: crawlingContext.request as LoadedRequest<Request>,
- };
+ return { request: crawlingContext.request as LoadedRequest<Request> } as Partial<Context>;
}
private async handleBlockedRequestByContent(
diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.ts
index 6a9733c19..37e6b54a6 100644
--- a/packages/http-crawler/src/internals/http-crawler.ts
+++ b/packages/http-crawler/src/internals/http-crawler.ts
@@ -4,6 +4,7 @@ import util from 'node:util';
import type {
AutoscaledPoolOptions,
BasicCrawlerOptions,
+ ContextMiddleware,
CrawlingContext,
ErrorHandler,
GetUserDataFromRequest,
@@ -78,6 +79,9 @@ export interface HttpCrawlerOptions<
* Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies
* or browser properties before navigation. The function accepts one parameter `crawlingContext`,
* which is passed to the `requestAsBrowser()` function the crawler calls to navigate.
+ *
+ * A hook may optionally return a partial object whose properties are merged into the crawling context,
+ * allowing the hook to override context members for subsequent hooks and pipeline stages.
* Example:
* ```
* preNavigationHooks: [
@@ -86,25 +90,29 @@ export interface HttpCrawlerOptions<
* },
* ]
* ```
- *
- * Modyfing `pageOptions` is supported only in Playwright incognito.
- * See {@apilink PrePageCreateHook}
*/
preNavigationHooks?: InternalHttpHook<CrawlingContext>[];
/**
* Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
* The function accepts `crawlingContext` as the only parameter.
+ *
+ * A hook may optionally return a partial object whose properties are merged into the crawling context,
+ * which is useful for overriding the `response` after solving a challenge or re-fetching the resource.
* Example:
* ```
* postNavigationHooks: [
* async (crawlingContext) => {
- * // ...
+ * if (await needsRevalidation(crawlingContext)) {
+ * return { response: await refetch(crawlingContext.request) };
+ * }
* },
* ]
* ```
*/
- postNavigationHooks?: ((crawlingContext: CrawlingContextWithReponse) => Awaitable<void>)[];
+ postNavigationHooks?: ((
+ crawlingContext: CrawlingContextWithResponse,
+ ) => Awaitable<void | Partial<CrawlingContextWithResponse>>)[];
/**
* An array of [MIME types](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Complete_list_of_MIME_types)
@@ -149,14 +157,14 @@ export interface HttpCrawlerOptions<
/**
* @internal
*/
-export type InternalHttpHook<Context> = (crawlingContext: Context) => Awaitable<void>;
+export type InternalHttpHook<Context> = (crawlingContext: Context) => Awaitable<void | Partial<Context>>;
export type HttpHook<
UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
JSONData extends JsonValue = any, // with default to Dictionary we cant use a typed router in untyped crawler
> = InternalHttpHook<HttpCrawlingContext<UserData, JSONData>>;
-interface CrawlingContextWithReponse<
+interface CrawlingContextWithResponse<
UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
> extends CrawlingContext<UserData> {
/**
@@ -176,7 +184,7 @@ interface CrawlingContextWithReponse<
export interface InternalHttpCrawlingContext<
UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
JSONData extends JsonValue = any, // with default to Dictionary we cant use a typed router in untyped crawler
-> extends CrawlingContextWithReponse<UserData> {
+> extends CrawlingContextWithResponse<UserData> {
/**
* The request body of the web page.
* The type depends on the `Content-Type` header of the web page:
@@ -307,7 +315,9 @@ export class HttpCrawler<
ExtendedContext extends Context = Context & ContextExtension,
> extends BasicCrawler<Context, ContextExtension, ExtendedContext> {
protected preNavigationHooks: InternalHttpHook<CrawlingContext>[];
- protected postNavigationHooks: ((crawlingContext: CrawlingContextWithReponse) => Awaitable<void>)[];
+ protected postNavigationHooks: ((
+ crawlingContext: CrawlingContextWithResponse,
+ ) => Awaitable<void | Partial<CrawlingContextWithResponse>>)[];
protected saveResponseCookies: boolean;
protected navigationTimeoutMillis: number;
protected ignoreSslErrors: boolean;
@@ -385,18 +395,35 @@ export class HttpCrawler<
}
protected override buildContextPipeline(): ContextPipeline<CrawlingContext, InternalHttpCrawlingContext> {
- return ContextPipeline.create<CrawlingContext>()
- .compose({
- action: this.makeHttpRequest.bind(this),
- })
+ // When navigation is skipped, `prepareHttpRequest` has already installed throwing getters for
+ // the response-derived members, so the guarded action is bypassed and the context left untouched.
+ const skipGuard = <Ctx extends CrawlingContext, Ext>(
+ action: (ctx: Ctx) => Awaitable<void | Ext>,
+ ): ContextMiddleware<Ctx, Ext> => ({
+ action: async (ctx) => (ctx.request.skipNavigation ? {} : ((await action(ctx)) ?? {})) as Ext,
+ });
+
+ let pipeline = ContextPipeline.create<CrawlingContext>().compose({
+ action: this.prepareHttpRequest.bind(this),
+ });
+
+ for (const hook of this.preNavigationHooks) {
+ pipeline = pipeline.compose(skipGuard(hook));
+ }
+
+ let pipelineWithNavigation = pipeline.compose(skipGuard(this.makeHttpRequest.bind(this)));
+
+ for (const hook of this.postNavigationHooks) {
+ pipelineWithNavigation = pipelineWithNavigation.compose(skipGuard(hook));
+ }
+
+ return pipelineWithNavigation
.compose({ action: this.processHttpResponse.bind(this) })
.compose({ action: this.handleBlockedRequestByContent.bind(this) });
}
- private async makeHttpRequest(
- crawlingContext: CrawlingContext,
- ): Promise<Omit<CrawlingContextWithReponse, keyof CrawlingContext> & Partial<CrawlingContextWithReponse>> {
- const { request, session } = crawlingContext;
+ private async prepareHttpRequest(crawlingContext: CrawlingContext): Promise<Partial<CrawlingContextWithResponse>> {
+ const { request } = crawlingContext;
if (request.skipNavigation) {
return {
@@ -415,13 +442,19 @@ export class HttpCrawler<
'The `response` property is not available - `skipNavigation` was used',
);
},
- };
+ } as Partial<CrawlingContextWithResponse>;
}
request.state = RequestState.BEFORE_NAV;
- await this._executeHooks(this.preNavigationHooks, crawlingContext);
+ return {};
+ }
+
+ private async makeHttpRequest(
+ crawlingContext: CrawlingContext,
+ ): Promise<Omit<CrawlingContextWithResponse, keyof CrawlingContext> & Partial<CrawlingContextWithResponse>> {
tryCancel();
+ const { request, session } = crawlingContext;
const proxyUrl = crawlingContext.proxyInfo?.url;
const httpResponse = await addTimeoutToPromise(
@@ -438,9 +471,9 @@ export class HttpCrawler<
}
private async processHttpResponse(
- crawlingContext: CrawlingContextWithReponse,
+ crawlingContext: CrawlingContextWithResponse,
): Promise<
- Omit<InternalHttpCrawlingContext, keyof CrawlingContextWithReponse> & Partial<InternalHttpCrawlingContext>
+ Omit<InternalHttpCrawlingContext, keyof CrawlingContextWithResponse> & Partial<InternalHttpCrawlingContext>
> {
if (crawlingContext.request.skipNavigation) {
return {
@@ -472,7 +505,6 @@ export class HttpCrawler<
};
}
- await this._executeHooks(this.postNavigationHooks, crawlingContext);
tryCancel();
const parsed = await this._parseResponse(crawlingContext.request, crawlingContext.response);
diff --git a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts
index 42320882e..09910967f 100644
--- a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts
+++ b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts
@@ -46,7 +46,7 @@ import type { SetRequired } from 'type-fest';
import { addTimeoutToPromise } from '@apify/timeout';
-import type { PlaywrightCrawlingContext, PlaywrightGotoOptions } from './playwright-crawler.js';
+import type { PlaywrightCrawlingContext, PlaywrightGotoOptions, PlaywrightHook } from './playwright-crawler.js';
import { PlaywrightCrawler } from './playwright-crawler.js';
import { type RenderingType, RenderingTypePredictor } from './utils/rendering-type-prediction.js';
@@ -164,12 +164,16 @@ export interface AdaptivePlaywrightCrawlerContext<
enqueueLinks(options?: EnqueueLinksOptions): Promise<unknown>;
}
-interface AdaptiveHook extends BrowserHook<
- Pick<AdaptivePlaywrightCrawlerContext, 'id' | 'session' | 'proxyInfo' | 'log'> & {
- page?: Page;
- request: Request;
- },
- PlaywrightGotoOptions
+interface AdaptiveHookContext extends Pick<AdaptivePlaywrightCrawlerContext, 'id' | 'session' | 'proxyInfo' | 'log'> {
+ page?: Page;
+ request: Request;
+ gotoOptions?: PlaywrightGotoOptions;
+}
+
+interface AdaptiveHook extends BrowserHook<AdaptiveHookContext> {}
+
+interface AdaptivePostNavigationHook extends BrowserHook<
+ Omit<AdaptiveHookContext, 'request'> & { request: LoadedRequest<Request> }
> {}
export interface AdaptivePlaywrightCrawlerOptions<
@@ -182,6 +186,9 @@ export interface AdaptivePlaywrightCrawlerOptions<
* Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies.
* The function accepts a subset of the crawling context. If you attempt to access the `page` property during HTTP-only crawling,
* an exception will be thrown. If it's not caught, the request will be transparently retried in a browser.
+ *
+ * A hook may optionally return a partial object whose properties are merged into the crawling context,
+ * allowing the hook to override context members for subsequent hooks and pipeline stages.
*/
preNavigationHooks?: AdaptiveHook[];
@@ -189,8 +196,11 @@ export interface AdaptivePlaywrightCrawlerOptions<
* Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
* The function accepts a subset of the crawling context. If you attempt to access the `page` property during HTTP-only crawling,
* an exception will be thrown. If it's not caught, the request will be transparently retried in a browser.
+ *
+ * A hook may optionally return a partial object whose properties are merged into the crawling context
+ * (e.g. to override `response` after solving a challenge).
*/
- postNavigationHooks?: AdaptiveHook[]; // TODO should contain a LoadedRequest - reflect that
+ postNavigationHooks?: AdaptivePostNavigationHook[];
/**
* Specifies the frequency of rendering type detection checks - 0.1 means roughly 10% of requests.
@@ -323,8 +333,8 @@ export class AdaptivePlaywrightCrawler<
requestHandlerTimeoutSecs = 60,
errorHandler,
failedRequestHandler,
- preNavigationHooks,
- postNavigationHooks,
+ preNavigationHooks = [],
+ postNavigationHooks = [],
extendContext,
contextPipelineBuilder,
...rest
@@ -359,25 +369,17 @@ export class AdaptivePlaywrightCrawler<
);
};
}
+ // Each adaptive hook is registered as its own static/browser hook so the underlying
+ // `ContextPipeline` handles override merging between hooks for free. The hook signatures
+ // are structurally compatible with the underlying crawlers' contexts (subset of fields);
+ // the casts just relax the nominal type difference.
const staticCrawler = new CheerioCrawler({
...rest,
statisticsOptions: {
persistenceOptions: { enable: false },
},
- preNavigationHooks: [
- async (context) => {
- for (const hook of preNavigationHooks ?? []) {
- await hook(context, undefined);
- }
- },
- ],
- postNavigationHooks: [
- async (context) => {
- for (const hook of postNavigationHooks ?? []) {
- await hook(context, undefined);
- }
- },
- ],
+ preNavigationHooks,
+ postNavigationHooks,
});
const browserCrawler = new PlaywrightCrawler({
@@ -385,20 +387,8 @@ export class AdaptivePlaywrightCrawler<
statisticsOptions: {
persistenceOptions: { enable: false },
},
- preNavigationHooks: [
- async (context, gotoOptions) => {
- for (const hook of preNavigationHooks ?? []) {
- await hook(context, gotoOptions);
- }
- },
- ],
- postNavigationHooks: [
- async (context, gotoOptions) => {
- for (const hook of postNavigationHooks ?? []) {
- await hook(context, gotoOptions);
- }
- },
- ],
+ preNavigationHooks: preNavigationHooks as unknown as PlaywrightHook[],
+ postNavigationHooks: postNavigationHooks as unknown as PlaywrightHook[],
});
this.teardownHooks.push(browserCrawler.teardown.bind(browserCrawler));
diff --git a/packages/playwright-crawler/src/internals/playwright-crawler.ts b/packages/playwright-crawler/src/internals/playwright-crawler.ts
index 991aa65bb..8633ba6a5 100644
--- a/packages/playwright-crawler/src/internals/playwright-crawler.ts
+++ b/packages/playwright-crawler/src/internals/playwright-crawler.ts
@@ -29,10 +29,13 @@ import type {
} from './utils/playwright-utils.js';
import { gotoExtended, playwrightUtils } from './utils/playwright-utils.js';
+export type PlaywrightGotoOptions = NonNullable<Parameters<Page['goto']>[1]>;
+
export interface PlaywrightCrawlingContext<UserData extends Dictionary = Dictionary>
- extends BrowserCrawlingContext<Page, Response, PlaywrightController, UserData>, PlaywrightContextUtils {}
-export interface PlaywrightHook extends BrowserHook<PlaywrightCrawlingContext, PlaywrightGotoOptions> {}
-export type PlaywrightGotoOptions = Parameters<Page['goto']>[1];
+ extends
+ BrowserCrawlingContext<Page, Response, PlaywrightController, UserData, PlaywrightGotoOptions>,
+ PlaywrightContextUtils {}
+export interface PlaywrightHook extends BrowserHook<PlaywrightCrawlingContext> {}
export interface PlaywrightCrawlerOptions<
ContextExtension = Dictionary<never>,
@@ -79,26 +82,26 @@ export interface PlaywrightCrawlerOptions<
/**
* Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies
- * or browser properties before navigation. The function accepts two parameters, `crawlingContext` and `gotoOptions`,
- * which are passed to the `page.goto()` function the crawler calls to navigate.
+ * or browser properties before navigation. The function receives the `crawlingContext`; the options object
+ * forwarded to `page.goto()` is available as `crawlingContext.gotoOptions` and can be mutated in place.
+ * A hook may optionally return a partial object whose properties are merged into the crawling context
+ * (e.g. to override context members for subsequent hooks and pipeline stages).
* Example:
* ```
* preNavigationHooks: [
- * async (crawlingContext, gotoOptions) => {
- * const { page } = crawlingContext;
+ * async ({ page, gotoOptions }) => {
* await page.evaluate((attr) => { window.foo = attr; }, 'bar');
+ * gotoOptions.timeout = 60_000;
* },
* ]
* ```
- *
- * Modyfing `pageOptions` is supported only in Playwright incognito.
- * See {@apilink PrePageCreateHook}
*/
preNavigationHooks?: PlaywrightHook[];
/**
* Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
- * The function accepts `crawlingContext` as the only parameter.
+ * The function accepts `crawlingContext` as the only parameter. A hook may optionally return a partial object
+ * whose properties are merged into the crawling context (e.g. to override `response` after solving a challenge).
* Example:
* ```
* postNavigationHooks: [
@@ -299,17 +302,35 @@ export class PlaywrightCrawler<
compileScript: (scriptString: string, ctx?: Dictionary) => playwrightUtils.compileScript(scriptString, ctx),
closeCookieModals: async () => playwrightUtils.closeCookieModals(context.page),
handleCloudflareChallenge: async (options?: HandleCloudflareChallengeOptions) => {
- return playwrightUtils.handleCloudflareChallenge(
- context.page,
- context.request.url,
- options,
- this.blockedStatusCodes,
- );
+ return playwrightUtils.handleCloudflareChallenge(context.page, context.request.url, options);
},
};
}
}
+/**
+ * Returns a `postNavigationHooks`-ready hook that runs {@apilink PlaywrightContextUtils.handleCloudflareChallenge}
+ * and propagates the post-challenge {@apilink Response} back into the crawling context via its return value.
+ *
+ * **Example usage**
+ * ```ts
+ * import { PlaywrightCrawler, handleCloudflareChallengeHook } from 'crawlee';
+ *
+ * const crawler = new PlaywrightCrawler({
+ * postNavigationHooks: [handleCloudflareChallengeHook()],
+ * });
+ * ```
+ */
+export function handleCloudflareChallengeHook(options?: HandleCloudflareChallengeOptions): PlaywrightHook {
+ return async (context) => {
+ const response = await context.handleCloudflareChallenge(options);
+ if (response !== undefined) {
+ return { response };
+ }
+ return undefined;
+ };
+}
+
/**
* Creates new {@apilink Router} instance that works based on request labels.
* This instance can then serve as a `requestHandler` of your {@apilink PlaywrightCrawler}.
diff --git a/packages/playwright-crawler/src/internals/utils/playwright-utils.ts b/packages/playwright-crawler/src/internals/utils/playwright-utils.ts
index e86c69349..11b634eee 100644
--- a/packages/playwright-crawler/src/internals/utils/playwright-utils.ts
+++ b/packages/playwright-crawler/src/internals/utils/playwright-utils.ts
@@ -707,14 +707,16 @@ export interface HandleCloudflareChallengeOptions {
* result in a SessionError which will be automatically retried, so only successful requests will get
* into the `requestHandler`.
*
+ * On a successfully solved challenge the page is reloaded and the new {@apilink Response} is returned, so
+ * it can be propagated back to the crawling context via a hook return value (see
+ * {@apilink handleCloudflareChallengeHook}).
+ *
* Works best with camoufox.
*
* **Example usage**
* ```ts
* postNavigationHooks: [
- * async ({ handleCloudflareChallenge }) => {
- * await handleCloudflareChallenge();
- * },
+ * async (context) => ({ response: await context.handleCloudflareChallenge() }),
* ],
* ```
*
@@ -726,12 +728,7 @@ async function handleCloudflareChallenge(
page: Page,
url: string,
options: HandleCloudflareChallengeOptions = {},
- blockedStatusCodes?: Set<number>,
-): Promise<void> {
- // Cloudflare pages return 403, which is blocked by default — temporarily allow it during challenge handling
- const had403 = blockedStatusCodes?.has(403);
- blockedStatusCodes?.delete(403);
-
+): Promise<Response | undefined> {
options.isBlockedCallback ??= async () => {
const isBlocked = await page.evaluate(() => {
return document.querySelector('h1')?.textContent?.trim().includes('Sorry, you have been blocked');
@@ -760,87 +757,85 @@ async function handleCloudflareChallenge(
return options.isChallengeCallback!(page).catch(() => false);
};
- try {
- if (!(await isChallenge())) {
- await retryBlocked();
- return;
- }
-
- const logLevel = options.verbose ? 'info' : 'debug';
- getLog()[logLevel](
- `Detected Cloudflare challenge at ${url}, trying to solve it. This can take up to ${10 + (options.sleepSecs ?? 10)} seconds.`,
- );
+ if (!(await isChallenge())) {
+ await retryBlocked();
+ return undefined;
+ }
- const bb = await page
- .evaluate(() => {
- // Prefer the actual challenge widget (the box holding the Turnstile checkbox input);
- // fall back to the first content div for older challenge layouts.
- const div =
- document.querySelector('.main-content div:has(input[id^="cf-chl-widget-"])') ??
- document.querySelector('.main-content div');
- return div?.getBoundingClientRect();
- })
- .catch(() => undefined);
-
- if (!bb) {
- return;
- }
+ const logLevel = options.verbose ? 'info' : 'debug';
+ getLog()[logLevel](
+ `Detected Cloudflare challenge at ${url}, trying to solve it. This can take up to ${10 + (options.sleepSecs ?? 10)} seconds.`,
+ );
- const randomOffset = (range: number) => {
- return Math.round(100 * range * Math.random()) / 100;
- };
+ const bb = await page
+ .evaluate(() => {
+ // Prefer the actual challenge widget (the box holding the Turnstile checkbox input);
+ // fall back to the first content div for older challenge layouts.
+ const div =
+ document.querySelector('.main-content div:has(input[id^="cf-chl-widget-"])') ??
+ document.querySelector('.main-content div');
+ return div?.getBoundingClientRect();
+ })
+ .catch(() => undefined);
+
+ if (!bb) {
+ return undefined;
+ }
- let x = bb.x + 30;
- let y = bb.y + 25;
+ const randomOffset = (range: number) => {
+ return Math.round(100 * range * Math.random()) / 100;
+ };
- // try to click the checkbox every second
- for (let i = 0; i < 10; i++) {
- await sleep((options.preChallengeSleepSecs ?? 1) * 1000);
+ let x = bb.x + 30;
+ let y = bb.y + 25;
- // break early if we are no longer on the CF challenge page
- if (!(await isChallenge())) {
- break;
- }
+ // try to click the checkbox every second
+ for (let i = 0; i < 10; i++) {
+ await sleep((options.preChallengeSleepSecs ?? 1) * 1000);
- if (options.clickPositionCallback) {
- const pos = await options.clickPositionCallback(page);
- if (pos) {
- x = pos.x;
- y = pos.y;
- }
- }
+ // break early if we are no longer on the CF challenge page
+ if (!(await isChallenge())) {
+ break;
+ }
- if (options.clickCallback) {
- await options.clickCallback(page, { x, y });
- continue;
+ if (options.clickPositionCallback) {
+ const pos = await options.clickPositionCallback(page);
+ if (pos) {
+ x = pos.x;
+ y = pos.y;
}
+ }
- // we can click on the text too, so X can be a bit larger
- const xRandomized = x + randomOffset(10);
- const yRandomized = y + randomOffset(10);
+ if (options.clickCallback) {
+ await options.clickCallback(page, { x, y });
+ continue;
+ }
- getLog()[logLevel](`Trying to click on the Cloudflare checkbox at ${url}`, {
- x: xRandomized,
- y: yRandomized,
- });
- await page.mouse.click(xRandomized, yRandomized);
+ // we can click on the text too, so X can be a bit larger
+ const xRandomized = x + randomOffset(10);
+ const yRandomized = y + randomOffset(10);
- // sometimes the checkbox is lower (could be caused by a lag when rendering the logo)
- await page.mouse.click(xRandomized, yRandomized + 35);
- }
+ getLog()[logLevel](`Trying to click on the Cloudflare checkbox at ${url}`, {
+ x: xRandomized,
+ y: yRandomized,
+ });
+ await page.mouse.click(xRandomized, yRandomized);
- await sleep((options.sleepSecs ?? 10) * 1000);
+ // sometimes the checkbox is lower (could be caused by a lag when rendering the logo)
+ await page.mouse.click(xRandomized, yRandomized + 35);
+ }
- if (await isChallenge()) {
- throw new SessionError(`Blocked by Cloudflare when processing ${url}`);
- }
+ await sleep((options.sleepSecs ?? 10) * 1000);
- await retryBlocked();
- } finally {
- if (had403) {
- blockedStatusCodes?.add(403);
- }
+ if (await isChallenge()) {
+ throw new SessionError(`Blocked by Cloudflare when processing ${url}`);
}
+
+ await retryBlocked();
+
+ // Reload to obtain a fresh Response without the challenge interstitial, which the caller can
+ // propagate back into the crawling context so downstream status-code checks see the new value.
+ return (await page.reload()) ?? undefined;
}
/** @internal */
@@ -1054,20 +1049,20 @@ export interface PlaywrightContextUtils {
* result in a SessionError which will be automatically retried, so only successful requests will get
* into the `requestHandler`.
*
- * Works best with camoufox.
+ * On a successfully solved challenge the page is reloaded and the new {@apilink Response} is returned,
+ * which can be returned from the hook to update the crawling context's `response`. For the common case,
+ * prefer the pre-wrapped {@apilink handleCloudflareChallengeHook} hook.
*
* **Example usage**
* ```ts
* postNavigationHooks: [
- * async ({ handleCloudflareChallenge }) => {
- * await handleCloudflareChallenge();
- * },
+ * async (context) => ({ response: await context.handleCloudflareChallenge() }),
* ],
* ```
*
* @param [options]
*/
- handleCloudflareChallenge(options?: HandleCloudflareChallengeOptions): Promise<void>;
+ handleCloudflareChallenge(options?: HandleCloudflareChallengeOptions): Promise<Response | undefined>;
/**
* Returns the list of {@link https://playwright.dev/docs/api/class-download | Download} objects
diff --git a/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts b/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts
index 51ac13d97..92cc5bdef 100644
--- a/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts
+++ b/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts
@@ -30,10 +30,13 @@ import type {
} from './utils/puppeteer_utils.js';
import { gotoExtended, puppeteerUtils } from './utils/puppeteer_utils.js';
+export type PuppeteerGoToOptions = NonNullable<Parameters<Page['goto']>[1]>;
+
export interface PuppeteerCrawlingContext<UserData extends Dictionary = Dictionary>
- extends BrowserCrawlingContext<Page, HTTPResponse, PuppeteerController, UserData>, PuppeteerContextUtils {}
-export interface PuppeteerHook extends BrowserHook<PuppeteerCrawlingContext, PuppeteerGoToOptions> {}
-export type PuppeteerGoToOptions = Parameters<Page['goto']>[1];
+ extends
+ BrowserCrawlingContext<Page, HTTPResponse, PuppeteerController, UserData, PuppeteerGoToOptions>,
+ PuppeteerContextUtils {}
+export interface PuppeteerHook extends BrowserHook<PuppeteerCrawlingContext> {}
export interface PuppeteerCrawlerOptions<
ContextExtension = Dictionary<never>,
@@ -54,26 +57,26 @@ export interface PuppeteerCrawlerOptions<
/**
* Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies
- * or browser properties before navigation. The function accepts two parameters, `crawlingContext` and `gotoOptions`,
- * which are passed to the `page.goto()` function the crawler calls to navigate.
+ * or browser properties before navigation. The function receives the `crawlingContext`; the options object
+ * forwarded to `page.goto()` is available as `crawlingContext.gotoOptions` and can be mutated in place.
+ * A hook may optionally return a partial object whose properties are merged into the crawling context
+ * (e.g. to override context members for subsequent hooks and pipeline stages).
* Example:
* ```
* preNavigationHooks: [
- * async (crawlingContext, gotoOptions) => {
- * const { page } = crawlingContext;
+ * async ({ page, gotoOptions }) => {
* await page.evaluate((attr) => { window.foo = attr; }, 'bar');
+ * gotoOptions.timeout = 60_000;
* },
* ]
* ```
- *
- * Modyfing `pageOptions` is supported only in Playwright incognito.
- * See {@apilink PrePageCreateHook}
*/
preNavigationHooks?: PuppeteerHook[];
/**
* Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
- * The function accepts `crawlingContext` as the only parameter.
+ * The function accepts `crawlingContext` as the only parameter. A hook may optionally return a partial object
+ * whose properties are merged into the crawling context (e.g. to override `response` after solving a challenge).
* Example:
* ```
* postNavigationHooks: [
diff --git a/packages/stagehand-crawler/src/internals/stagehand-crawler.ts b/packages/stagehand-crawler/src/internals/stagehand-crawler.ts
index ab0291ba0..17be591e7 100644
--- a/packages/stagehand-crawler/src/internals/stagehand-crawler.ts
+++ b/packages/stagehand-crawler/src/internals/stagehand-crawler.ts
@@ -210,11 +210,17 @@ export interface StagehandPage extends Page {
/**
* Crawling context for StagehandCrawler with enhanced page object.
*/
+/**
+ * Goto options for StagehandCrawler navigation.
+ */
+export type StagehandGotoOptions = NonNullable<Parameters<Page['goto']>[1]>;
+
export interface StagehandCrawlingContext<UserData extends Dictionary = Dictionary> extends BrowserCrawlingContext<
StagehandPage,
Response,
StagehandController,
- UserData
+ UserData,
+ StagehandGotoOptions
> {
/**
* Enhanced Playwright page with Stagehand AI methods.
@@ -232,18 +238,13 @@ export interface StagehandCrawlingContext<UserData extends Dictionary = Dictiona
/**
* Hook function for StagehandCrawler.
*/
-export interface StagehandHook extends BrowserHook<StagehandCrawlingContext, StagehandGotoOptions> {}
+export interface StagehandHook extends BrowserHook<StagehandCrawlingContext> {}
/**
* Request handler for StagehandCrawler.
*/
export interface StagehandRequestHandler extends RequestHandler<LoadedContext<StagehandCrawlingContext>> {}
-/**
- * Goto options for StagehandCrawler navigation.
- */
-export type StagehandGotoOptions = Dictionary & Parameters<Page['goto']>[1];
-
/**
* Options for StagehandCrawler.
*/
diff --git a/test/core/crawlers/browser_crawler.test.ts b/test/core/crawlers/browser_crawler.test.ts
index 24017204c..1d631e210 100644
--- a/test/core/crawlers/browser_crawler.test.ts
+++ b/test/core/crawlers/browser_crawler.test.ts
@@ -244,6 +244,52 @@ describe('BrowserCrawler', () => {
}
});
+ test.concurrent('postNavigationHooks can override response, observed downstream', async () => {
+ const localStorageEmulator = new MemoryStorageEmulator();
+ await localStorageEmulator.init();
+ const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
+
+ try {
+ const requestList = await RequestList.open({
+ sources: [{ url: `${serverAddress}/?q=1` }],
+ });
+
+ const observed: { fromSecondHook?: number; fromHandler?: number } = {};
+ const fakeStatus = 418;
+
+ const browserCrawler = new BrowserCrawlerTest({
+ browserPoolOptions: {
+ browserPlugins: [puppeteerPlugin],
+ },
+ requestList,
+ maxRequestRetries: 0,
+ postNavigationHooks: [
+ async ({ response }) => ({
+ response: new Proxy(response, {
+ get(target, key, receiver) {
+ if (key === 'status') return () => fakeStatus;
+ return Reflect.get(target, key, receiver);
+ },
+ }),
+ }),
+ async ({ response }) => {
+ observed.fromSecondHook = response.status();
+ },
+ ],
+ requestHandler: async ({ response }) => {
+ observed.fromHandler = response.status();
+ },
+ });
+
+ await browserCrawler.run();
+
+ expect(observed.fromSecondHook).toBe(fakeStatus);
+ expect(observed.fromHandler).toBe(fakeStatus);
+ } finally {
+ await localStorageEmulator.destroy();
+ }
+ });
+
test.concurrent('errorHandler has open page', async () => {
const localStorageEmulator = new MemoryStorageEmulator();
await localStorageEmulator.init();
@@ -394,8 +440,8 @@ describe('BrowserCrawler', () => {
requestHandler: async () => {},
maxRequestRetries: 0,
preNavigationHooks: [
- async (_crawlingContext, gotoOptions) => {
- gotoOptions!.timeout = 60000;
+ async ({ gotoOptions }) => {
+ gotoOptions.timeout = 60000;
},
],
});
diff --git a/test/core/crawlers/http_crawler.test.ts b/test/core/crawlers/http_crawler.test.ts
index 8fe4c2d27..b82b65fb1 100644
--- a/test/core/crawlers/http_crawler.test.ts
+++ b/test/core/crawlers/http_crawler.test.ts
@@ -403,6 +403,38 @@ test('should retry on 403 even with disallowed content-type', async () => {
expect(succeeded[0].retryCount).toBe(1);
});
+test('navigation hooks can override context members via return value', async () => {
+ let observedBody: string | undefined;
+ let observedStatus: number | undefined;
+ let postHookSawOverride = false;
+
+ const crawler = new HttpCrawler({
+ maxRequestRetries: 0,
+ postNavigationHooks: [
+ async ({ request }) => ({
+ response: new ResponseWithUrl('<html>overridden body</html>', {
+ url: request.url,
+ status: 201,
+ headers: { 'content-type': 'text/html; charset=utf-8' },
+ }),
+ }),
+ async ({ response }) => {
+ postHookSawOverride = response.status === 201;
+ },
+ ],
+ requestHandler: async ({ body, response }) => {
+ observedBody = body.toString();
+ observedStatus = response.status;
+ },
+ });
+
+ await crawler.run([url]);
+
+ expect(postHookSawOverride).toBe(true);
+ expect(observedStatus).toBe(201);
+ expect(observedBody).toContain('overridden body');
+});
+
test('works with a custom HttpClient', async () => {
const results: string[] = [];
diff --git a/test/core/crawlers/playwright_crawler.test.ts b/test/core/crawlers/playwright_crawler.test.ts
index 57dd5e8dd..4f132d8bf 100644
--- a/test/core/crawlers/playwright_crawler.test.ts
+++ b/test/core/crawlers/playwright_crawler.test.ts
@@ -178,7 +178,7 @@ describe('PlaywrightCrawler', () => {
maxConcurrency: 1,
requestHandler: () => {},
preNavigationHooks: [
- (_context, gotoOptions) => {
+ ({ gotoOptions }) => {
options = gotoOptions;
},
],
diff --git a/test/core/crawlers/puppeteer_crawler.test.ts b/test/core/crawlers/puppeteer_crawler.test.ts
index bd644497e..ba2720798 100644
--- a/test/core/crawlers/puppeteer_crawler.test.ts
+++ b/test/core/crawlers/puppeteer_crawler.test.ts
@@ -154,7 +154,7 @@ describe('PuppeteerCrawler', () => {
maxConcurrency: 1,
requestHandler: () => {},
preNavigationHooks: [
- (_context, gotoOptions) => {
+ ({ gotoOptions }) => {
options = gotoOptions;
},
],