Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: mark `context.request.loadedUrl` and `id` as required inside the request handler (#2531)

Martin Adámek2b546600691d
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
index 998c0c371..61b10b743 100644
--- a/packages/basic-crawler/src/internals/basic-crawler.ts
+++ b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -26,6 +26,7 @@ import type {
Source,
StatisticState,
StatisticsOptions,
+ LoadedContext,
} from '@crawlee/core';
import {
AutoscaledPool,
@@ -99,11 +100,11 @@ export interface BasicCrawlingContext<UserData extends Dictionary = Dictionary>
const SAFE_MIGRATION_WAIT_MILLIS = 20000;
export type RequestHandler<Context extends CrawlingContext = BasicCrawlingContext> = (
- inputs: Context,
+ inputs: LoadedContext<Context>,
) => Awaitable<void>;
export type ErrorHandler<Context extends CrawlingContext = BasicCrawlingContext> = (
- inputs: Context,
+ inputs: LoadedContext<Context>,
error: Error,
) => Awaitable<void>;
@@ -140,7 +141,7 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCraw
* The exceptions are logged to the request using the
* {@apilink Request.pushErrorMessage|`Request.pushErrorMessage()`} function.
*/
- requestHandler?: RequestHandler<Context>;
+ requestHandler?: RequestHandler<LoadedContext<Context>>;
/**
* User-provided function that performs the logic of the crawler. It is called for each URL to crawl.
@@ -472,7 +473,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
* Default {@apilink Router} instance that will be used if we don't specify any {@apilink BasicCrawlerOptions.requestHandler|`requestHandler`}.
* See {@apilink Router.addHandler|`router.addHandler()`} and {@apilink Router.addDefaultHandler|`router.addDefaultHandler()`}.
*/
- readonly router: RouterHandler<Context> = Router.create<Context>();
+ readonly router: RouterHandler<LoadedContext<Context>> = Router.create<LoadedContext<Context>>();
running = false;
hasFinishedBefore = false;
@@ -1079,7 +1080,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
}
protected async _runRequestHandler(crawlingContext: Context): Promise<void> {
- await this.requestHandler(crawlingContext);
+ await this.requestHandler(crawlingContext as LoadedContext<Context>);
}
/**
@@ -1554,7 +1555,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
configurable: true,
});
- return context;
+ return context as LoadedContext<Context>;
}
/**
diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.ts
index 5e1a3e5b3..6f514599a 100644
--- a/packages/browser-crawler/src/internals/browser-crawler.ts
+++ b/packages/browser-crawler/src/internals/browser-crawler.ts
@@ -6,6 +6,7 @@ import type {
Dictionary,
EnqueueLinksOptions,
ErrorHandler,
+ LoadedContext,
ProxyConfiguration,
ProxyInfo,
RequestHandler,
@@ -111,7 +112,7 @@ export interface BrowserCrawlerOptions<
* The exceptions are logged to the request using the
* {@apilink Request.pushErrorMessage|`Request.pushErrorMessage()`} function.
*/
- requestHandler?: BrowserRequestHandler<Context>;
+ requestHandler?: BrowserRequestHandler<LoadedContext<Context>>;
/**
* Function that is called to process each request.
@@ -143,7 +144,7 @@ export interface BrowserCrawlerOptions<
* @deprecated `handlePageFunction` has been renamed to `requestHandler` and will be removed in a future version.
* @ignore
*/
- handlePageFunction?: BrowserRequestHandler<Context>;
+ handlePageFunction?: BrowserRequestHandler<LoadedContext<Context>>;
/**
* User-provided function that allows modifying the request object before it gets retried by the crawler.
@@ -375,7 +376,7 @@ export abstract class BrowserCrawler<
super(
{
...basicCrawlerOptions,
- requestHandler: async (...args) => this._runRequestHandler(...args),
+ requestHandler: async (...args) => this._runRequestHandler(...(args as [Context])),
requestHandlerTimeoutSecs:
navigationTimeoutSecs + requestHandlerTimeoutSecs + BASIC_CRAWLER_TIMEOUT_BUFFER_SECS,
},
@@ -569,7 +570,7 @@ export abstract class BrowserCrawler<
request.state = RequestState.REQUEST_HANDLER;
try {
await addTimeoutToPromise(
- async () => Promise.resolve(this.userProvidedRequestHandler(crawlingContext)),
+ async () => Promise.resolve(this.userProvidedRequestHandler(crawlingContext as LoadedContext<Context>)),
this.requestHandlerTimeoutInnerMillis,
`requestHandler timed out after ${this.requestHandlerTimeoutInnerMillis / 1000} seconds.`,
);
diff --git a/packages/core/src/crawlers/crawler_commons.ts b/packages/core/src/crawlers/crawler_commons.ts
index e32bb329b..5ea0e71e3 100644
--- a/packages/core/src/crawlers/crawler_commons.ts
+++ b/packages/core/src/crawlers/crawler_commons.ts
@@ -12,6 +12,21 @@ import type { Session } from '../session_pool/session';
import type { RequestQueueOperationOptions, Dataset, RecordOptions } from '../storages';
import { KeyValueStore } from '../storages';
+/** @internal */
+export type IsAny<T> = 0 extends 1 & T ? true : false;
+
+/** @internal */
+export type WithRequired<T, K extends keyof T> = T & { [P in K]-?: T[P] };
+
+export type LoadedRequest<R extends Request> = WithRequired<R, 'id' | 'loadedUrl'>;
+
+/** @internal */
+export type LoadedContext<Context extends RestrictedCrawlingContext> = IsAny<Context> extends true
+ ? Context
+ : {
+ request: LoadedRequest<Context['request']>;
+ } & Omit<Context, 'request'>;
+
export interface RestrictedCrawlingContext<UserData extends Dictionary = Dictionary>
// we need `Record<string & {}, unknown>` here, otherwise `Omit<Context>` is resolved badly
extends Record<string & {}, unknown> {
diff --git a/packages/core/src/router.ts b/packages/core/src/router.ts
index 27f92557b..545bb3360 100644
--- a/packages/core/src/router.ts
+++ b/packages/core/src/router.ts
@@ -1,6 +1,6 @@
import type { Dictionary } from '@crawlee/types';
-import type { CrawlingContext, RestrictedCrawlingContext } from './crawlers/crawler_commons';
+import type { CrawlingContext, LoadedRequest, RestrictedCrawlingContext } from './crawlers/crawler_commons';
import { MissingRouteError } from './errors';
import type { Request } from './request';
import type { Awaitable } from './typedefs';
@@ -84,7 +84,7 @@ export type RouterRoutes<Context, UserData extends Dictionary> = {
* ```
*/
export class Router<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'>> {
- private readonly routes: Map<string | symbol, (ctx: Context) => Awaitable<void>> = new Map();
+ private readonly routes: Map<string | symbol, (ctx: any) => Awaitable<void>> = new Map();
private readonly middlewares: ((ctx: Context) => Awaitable<void>)[] = [];
/**
@@ -98,7 +98,7 @@ export class Router<Context extends Omit<RestrictedCrawlingContext, 'enqueueLink
*/
addHandler<UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(
label: string | symbol,
- handler: (ctx: Omit<Context, 'request'> & { request: Request<UserData> }) => Awaitable<void>,
+ handler: (ctx: Omit<Context, 'request'> & { request: LoadedRequest<Request<UserData>> }) => Awaitable<void>,
) {
this.validate(label);
this.routes.set(label, handler);
@@ -108,7 +108,7 @@ export class Router<Context extends Omit<RestrictedCrawlingContext, 'enqueueLink
* Registers default route handler.
*/
addDefaultHandler<UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(
- handler: (ctx: Omit<Context, 'request'> & { request: Request<UserData> }) => Awaitable<void>,
+ handler: (ctx: Omit<Context, 'request'> & { request: LoadedRequest<Request<UserData>> }) => Awaitable<void>,
) {
this.validate(defaultRoute);
this.routes.set(defaultRoute, handler);
diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.ts
index 114bb849c..79bd77a8f 100644
--- a/packages/http-crawler/src/internals/http-crawler.ts
+++ b/packages/http-crawler/src/internals/http-crawler.ts
@@ -10,6 +10,7 @@ import type {
CrawlingContext,
ErrorHandler,
GetUserDataFromRequest,
+ LoadedContext,
ProxyConfiguration,
Request,
RequestHandler,
@@ -572,7 +573,7 @@ export class HttpCrawler<
request.state = RequestState.REQUEST_HANDLER;
try {
await addTimeoutToPromise(
- async () => Promise.resolve(this.requestHandler(crawlingContext)),
+ async () => Promise.resolve(this.requestHandler(crawlingContext as LoadedContext<Context>)),
this.userRequestHandlerTimeoutMillis,
`requestHandler timed out after ${this.userRequestHandlerTimeoutMillis / 1000} seconds.`,
);
diff --git a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts
index 305c41760..e35b82814 100644
--- a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts
+++ b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts
@@ -1,6 +1,12 @@
import type { Log } from '@apify/log';
import { addTimeoutToPromise } from '@apify/timeout';
-import { extractUrlsFromPage, type RouterHandler } from '@crawlee/browser';
+import {
+ extractUrlsFromPage,
+ type LoadedContext,
+ type LoadedRequest,
+ type Request,
+ type RouterHandler,
+} from '@crawlee/browser';
import type {
RestrictedCrawlingContext,
StatisticState,
@@ -131,7 +137,7 @@ export interface AdaptivePlaywrightCrawlerOptions
* If the function throws an exception, the crawler will try to re-crawl the
* request later, up to `option.maxRequestRetries` times.
*/
- requestHandler?: (crawlingContext: AdaptivePlaywrightCrawlerContext) => Awaitable<void>;
+ requestHandler?: (crawlingContext: LoadedContext<AdaptivePlaywrightCrawlerContext>) => Awaitable<void>;
/**
* Specifies the frequency of rendering type detection checks - 0.1 means roughly 10% of requests.
@@ -386,7 +392,7 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler {
id: crawlingContext.id,
session: crawlingContext.session,
proxyInfo: crawlingContext.proxyInfo,
- request: crawlingContext.request,
+ request: crawlingContext.request as LoadedRequest<Request>,
log: crawlingContext.log,
querySelector: async (selector, timeoutMs = 5_000) => {
const locator = playwrightContext.page.locator(selector).first();
@@ -468,7 +474,7 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler {
id: crawlingContext.id,
session: crawlingContext.session,
proxyInfo: crawlingContext.proxyInfo,
- request: crawlingContext.request,
+ request: crawlingContext.request as LoadedRequest<Request>,
log: this.createLogProxy(crawlingContext.log, logs),
async querySelector(selector, _timeoutMs?: number) {
return $(selector) as Cheerio<Element>;
diff --git a/packages/playwright-crawler/src/internals/playwright-crawler.ts b/packages/playwright-crawler/src/internals/playwright-crawler.ts
index 041315a9d..2b1f5c643 100644
--- a/packages/playwright-crawler/src/internals/playwright-crawler.ts
+++ b/packages/playwright-crawler/src/internals/playwright-crawler.ts
@@ -4,6 +4,7 @@ import type {
BrowserHook,
BrowserRequestHandler,
GetUserDataFromRequest,
+ LoadedContext,
RouterRoutes,
} from '@crawlee/browser';
import { BrowserCrawler, Configuration, Router } from '@crawlee/browser';
@@ -21,7 +22,7 @@ export interface PlaywrightCrawlingContext<UserData extends Dictionary = Diction
extends BrowserCrawlingContext<PlaywrightCrawler, Page, Response, PlaywrightController, UserData>,
PlaywrightContextUtils {}
export interface PlaywrightHook extends BrowserHook<PlaywrightCrawlingContext, PlaywrightGotoOptions> {}
-export interface PlaywrightRequestHandler extends BrowserRequestHandler<PlaywrightCrawlingContext> {}
+export interface PlaywrightRequestHandler extends BrowserRequestHandler<LoadedContext<PlaywrightCrawlingContext>> {}
export type PlaywrightGotoOptions = Parameters<Page['goto']>[1];
export interface PlaywrightCrawlerOptions
diff --git a/test/core/crawlers/cheerio_crawler.test.ts b/test/core/crawlers/cheerio_crawler.test.ts
index 961e1f12e..8734553b6 100644
--- a/test/core/crawlers/cheerio_crawler.test.ts
+++ b/test/core/crawlers/cheerio_crawler.test.ts
@@ -252,7 +252,7 @@ describe('CheerioCrawler', () => {
});
test('should serialize body and html', async () => {
- expect.assertions(2);
+ expect.assertions(3);
const sources = [`${serverAddress}/special/html-type`];
const requestList = await RequestList.open(null, sources);
@@ -260,8 +260,10 @@ describe('CheerioCrawler', () => {
requestList,
maxRequestRetries: 0,
maxConcurrency: 1,
- requestHandler: ({ $, body }) => {
+ requestHandler: ({ $, body, request }) => {
expect(body).toBe(responseSamples.html);
+ // test that `request.loadedUrl` is no longer optional
+ expect(request.loadedUrl.length).toBe(sources[0].length);
expect($.html()).toBe(body);
},
});