Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

refactor!: keep crawler vocabulary out of @crawlee/core's Request (#4148)

Jan Buchar8beb96ef8f8d
diff --git a/docs/public-api/crawlee-basic.api.md b/docs/public-api/crawlee-basic.api.md
index 7dbf18315..0a351957e 100644
--- a/docs/public-api/crawlee-basic.api.md
+++ b/docs/public-api/crawlee-basic.api.md
@@ -36,13 +36,13 @@ import { RequestOptions } from '@crawlee/core';
import { RequestQueue } from '@crawlee/core';
import type { RequestQueueOperationInfo } from '@crawlee/core';
import type { RequestQueueOperationOptions } from '@crawlee/core';
+import type { RequestSchema } from '@crawlee/types';
import type { RequestsLike } from '@crawlee/core';
import type { RequestSourceStatus } from '@crawlee/core';
import { RobotsTxtFile } from '@crawlee/utils';
import type { SendRequestOptions } from '@crawlee/types';
import type { SessionFingerprint } from '@crawlee/types';
import type { SetStatusMessageOptions } from '@crawlee/types';
-import type { SkippedRequestReason } from '@crawlee/core';
import { Source } from '@crawlee/core';
import type { StandardSchemaV1 } from '@standard-schema/spec';
import type { StorageBackend } from '@crawlee/types';
@@ -62,7 +62,7 @@ export class BasicCrawler<Context extends CrawlingContext = CrawlingContext, Con
protected readonly additionalHttpErrorStatusCodes: Set<number>;
addRequests(requests: ReadonlyDeep<TypedRequestsLike<Routes>>, options?: CrawlerAddRequestsOptions): Promise<CrawlerAddRequestsResult>;
get basicContextPipeline(): ContextPipeline<{
- request: Request_2;
+ request: CrawlingRequest;
}, CrawlingContext>;
// (undocumented)
protected blockedStatusCodes: Set<number>;
@@ -313,6 +313,23 @@ export interface CrawlingContext<UserData extends Dictionary = Dictionary> exten
sendRequest: (requestOverrides?: Partial<HttpRequestOptions>, optionsOverrides?: SendRequestOptions) => Promise<Response>;
}
+// @public
+export class CrawlingRequest<UserData extends Dictionary = Dictionary> extends Request_2<UserData> {
+ get crawlDepth(): number;
+ set crawlDepth(value: number);
+ // (undocumented)
+ static fromSchema<UserData extends Dictionary = Dictionary>(schema: RequestSchema): CrawlingRequest<UserData>;
+ get maxRetries(): number | undefined;
+ set maxRetries(value: number | undefined);
+ pushErrorMessage(errorOrMessage: unknown, options?: PushErrorMessageOptions): void;
+ get sessionId(): string | undefined;
+ set sessionId(value: string | undefined);
+ get skipNavigation(): boolean;
+ set skipNavigation(value: boolean);
+ get state(): RequestState;
+ set state(value: RequestState);
+}
+
// @public
export function createBasicRouter<Context extends BasicCrawlingContext = BasicCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
@@ -324,7 +341,7 @@ export interface CreateContextOptions {
// (undocumented)
proxyInfo?: ProxyInfo;
// (undocumented)
- request: Request_2;
+ request: CrawlingRequest;
// (undocumented)
session: ISession;
}
@@ -505,7 +522,7 @@ export interface FinalStatistics {
}
// @public (undocumented)
-export type GetUserDataFromRequest<T> = T extends Request_2<infer Y> ? Y : never;
+export type GetUserDataFromRequest<T> = T extends CrawlingRequest<infer Y> ? Y : never;
// @public (undocumented)
export type GlobInput = string | GlobObject;
@@ -637,6 +654,11 @@ export interface PersistenceOptions {
export class PersistentRateLimitError extends CriticalError {
}
+// @public (undocumented)
+export interface PushErrorMessageOptions {
+ omitStack?: boolean;
+}
+
// @public (undocumented)
export type RegExpInput = RegExp | RegExpObject;
@@ -657,6 +679,26 @@ export class RequestHandlerError extends Error {
// @public
export type RequestManagerOpener<T extends IRequestManager = IRequestManager> = (identifier?: string | StorageIdentifier | null, options?: StorageOpenOptions) => Promise<T>;
+// @public (undocumented)
+export enum RequestState {
+ // (undocumented)
+ AFTER_NAV = 2,
+ // (undocumented)
+ BEFORE_NAV = 1,
+ // (undocumented)
+ DONE = 4,
+ // (undocumented)
+ ERROR = 6,
+ // (undocumented)
+ ERROR_HANDLER = 5,
+ // (undocumented)
+ REQUEST_HANDLER = 3,
+ // (undocumented)
+ SKIPPED = 7,
+ // (undocumented)
+ UNPROCESSED = 0
+}
+
// @public
export class RequestThrottledError extends RetryRequestError {
constructor(message?: string);
@@ -690,7 +732,7 @@ export interface RestrictedCrawlingContext<UserData extends Dictionary = Diction
log: CrawleeLogger;
proxyInfo?: ProxyInfo;
pushData(data: ReadonlyDeep<Parameters<Dataset['pushData']>[0]>, datasetIdentifier?: string | StorageIdentifier): Promise<void>;
- request: Request_2<UserData>;
+ request: CrawlingRequest<UserData>;
// (undocumented)
session: ISession;
useState: <State extends Dictionary = Dictionary>(defaultValue?: State) => Promise<State>;
@@ -730,7 +772,7 @@ export interface RouterHandler<Context extends RestrictedCrawlingContext = Crawl
// @public
export type RouterHandlerContext<Context, UserData extends Dictionary, Routes extends Record<keyof Routes, Dictionary>> = Omit<Context, 'request' | 'addRequests' | 'enqueueLinks'> & {
- request: LoadedRequest<Request_2<UserData>>;
+ request: LoadedRequest<CrawlingRequest<UserData>>;
addRequests: TypedContextAddRequests<Routes>;
} & (Context extends {
enqueueLinks: infer EnqueueLinks;
@@ -744,7 +786,7 @@ export type RouterLabel<Routes extends Record<keyof Routes, Dictionary>> = strin
// @public (undocumented)
export type RouterRoutes<Context, Routes extends Record<keyof Routes, Dictionary>> = {
[Label in keyof Routes]: (ctx: Omit<Context, 'request'> & {
- request: Request_2<Routes[Label]>;
+ request: CrawlingRequest<Routes[Label]>;
}) => Awaitable<void>;
};
@@ -912,6 +954,9 @@ export type SkippedRequestCallback = (args: {
reason: SkippedRequestReason;
}) => Awaitable<void>;
+// @public (undocumented)
+export type SkippedRequestReason = 'robotsTxt' | 'limit' | 'enqueueLimit' | 'filters' | 'transform' | 'redirect' | 'depth';
+
// @public (undocumented)
export interface SnapshotResult {
// (undocumented)
diff --git a/docs/public-api/crawlee-browser.api.md b/docs/public-api/crawlee-browser.api.md
index 140e0132e..31df7f0f7 100644
--- a/docs/public-api/crawlee-browser.api.md
+++ b/docs/public-api/crawlee-browser.api.md
@@ -15,6 +15,7 @@ import type { CommonPage } from '@crawlee/browser-pool';
import { ContextPipeline } from '@crawlee/basic';
import type { CrawlerRemoteBrowserOptions } from '@crawlee/browser-pool';
import type { CrawlingContext } from '@crawlee/basic';
+import { CrawlingRequest } from '@crawlee/basic';
import type { Dictionary } from '@crawlee/types';
import type { EnqueueLinksOptions } from '@crawlee/basic';
import type { ErrorHandler } from '@crawlee/basic';
@@ -23,7 +24,6 @@ import type { GetUserDataFromRequest } from '@crawlee/basic';
import type { IBrowserPool } from '@crawlee/types';
import type { LoadedRequest } from '@crawlee/basic';
import type { RemoteBrowserPoolOptions } from '@crawlee/browser-pool';
-import { Request as Request_2 } from '@crawlee/basic';
import type { RequestHandler } from '@crawlee/basic';
import type { RouterHandler } from '@crawlee/basic';
@@ -87,7 +87,7 @@ GoToOptions extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
extractLinks: (options?: ExtractLinksOptions) => Promise<string[]>;
gotoOptions: GoToOptions;
page: Page;
- request: LoadedRequest<Request_2<UserData>>;
+ request: LoadedRequest<CrawlingRequest<UserData>>;
response: Response;
}
diff --git a/docs/public-api/crawlee-core.api.md b/docs/public-api/crawlee-core.api.md
index 4d65b43a0..75687aa64 100644
--- a/docs/public-api/crawlee-core.api.md
+++ b/docs/public-api/crawlee-core.api.md
@@ -35,6 +35,7 @@ import type { QueueOperationInfo } from '@crawlee/types';
import type { ReadonlyDeep } from 'type-fest';
import type { RequestQueueBackend } from '@crawlee/types';
import type { RequestQueueInfo } from '@crawlee/types';
+import type { RequestSchema } from '@crawlee/types';
import type { StandardSchemaV1 } from '@standard-schema/spec';
import type * as storage from '@crawlee/types';
import { StorageBackend } from '@crawlee/types';
@@ -444,7 +445,7 @@ export interface IStorage {
export interface JournaledRequest {
// (undocumented)
label?: string;
- snapshot?: Dictionary;
+ snapshot?: RequestSchema;
// (undocumented)
uniqueKey: string;
// (undocumented)
@@ -671,11 +672,6 @@ export function purgeDefaultStorages(options?: PurgeDefaultStorageOptions): Prom
// @public
export function purgeDefaultStorages(configuration?: Configuration, storageBackend?: StorageBackend): Promise<void>;
-// @public (undocumented)
-export interface PushErrorMessageOptions {
- omitStack?: boolean;
-}
-
// @public (undocumented)
export interface RecordOptions {
contentType?: string;
@@ -712,9 +708,8 @@ export interface RecoverableStatePersistenceOptions {
// @public
class Request_2<UserData extends Dictionary = Dictionary> {
constructor(options: RequestOptions<UserData>);
- get crawlDepth(): number;
- set crawlDepth(value: number);
errorMessages: string[];
+ static fromSchema<UserData extends Dictionary = Dictionary>(schema: RequestSchema): Request_2<UserData>;
handledAt?: string;
headers?: Record<string, string>;
id?: string;
@@ -722,21 +717,10 @@ class Request_2<UserData extends Dictionary = Dictionary> {
get label(): string | undefined;
set label(value: string | undefined);
loadedUrl?: string;
- get maxRetries(): number | undefined;
- set maxRetries(value: number | undefined);
method: AllowedHttpMethods;
noRetry: boolean;
payload?: string;
- pushErrorMessage(errorOrMessage: unknown, options?: PushErrorMessageOptions): void;
retryCount: number;
- get sessionId(): string | undefined;
- set sessionId(value: string | undefined);
- get skipNavigation(): boolean;
- set skipNavigation(value: boolean);
- get skippedReason(): SkippedRequestReason | undefined;
- set skippedReason(value: SkippedRequestReason | undefined);
- get state(): RequestState;
- set state(value: RequestState);
uniqueKey: string;
url: string;
userData: UserData;
@@ -981,26 +965,6 @@ export type RequestSourceStatus = {
status: 'finished';
};
-// @public (undocumented)
-export enum RequestState {
- // (undocumented)
- AFTER_NAV = 2,
- // (undocumented)
- BEFORE_NAV = 1,
- // (undocumented)
- DONE = 4,
- // (undocumented)
- ERROR = 6,
- // (undocumented)
- ERROR_HANDLER = 5,
- // (undocumented)
- REQUEST_HANDLER = 3,
- // (undocumented)
- SKIPPED = 7,
- // (undocumented)
- UNPROCESSED = 0
-}
-
// @public
export class RequestValidationError extends NonRetryableError {
constructor(label: string | symbol, issues: readonly SchemaIssue[]);
@@ -1087,9 +1051,6 @@ export class SessionError extends Error {
constructor(message?: string);
}
-// @public (undocumented)
-export type SkippedRequestReason = 'robotsTxt' | 'limit' | 'enqueueLimit' | 'filters' | 'transform' | 'redirect' | 'depth';
-
// @public (undocumented)
export type Source = (Partial<RequestOptions> & {
requestsFromUrl?: string;
diff --git a/docs/public-api/crawlee-http.api.md b/docs/public-api/crawlee-http.api.md
index 1f930843a..9c731589f 100644
--- a/docs/public-api/crawlee-http.api.md
+++ b/docs/public-api/crawlee-http.api.md
@@ -13,6 +13,7 @@ import { ConcurrencySystem } from '@crawlee/basic';
import type { ConcurrencySystemOptions } from '@crawlee/basic';
import { ContextPipeline } from '@crawlee/basic';
import type { CrawlingContext } from '@crawlee/basic';
+import type { CrawlingRequest } from '@crawlee/basic';
import type { Dictionary } from '@crawlee/types';
import type { EnqueueLinksOptions } from '@crawlee/basic';
import { ErrorHandler } from '@crawlee/basic';
@@ -20,8 +21,6 @@ import type { ExtractLinksOptions } from '@crawlee/basic';
import { GetUserDataFromRequest } from '@crawlee/basic';
import type { JsonValue } from 'type-fest';
import type { LoadedRequest } from '@crawlee/basic';
-import { Request as Request_2 } from '@crawlee/basic';
-import type { Request as Request_3 } from '@crawlee/core';
import { RequestHandler } from '@crawlee/basic';
import type { RequireContextPipeline } from '@crawlee/basic';
import { RouterHandler } from '@crawlee/basic';
@@ -39,7 +38,7 @@ export function ByteCounterStream(input: {
// Not exported by the entry point; reachable only as a referenced type.
// @public (undocumented)
interface CrawlingContextWithResponse<UserData extends Dictionary = any> extends CrawlingContext<UserData> {
- request: LoadedRequest<Request_2<UserData>>;
+ request: LoadedRequest<CrawlingRequest<UserData>>;
response: Response;
}
@@ -119,7 +118,7 @@ export interface FileDownloadCrawlingContext<UserData extends Dictionary = any>
encoding: BufferEncoding;
};
// (undocumented)
- request: LoadedRequest<Request_3<UserData>>;
+ request: LoadedRequest<CrawlingRequest<UserData>>;
// (undocumented)
response: Response;
}
diff --git a/docs/public-api/crawlee-playwright.api.md b/docs/public-api/crawlee-playwright.api.md
index 749617921..34e6fef4f 100644
--- a/docs/public-api/crawlee-playwright.api.md
+++ b/docs/public-api/crawlee-playwright.api.md
@@ -26,6 +26,7 @@ import type { ContextPipeline } from '@crawlee/browser';
import type { ContextPipeline as ContextPipeline_2 } from '@crawlee/basic';
import type { CrawlingContext } from '@crawlee/browser';
import type { CrawlingContext as CrawlingContext_2 } from '@crawlee/basic';
+import type { CrawlingRequest } from '@crawlee/browser';
import { Dictionary } from '@crawlee/types';
import type { Download } from 'playwright';
import type { EnqueueLinksOptions } from '@crawlee/basic';
@@ -66,7 +67,7 @@ interface AdaptiveHookContext extends Pick<AdaptivePlaywrightCrawlerContext, 'id
// (undocumented)
page?: Page;
// (undocumented)
- request: Request_3;
+ request: CrawlingRequest;
}
// @public
@@ -96,7 +97,7 @@ export interface AdaptivePlaywrightCrawlerContext<UserData extends Dictionary =
querySelector(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
querySelectorAll(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
// (undocumented)
- request: LoadedRequest<Request_3<UserData>>;
+ request: LoadedRequest<CrawlingRequest<UserData>>;
response: Response;
waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
}
@@ -127,7 +128,7 @@ export const adaptivePlaywrightCrawlerStatisticState: {
// Not exported by the entry point; reachable only as a referenced type.
// @public (undocumented)
type AdaptivePostNavigationHook<ContextExtension = Dictionary<never>> = BrowserHook<Omit<AdaptiveHookContext, 'request'> & {
- request: LoadedRequest<Request_3>;
+ request: LoadedRequest<CrawlingRequest>;
}, ContextExtension>;
// Not exported by the entry point; reachable only as a referenced type.
diff --git a/docs/upgrading/upgrading_v4.md b/docs/upgrading/upgrading_v4.md
index 6acd734ef..b0d7b3f3f 100644
--- a/docs/upgrading/upgrading_v4.md
+++ b/docs/upgrading/upgrading_v4.md
@@ -977,6 +977,15 @@ To support the new `ContextPipeline` / `extendContext` composition, the crawler
The exported handler types were reshaped accordingly. `ErrorHandler` and `RequestHandler` no longer wrap their context in `LoadedContext<...>`; `ErrorHandler` now takes two type parameters (`ErrorHandler<BaseContext, ExtendedContext>`), receiving `inputs: BaseContext & Partial<ExtendedContext>`. The `RestrictedCrawlingContext` and `LoadedContext` types are no longer exported from `@crawlee/basic`. If you imported or annotated these directly, update the references; if you only used the crawler options' `requestHandler` / `errorHandler` / `failedRequestHandler` callbacks with inferred parameter types, nothing changes.
+### `Request` is split into `Request` and `CrawlingRequest`
+
+`Request` in `@crawlee/core` is now just the stored record: URL, method, headers, payload, `userData`, `label` and the processing state a request queue keeps (`id`, `retryCount`, `errorMessages`, `handledAt`, `loadedUrl`). Everything a crawler reads off a request — `skipNavigation`, `crawlDepth`, `sessionId`, `maxRetries`, `state`, `pushErrorMessage()` — lives on `CrawlingRequest` in `@crawlee/basic`, which is what `request` is in every request handler and hook. `RequestState` moved along with it, so import it from `@crawlee/basic` (or `crawlee`), not `@crawlee/core`. Handler code is unaffected; the change only shows if you construct requests yourself or read them straight from a request manager:
+
+- The options are unchanged: `new Request({ url, skipNavigation: true, maxRetries: 2 })` and `queue.addRequest({ url, sessionId })` still record the settings, they just are not readable as properties on the core class. Where you need them, `CrawlingRequest.fromSchema(request)` rebuilds the crawler view.
+- `RequestOptions` no longer accepts `id` or `handledAt`, and the constructor ignores `retryCount`, `errorMessages` and `loadedUrl`. A request coming back from a storage is rebuilt with `Request.fromSchema(record)`, which is also what `RequestQueue` does internally.
+- `RequestOptions.skippedReason` and `Request.skippedReason` are removed. They were an internal side-channel for reporting `maxCrawlDepth` skips out of a `transformRequestFunction`; the depth check no longer runs inside the transform, so nothing writes them.
+- `crawlingContext.request` is never the object your request manager handed out — the crawler rebuilds it as a `CrawlingRequest`. With a `RequestQueue` that was already true (requests are rebuilt from the stored record on every fetch), and in v4 `RequestList` and `SitemapRequestLoader` feed the crawler through a queue too; the change is only observable with a custom `IRequestManager` whose `fetchNextRequest()` returned its own instances. The object passed back to `reclaimRequest()` / `markRequestAsHandled()` is the crawler's copy, so match on `uniqueKey` (or `id`), not by identity.
+
### Navigation hook types are now generic
`PlaywrightHook`, `PuppeteerHook` and `StagehandHook` are now type aliases (previously interfaces) generic over the request's `userData` type. A hook that types its context — via the generic (e.g. `PlaywrightHook<MyUserData>`) or an explicit context annotation — is now assignable to the `preNavigationHooks` / `postNavigationHooks` options of an untyped crawler. If you extended one of these interfaces, use an intersection type instead.
diff --git a/packages/basic-crawler/src/index.ts b/packages/basic-crawler/src/index.ts
index a0dc62e53..d974669fc 100644
--- a/packages/basic-crawler/src/index.ts
+++ b/packages/basic-crawler/src/index.ts
@@ -3,6 +3,7 @@ export * from './internals/autoscaling/index.js';
export * from './internals/basic-crawler.js';
export * from './internals/cookie_utils.js';
export * from './internals/crawlers/index.js';
+export * from './internals/crawling_request.js';
export * from './internals/enqueue_links/index.js';
export * from './internals/errors.js';
export * from './internals/http.js';
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
index 5e3f6c832..f2c5c9be2 100644
--- a/packages/basic-crawler/src/internals/basic-crawler.ts
+++ b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -12,7 +12,6 @@ import type {
IRequestManager,
RequestOptions,
RequestsLike,
- SkippedRequestReason,
Source,
StorageIdentifier,
StorageWritePolicy,
@@ -36,7 +35,6 @@ import {
purgeDefaultStorages,
RequestManagerTandem,
RequestQueue,
- RequestState,
ServiceLocator,
serviceLocator,
SessionError,
@@ -70,6 +68,8 @@ import type { FinalStatistics } from './autoscaling/system_status.js';
import { mergeCookies } from './cookie_utils.js';
import { ContextPipeline } from './crawlers/context_pipeline.js';
import type { CrawlingContext, TypedRequestsLike } from './crawlers/crawler_commons.js';
+import type { CrawlingRequestData, SkippedRequestReason } from './crawling_request.js';
+import { CrawlingRequest, RequestState } from './crawling_request.js';
import {
ContextPipelineCleanupError,
ContextPipelineInitializationError,
@@ -182,7 +182,7 @@ const addRequestsOptionsSchema = z.looseObject({
});
/** The in-flight context, carrying the timeout slots ({@apilink raceWithTimeout} hangs its extender on them). */
-type PendingCrawlingContext = { request: Request } & Partial<CrawlingContext> & RequestTimeoutContext;
+type PendingCrawlingContext = { request: CrawlingRequest } & Partial<CrawlingContext> & RequestTimeoutContext;
export type RequestHandler<Context extends CrawlingContext = CrawlingContext> = (inputs: Context) => Awaitable<void>;
@@ -243,7 +243,7 @@ export interface BasicCrawlerOptions<
* To make this work, we should **always**
* let our function throw exceptions rather than catch them.
* The exceptions are logged to the request using the
- * {@apilink Request.pushErrorMessage|`Request.pushErrorMessage()`} function.
+ * {@apilink CrawlingRequest.pushErrorMessage|`request.pushErrorMessage()`} function.
*/
requestHandler?: RouterHandler<ExtendedContext, Routes> | RequestHandler<ExtendedContext>;
@@ -811,7 +811,7 @@ export class BasicCrawler<
Routes
>;
- #basicContextPipeline?: ContextPipeline<{ request: Request }, CrawlingContext>;
+ #basicContextPipeline?: ContextPipeline<{ request: CrawlingRequest }, CrawlingContext>;
/**
* The basic part of the context pipeline. Unlike the subclass pipeline, this
@@ -821,7 +821,7 @@ export class BasicCrawler<
* Context built with this pipeline can be passed into multiple crawler pipelines at once.
* This is used e.g. in the {@apilink AdaptivePlaywrightCrawler|`AdaptivePlaywrightCrawler`}.
*/
- get basicContextPipeline(): ContextPipeline<{ request: Request }, CrawlingContext> {
+ get basicContextPipeline(): ContextPipeline<{ request: CrawlingRequest }, CrawlingContext> {
if (this.#basicContextPipeline === undefined) {
this.#basicContextPipeline = this.buildBasicContextPipeline();
}
@@ -1237,7 +1237,7 @@ export class BasicCrawler<
// matching `recordRequestStart`.
this.statistics.recordRequestStart(request.id || request.uniqueKey);
- const crawlingContext = { request } as { request: Request } & Partial<CrawlingContext>;
+ const crawlingContext = { request } as { request: CrawlingRequest } & Partial<CrawlingContext>;
try {
// The transaction spans the whole pipeline call, covering the navigation hooks
// and `extendContext` too; `handleRequest` drives its outcome explicitly.
@@ -1419,15 +1419,15 @@ export class BasicCrawler<
* Builds the basic context pipeline that transforms `{ request }` into a full `CrawlingContext`.
* This handles base context creation, session resolution, and context helpers.
*/
- private buildBasicContextPipeline(): ContextPipeline<{ request: Request }, CrawlingContext> {
- return ContextPipeline.create<{ request: Request }>()
+ private buildBasicContextPipeline(): ContextPipeline<{ request: CrawlingRequest }, CrawlingContext> {
+ return ContextPipeline.create<{ request: CrawlingRequest }>()
.compose({ action: this.checkRobotsTxt.bind(this) })
.compose({ action: (context) => this.createBaseContext(context) })
.compose({ action: this.resolveSession.bind(this) })
.compose({ action: this.createContextHelpers.bind(this) });
}
- private async checkRobotsTxt({ request }: { request: Request }) {
+ private async checkRobotsTxt({ request }: { request: CrawlingRequest }) {
if (!(await this.isAllowedBasedOnRobotsTxtFile(request.url))) {
this.log.warning(
`Skipping request ${request.url} (${request.id}) because it is disallowed based on robots.txt`,
@@ -1495,22 +1495,30 @@ export class BasicCrawler<
};
}
- private async resolveRequest(): Promise<Request | null> {
- const request = await this.timeoutAndRetry(
+ /**
+ * Fetches the next request and rebuilds it as a {@apilink CrawlingRequest}. A copy rather than the manager's
+ * instance: managers key on `uniqueKey`/`id`, and it is this copy that flows back into `reclaimRequest` /
+ * `markRequestAsHandled`, so nothing observes the difference.
+ */
+ private async resolveRequest(): Promise<CrawlingRequest | null> {
+ const fetched = await this.timeoutAndRetry(
this.fetchNextRequest.bind(this),
this.internalTimeoutMillis,
`Fetching next request timed out after ${this.internalTimeoutMillis / 1e3} seconds.`,
);
- // Reset loadedUrl so an old one is not carried over to retries.
- if (request) {
- request.loadedUrl = undefined;
+ if (!fetched) {
+ return null;
}
+ const request = CrawlingRequest.fromSchema(fetched);
+ // Reset loadedUrl so an old one is not carried over to retries.
+ request.loadedUrl = undefined;
+
return request;
}
- private async resolveSession({ request }: { request: Request }) {
+ private async resolveSession({ request }: { request: CrawlingRequest }) {
const session = await this.timeoutAndRetry(
async () => {
const existingSession = await this.sessionPool.getSession(request.sessionId);
@@ -1528,7 +1536,7 @@ export class BasicCrawler<
return { session, proxyInfo: session?.proxyInfo };
}
- private async createContextHelpers({ request, session }: { request: Request; session: ISession }) {
+ private async createContextHelpers({ request, session }: { request: CrawlingRequest; session: ISession }) {
const addRequests: CrawlingContext['addRequests'] = async (requests, options = {}) => {
const newCrawlDepth = request!.crawlDepth + 1;
const requestsGenerator = this.addCrawlDepthRequestGenerator(requests, newCrawlDepth);
@@ -2219,7 +2227,7 @@ export class BasicCrawler<
if (options.transformRequestFunction) {
const transformed = applyRequestTransform([finalOptions], options.transformRequestFunction, (r) =>
- allSkipped.push({ source: keepSkippedSource(r), reason: r.skippedReason ?? 'transform' }),
+ allSkipped.push({ source: keepSkippedSource(r), reason: 'transform' }),
);
if (transformed.length === 0) {
@@ -2622,7 +2630,11 @@ export class BasicCrawler<
}
/** Handles a single request - runs the request handler with retries, error handling, and lifecycle management. */
- private async handleRequest(crawlingContext: ExtendedContext, requestSource: IRequestManager, request: Request) {
+ private async handleRequest(
+ crawlingContext: ExtendedContext,
+ requestSource: IRequestManager,
+ request: CrawlingRequest,
+ ) {
// An earlier phase we cannot cancel (e.g. a slow `extendContext`) may have run past the internal timeout,
// which already failed the request in `runTaskFunction`. Bail before running the handler so it does not
// execute (and re-report) on top of a request the crawler has already moved past.
@@ -2729,6 +2741,10 @@ export class BasicCrawler<
for await (const request of requests) {
if (typeof request === 'string') {
yield { url: request, crawlDepth: newRequestDepth };
+ } else if (request instanceof Request) {
+ const crawlee: CrawlingRequestData = ((request.userData as Dictionary).__crawlee ??= {});
+ crawlee.crawlDepth ??= newRequestDepth;
+ yield request;
} else {
request.crawlDepth ??= newRequestDepth;
yield request;
@@ -2801,7 +2817,7 @@ export class BasicCrawler<
private async requestFunctionErrorHandler(
error: Error,
crawlingContext: CrawlingContext,
- request: Request,
+ request: CrawlingRequest,
source: IRequestManager,
): Promise<void> {
if (error instanceof RequestThrottledError) {
@@ -2923,7 +2939,7 @@ export class BasicCrawler<
return error instanceof SessionError || error instanceof RequestThrottledError;
}
- private canRequestBeRetried(request: Request, error: Error) {
+ private canRequestBeRetried(request: CrawlingRequest, error: Error) {
// Request should never be retried, or the error encountered makes it not able to be retried.
if (request.noRetry || error instanceof NonRetryableError) {
return false;
@@ -2995,7 +3011,7 @@ export class BasicCrawler<
return request.headers?.Cookie || request.headers?.cookie || '';
}
- private requestMatchesEnqueueStrategy(request: Request) {
+ private requestMatchesEnqueueStrategy(request: CrawlingRequest) {
// If `skipNavigation` was used, just return `true`
try {
// eslint-disable-next-line @typescript-eslint/no-unused-expressions
@@ -3010,8 +3026,7 @@ export class BasicCrawler<
const { url, loadedUrl } = request;
- // eslint-disable-next-line dot-notation -- private access
- const strategy = request['enqueueStrategy'];
+ const strategy = request.enqueueStrategy;
// No strategy set, so we assume it matches, or it was added outside of enqueueLinks
if (!strategy) {
@@ -3055,7 +3070,7 @@ export class BasicCrawler<
}
export interface CreateContextOptions {
- request: Request;
+ request: CrawlingRequest;
session: ISession;
proxyInfo?: ProxyInfo;
}
diff --git a/packages/basic-crawler/src/internals/crawlers/crawler_commons.ts b/packages/basic-crawler/src/internals/crawlers/crawler_commons.ts
index 3cc5fba4a..8843397b2 100644
--- a/packages/basic-crawler/src/internals/crawlers/crawler_commons.ts
+++ b/packages/basic-crawler/src/internals/crawlers/crawler_commons.ts
@@ -20,6 +20,7 @@ import type {
} from '@crawlee/core';
import type { EnqueueUrlsOptions } from '../enqueue_links/index.js';
+import type { CrawlingRequest } from '../crawling_request.js';
/** @internal */
export type IsAny<T> = 0 extends 1 & T ? true : false;
@@ -114,9 +115,9 @@ export interface RestrictedCrawlingContext<UserData extends Dictionary = Diction
proxyInfo?: ProxyInfo;
/**
- * The original {@apilink Request} object.
+ * The request being processed.
*/
- request: Request<UserData>;
+ request: CrawlingRequest<UserData>;
/**
* This function allows you to push data to a {@apilink Dataset} specified by name, or the one currently used by the crawler.
diff --git a/packages/basic-crawler/src/internals/crawling_request.ts b/packages/basic-crawler/src/internals/crawling_request.ts
new file mode 100644
index 000000000..2b2ab87a7
--- /dev/null
+++ b/packages/basic-crawler/src/internals/crawling_request.ts
@@ -0,0 +1,171 @@
+import util from 'node:util';
+
+import type { EnqueueStrategyOption } from '@crawlee/core';
+import { Request } from '@crawlee/core';
+import type { Dictionary, RequestSchema } from '@crawlee/types';
+
+export type SkippedRequestReason =
+ | 'robotsTxt'
+ | 'limit'
+ | 'enqueueLimit'
+ | 'filters'
+ | 'transform'
+ | 'redirect'
+ | 'depth';
+
+export enum RequestState {
+ UNPROCESSED,
+ BEFORE_NAV,
+ AFTER_NAV,
+ REQUEST_HANDLER,
+ DONE,
+ ERROR_HANDLER,
+ ERROR,
+ SKIPPED,
+}
+
+export interface PushErrorMessageOptions {
+ /**
+ * Only push the error message without stack trace when true.
+ * @default false
+ */
+ omitStack?: boolean;
+}
+
+/**
+ * The crawler's per-request state, persisted inside `userData.__crawlee` so every storage carries it as plain user
+ * data. The settings are written by the `Request` constructor from `RequestOptions`; `state` only by a running crawler.
+ * @internal
+ */
+export interface CrawlingRequestData {
+ skipNavigation?: boolean;
+ crawlDepth?: number;
+ sessionId?: string;
+ maxRetries?: number;
+ enqueueStrategy?: EnqueueStrategyOption;
+ state?: RequestState;
+}
+
+/**
+ * A {@apilink Request} as seen by a crawler: the same stored record, with the crawler's own per-request state
+ * (`userData.__crawlee`) exposed as properties. This is what `request` is in every request handler.
+ *
+ * Requests fetched from a request manager are plain {@apilink Request}s; {@apilink BasicCrawler} rebuilds them
+ * as `CrawlingRequest`s before handing them to the pipeline.
+ * @category Sources
+ */
+export class CrawlingRequest<UserData extends Dictionary = Dictionary> extends Request<UserData> {
+ static override fromSchema<UserData extends Dictionary = Dictionary>(
+ schema: RequestSchema,
+ ): CrawlingRequest<UserData> {
+ return super.fromSchema(schema) as CrawlingRequest<UserData>;
+ }
+
+ private get crawleeData(): CrawlingRequestData {
+ return ((this.userData as Dictionary).__crawlee ??= {});
+ }
+
+ /**
+ * Tells the crawler processing this request to skip the navigation and process the request directly.
+ *
+ * When this is set to `true`, the crawling context will not contain the results of the navigation
+ * (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
+ * Accessing these properties will throw a {@apilink NavigationSkippedError} at runtime.
+ */
+ get skipNavigation(): boolean {
+ return this.crawleeData.skipNavigation ?? false;
+ }
+
+ set skipNavigation(value: boolean) {
+ this.crawleeData.skipNavigation = value;
+ }
+
+ /**
+ * Depth of the request in the current crawl tree.
+ * Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
+ */
+ get crawlDepth(): number {
+ return this.crawleeData.crawlDepth ?? 0;
+ }
+
+ set crawlDepth(value: number) {
+ this.crawleeData.crawlDepth = value;
+ }
+
+ /** ID of a session to use for this request. When set, the crawler will fetch this session from the session pool instead of creating a new one. */
+ get sessionId(): string | undefined {
+ return this.crawleeData.sessionId;
+ }
+
+ set sessionId(value: string | undefined) {
+ this.crawleeData.sessionId = value;
+ }
+
+ /** Maximum number of retries for this request. Allows to override the global `maxRequestRetries` option of `BasicCrawler`. */
+ get maxRetries(): number | undefined {
+ return this.crawleeData.maxRetries;
+ }
+
+ set maxRetries(value: number | undefined) {
+ this.crawleeData.maxRetries = value;
+ }
+
+ /** Describes the request's current lifecycle state. */
+ get state(): RequestState {
+ return this.crawleeData.state ?? RequestState.UNPROCESSED;
+ }
+
+ set state(value: RequestState) {
+ this.crawleeData.state = value;
+ }
+
+ /** The strategy the request was enqueued under, if it came from `enqueueLinks`. @internal */
+ get enqueueStrategy(): EnqueueStrategyOption | undefined {
+ return this.crawleeData.enqueueStrategy;
+ }
+
+ /**
+ * Stores information about an error that occurred during processing of this request.
+ *
+ * You should always use Error instances when throwing errors in JavaScript.
+ *
+ * Nevertheless, to improve the debugging experience when using third party libraries
+ * that may not always throw an Error instance, the function performs a type
+ * inspection of the passed argument and attempts to extract as much information
+ * as possible, since just throwing a bad type error makes any debugging rather difficult.
+ *
+ * @param errorOrMessage Error object or error message to be stored in the request.
+ * @param [options]
+ */
+ pushErrorMessage(errorOrMessage: unknown, options: PushErrorMessageOptions = {}): void {
+ const { omitStack } = options;
+ let message;
+ const type = typeof errorOrMessage;
+ if (type === 'object') {
+ if (!errorOrMessage) {
+ message = 'null';
+ } else if (errorOrMessage instanceof Error) {
+ message = omitStack
+ ? errorOrMessage.message
+ : // .stack includes the message
+ errorOrMessage.stack;
+ } else if (Reflect.has(Object(errorOrMessage), 'message')) {
+ message = Reflect.get(Object(errorOrMessage), 'message');
+ } else if ((errorOrMessage as string).toString() !== '[object Object]') {
+ message = (errorOrMessage as string).toString();
+ } else {
+ try {
+ message = util.inspect(errorOrMessage);
+ } catch (err) {
+ message = 'Unable to extract any message from the received object.';
+ }
+ }
+ } else if (type === 'undefined') {
+ message = 'undefined';
+ } else {
+ message = (errorOrMessage as string).toString();
+ }
+
+ this.errorMessages.push(message);
+ }
+}
diff --git a/packages/basic-crawler/src/internals/enqueue_links/shared.ts b/packages/basic-crawler/src/internals/enqueue_links/shared.ts
index 16274c7bf..c92263a51 100644
--- a/packages/basic-crawler/src/internals/enqueue_links/shared.ts
+++ b/packages/basic-crawler/src/internals/enqueue_links/shared.ts
@@ -4,10 +4,12 @@ import type { Awaitable, Dictionary } from '@crawlee/types';
import { Minimatch } from 'minimatch';
import { z } from 'zod';
-import type { EnqueueStrategyOption, RequestOptions, SkippedRequestReason, Source } from '@crawlee/core';
+import type { EnqueueStrategyOption, RequestOptions, Source } from '@crawlee/core';
import { Request } from '@crawlee/core';
import { schemas } from '@crawlee/utils/internal';
+import type { SkippedRequestReason } from '../crawling_request.js';
+
const MAX_ENQUEUE_LINKS_CACHE_SIZE = 1000;
/**
@@ -60,11 +62,10 @@ export function createSkippedRequestArgs(
source: string | Source,
reason: SkippedRequestReason,
): Parameters<SkippedRequestCallback>[0] {
- const sourceReason = typeof source === 'string' ? undefined : source.skippedReason;
let request: Request | undefined;
return {
- reason: sourceReason ?? reason,
+ reason,
get request() {
request ??=
source instanceof Request
diff --git a/packages/basic-crawler/src/internals/router.ts b/packages/basic-crawler/src/internals/router.ts
index 75d754a23..5af896513 100644
--- a/packages/basic-crawler/src/internals/router.ts
+++ b/packages/basic-crawler/src/internals/router.ts
@@ -1,4 +1,4 @@
-import type { Request } from '@crawlee/core';
+import type { CrawlingRequest } from './crawling_request.js';
import { RequestValidationError } from '@crawlee/core';
import type { Awaitable, Dictionary } from '@crawlee/types';
import type { StandardSchemaV1 } from '@standard-schema/spec';
@@ -29,7 +29,7 @@ export type RouterHandlerContext<
UserData extends Dictionary,
Routes extends Record<keyof Routes, Dictionary>,
> = Omit<Context, 'request' | 'addRequests' | 'enqueueLinks'> & {
- request: LoadedRequest<Request<UserData>>;
+ request: LoadedRequest<CrawlingRequest<UserData>>;
addRequests: TypedContextAddRequests<Routes>;
} & (Context extends { enqueueLinks: infer EnqueueLinks }
? { enqueueLinks: TypedContextEnqueueLinks<EnqueueLinks, Routes> }
@@ -129,7 +129,7 @@ export interface RouterHandler<
(ctx: Context): Awaitable<void>;
}
-export type GetUserDataFromRequest<T> = T extends Request<infer Y> ? Y : never;
+export type GetUserDataFromRequest<T> = T extends CrawlingRequest<infer Y> ? Y : never;
/**
* Per-route overrides, passed as the last argument of {@apilink Router.addHandler|`addHandler`} and
@@ -147,7 +147,9 @@ export interface RouteOptions {
}
export type RouterRoutes<Context, Routes extends Record<keyof Routes, Dictionary>> = {
- [Label in keyof Routes]: (ctx: Omit<Context, 'request'> & { request: Request<Routes[Label]> }) => Awaitable<void>;
+ [Label in keyof Routes]: (
+ ctx: Omit<Context, 'request'> & { request: CrawlingRequest<Routes[Label]> },
+ ) => Awaitable<void>;
};
/**
diff --git a/packages/basic-crawler/src/internals/throttling_request_manager.ts b/packages/basic-crawler/src/internals/throttling_request_manager.ts
index a0b32de39..1c753b04b 100644
--- a/packages/basic-crawler/src/internals/throttling_request_manager.ts
+++ b/packages/basic-crawler/src/internals/throttling_request_manager.ts
@@ -1160,7 +1160,7 @@ export class ThrottlingRequestManager<T extends IRequestManager = IRequestManage
async #migrateToSubQueue(request: Request, inner: T): Promise<void> {
try {
const subManager = await this.#selectManagerOrThrow(request.url);
- await subManager.addRequest(new Request({ ...request, id: undefined }));
+ await subManager.addRequest(Request.fromSchema({ ...request, id: undefined }));
} catch (error) {
// Whatever the reason, the request is out of the wrapped manager and in nobody's hands, so it
// goes back before the failure is raised.
diff --git a/packages/basic-crawler/test/sitemap_request_loader.test.ts b/packages/basic-crawler/test/sitemap_request_loader.test.ts
index a4b6b9aa8..8b733a814 100644
--- a/packages/basic-crawler/test/sitemap_request_loader.test.ts
+++ b/packages/basic-crawler/test/sitemap_request_loader.test.ts
@@ -3,7 +3,7 @@ import type { AddressInfo } from 'node:net';
import { Readable } from 'node:stream';
import { finished } from 'node:stream/promises';
-import { SitemapRequestLoader } from '@crawlee/basic';
+import { CrawlingRequest, SitemapRequestLoader } from '@crawlee/basic';
import { MemoryStorageBackend, type Request, serviceLocator } from '@crawlee/core';
import { sleep } from '@crawlee/utils';
import express from 'express';
@@ -671,7 +671,7 @@ describe('SitemapRequestLoader', () => {
expect(request).not.toBe(null);
// The strategy is persisted on the request so it keeps being enforced after navigation.
- expect((request as any).enqueueStrategy).toBe('same-hostname');
+ expect(CrawlingRequest.fromSchema(request!).enqueueStrategy).toBe('same-hostname');
});
test('persistState does not deadlock a backpressured sitemap load', async () => {
diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.ts
index 7722871da..3d093a99d 100644
--- a/packages/browser-crawler/src/internals/browser-crawler.ts
+++ b/packages/browser-crawler/src/internals/browser-crawler.ts
@@ -9,7 +9,7 @@ import type {
ExtractLinksOptions,
GetUserDataFromRequest,
LoadedRequest,
- Request,
+ CrawlingRequest,
RequestHandler,
RouterHandler,
} from '@crawlee/basic';
@@ -97,7 +97,7 @@ export interface BrowserCrawlingContext<
/**
* The request object that was successfully loaded and navigated to, including the {@apilink Request.loadedUrl|`loadedUrl`} property.
*/
- request: LoadedRequest<Request<UserData>>;
+ request: LoadedRequest<CrawlingRequest<UserData>>;
/**
* The HTTP response object returned by the browser's navigation.
@@ -197,7 +197,7 @@ export interface BrowserCrawlerOptions<
* To make this work, we should **always**
* let our function throw exceptions rather than catch them.
* The exceptions are logged to the request using the
- * {@apilink Request.pushErrorMessage|`Request.pushErrorMessage()`} function.
+ * {@apilink CrawlingRequest.pushErrorMessage|`request.pushErrorMessage()`} function.
*/
requestHandler?: RouterHandler<ExtendedContext, Routes> | RequestHandler<ExtendedContext>;
@@ -646,7 +646,7 @@ export abstract class BrowserCrawler<
}
return Reflect.get(target, propertyName, receiver);
},
- }) as LoadedRequest<Request>,
+ }) as LoadedRequest<CrawlingRequest>,
get response(): Response {
throw new NavigationSkippedError(
'The `response` property is not available - `skipNavigation` was used',
@@ -720,7 +720,7 @@ export abstract class BrowserCrawler<
// Cookies set during `requestHandler` are saved again afterwards.
await this.persistCookiesFromPage(crawlingContext);
- return { request: crawlingContext.request as LoadedRequest<Request> } as Partial<Context>;
+ return { request: crawlingContext.request as LoadedRequest<CrawlingRequest> } as Partial<Context>;
}
/**
diff --git a/packages/core/src/request.ts b/packages/core/src/request.ts
index af53e08ad..957caba82 100644
--- a/packages/core/src/request.ts
+++ b/packages/core/src/request.ts
@@ -1,8 +1,7 @@
import type { BinaryLike } from 'node:crypto';
import crypto from 'node:crypto';
-import util from 'node:util';
-import type { AllowedHttpMethods, Dictionary } from '@crawlee/types';
+import type { AllowedHttpMethods, Dictionary, RequestSchema } from '@crawlee/types';
import type { EnqueueStrategy } from '@crawlee/utils';
import { z } from 'zod';
@@ -15,48 +14,31 @@ import { parseArgument, schemas } from './validators.js';
/** The `strategy` option accepted by {@apilink ExtractLinksOptions} and {@apilink EnqueueUrlsOptions}. */
export type EnqueueStrategyOption = EnqueueStrategy | 'all' | 'same-domain' | 'same-hostname' | 'same-origin';
-export type SkippedRequestReason =
- | 'robotsTxt'
- | 'limit'
- | 'enqueueLimit'
- | 'filters'
- | 'transform'
- | 'redirect'
- | 'depth';
-
-const dateString = z.string().refine((value) => !Number.isNaN(Date.parse(value)), {
- message: 'Invalid input: expected a date string',
-});
-
-export enum RequestState {
- UNPROCESSED,
- BEFORE_NAV,
- AFTER_NAV,
- REQUEST_HANDLER,
- DONE,
- ERROR_HANDLER,
- ERROR,
- SKIPPED,
+/**
+ * What the constructor writes into `userData.__crawlee` from {@apilink RequestOptions}. The crawler layer owns
+ * the full shape (`CrawlingRequestData` in `@crawlee/basic`); this only types the writes made here.
+ */
+interface CrawleeRequestData {
+ skipNavigation?: boolean;
+ crawlDepth?: number;
+ sessionId?: string;
+ maxRetries?: number;
+ enqueueStrategy?: EnqueueStrategyOption;
}
const requestUrlSchema = z.object({ url: z.string() });
// new properties on the Request object breaks serialization
const requestOptionalSchemaShapes: Record<string, z.ZodType> = {
- id: z.string().optional(),
- loadedUrl: z.url().optional(),
uniqueKey: z.string().optional(),
method: z.string().optional(),
payload: z.union([z.string(), z.instanceof(Uint8Array)]).optional(),
noRetry: z.boolean().optional(),
- retryCount: schemas.anyNumber.optional(),
sessionId: z.string().optional(),
maxRetries: schemas.anyNumber.optional(),
- errorMessages: schemas.arrayOf(z.string(), 'strings').optional(),
headers: z.looseObject({}).optional(),
userData: z.looseObject({}).optional(),
label: z.string().optional(),
- handledAt: z.union([dateString, z.date()]).optional(),
keepUrlFragment: z.boolean().optional(),
useExtendedUniqueKey: z.boolean().optional(),
alwaysEnqueue: z.boolean().optional(),
@@ -64,7 +46,6 @@ const requestOptionalSchemaShapes: Record<string, z.ZodType> = {
crawlDepth: schemas.anyNumber
.refine((value) => value >= 0, 'Expected a number greater than or equal to 0')
.optional(),
- state: z.enum(RequestState).optional(),
};
// Each schema is wrapped in a single-key object so validation errors carry the property name.
@@ -73,8 +54,9 @@ const requestOptionalSchemas: Partial<Record<string, z.ZodType>> = Object.fromEn
);
/**
- * Represents a URL to be crawled, optionally including HTTP method, headers, payload and other metadata.
- * The `Request` object also stores information about errors that occurred during processing of the request.
+ * Represents a URL to be crawled, optionally including HTTP method, headers, payload and other metadata, together
+ * with the processing state a request queue keeps about it (`retryCount`, `errorMessages`, `handledAt`, ...).
+ * Inside a crawler, requests are {@apilink CrawlingRequest}s, which add the crawler's own per-request settings.
*
* Each `Request` instance has the `uniqueKey` property, which can be either specified
* manually in the constructor or generated automatically from the URL. Two requests with the same `uniqueKey`
@@ -90,21 +72,15 @@ const requestOptionalSchemas: Partial<Record<string, z.ZodType>> = Object.fromEn
* const request = new Request({
* url: 'http://www.example.com',
* headers: { Accept: 'application/json' },
+ * userData: { foo: 'bar' },
* });
*
- * ...
- *
- * request.userData.foo = 'bar';
- * request.pushErrorMessage(new Error('Request failed!'));
- *
- * ...
- *
- * const foo = request.userData.foo;
+ * await requestQueue.addRequest(request);
* ```
* @category Sources
*/
class CrawleeRequest<UserData extends Dictionary = Dictionary> {
- /** Request ID */
+ /** Storage-assigned request ID. Only present on requests that went through a {@apilink RequestQueue}. */
id?: string;
/** URL of the web page to crawl. */
@@ -163,6 +139,9 @@ class CrawleeRequest<UserData extends Dictionary = Dictionary> {
/**
* `Request` parameters including the URL, HTTP method and headers, and others.
+ *
+ * Processing state (`retryCount`, `errorMessages`, `handledAt`, `loadedUrl`, `id`) is not an option;
+ * requests coming back from a storage are rebuilt with {@apilink Request.fromSchema}.
*/
constructor(options: RequestOptions<UserData>) {
// A bare URL is a common slip — point at the object form instead of a generic type error.
@@ -193,33 +172,22 @@ class CrawleeRequest<UserData extends Dictionary = Dictionary> {
});
const {
- id,
url,
- loadedUrl,
uniqueKey,
payload,
noRetry = false,
- retryCount = 0,
sessionId,
maxRetries,
- errorMessages = [],
headers = {},
- userData = {},
+ userData = {} as UserData,
label,
- handledAt,
keepUrlFragment = false,
useExtendedUniqueKey = false,
alwaysEnqueue = false,
skipNavigation,
enqueueStrategy,
crawlDepth,
- } = options as RequestOptions & {
- loadedUrl?: string;
- retryCount?: number;
- sessionId?: string;
- errorMessages?: string[];
- handledAt?: string | Date;
- };
+ } = options;
let { method = 'GET' } = options;
@@ -231,9 +199,7 @@ class CrawleeRequest<UserData extends Dictionary = Dictionary> {
throw new Error('`alwaysEnqueue` cannot be used together with a custom `uniqueKey`.');
}
- this.id = id;
this.url = url;
- this.loadedUrl = loadedUrl;
this.uniqueKey =
uniqueKey ||
CrawleeRequest.computeUniqueKey({
@@ -247,13 +213,12 @@ class CrawleeRequest<UserData extends Dictionary = Dictionary> {
this.method = method;
this.payload = payload;
this.noRetry = noRetry;
- this.retryCount = retryCount;
- this.errorMessages = [...errorMessages];
+ this.retryCount = 0;
+ this.errorMessages = [];
this.headers = { ...headers };
- this.handledAt = (handledAt as unknown) instanceof Date ? (handledAt as Date).toISOString() : handledAt!;
if (label) {
- userData.label = label;
+ (userData as Dictionary).label = label;
}
// Read `__crawlee` explicitly - on a `userData` coming from another Request instance the
@@ -296,15 +261,30 @@ class CrawleeRequest<UserData extends Dictionary = Dictionary> {
// reassign userData to ensure internal `__crawlee` object is non-enumerable
this.userData = userData;
- if (skipNavigation != null) this.skipNavigation = skipNavigation;
- if (maxRetries != null) this.maxRetries = maxRetries;
- if (crawlDepth != null) this.userData.__crawlee.crawlDepth ??= crawlDepth;
- if (sessionId) this.sessionId = sessionId;
+ const crawlee: CrawleeRequestData = ((this.userData as Dictionary).__crawlee ??= {});
+ if (skipNavigation != null) crawlee.skipNavigation = skipNavigation;
+ if (maxRetries != null) crawlee.maxRetries = maxRetries;
+ if (crawlDepth != null) crawlee.crawlDepth ??= crawlDepth;
+ if (sessionId) crawlee.sessionId = sessionId;
+ // A stored request keeps the strategy it was enqueued under.
+ if (enqueueStrategy) crawlee.enqueueStrategy ??= enqueueStrategy;
+ }
- // If it's already set, don't override it (for instance when fetching from storage)
- if (enqueueStrategy) {
- this.enqueueStrategy ??= enqueueStrategy;
- }
+ /**
+ * Rebuilds a request from its stored form, including the processing state a
+ * {@apilink RequestOptions} object cannot carry. Subclasses get instances of themselves.
+ */
+ static fromSchema<UserData extends Dictionary = Dictionary>(schema: RequestSchema): CrawleeRequest<UserData> {
+ const { id, retryCount = 0, errorMessages = [], handledAt, loadedUrl, ...options } = schema;
+ const request = new this(options as RequestOptions<UserData>);
+ request.id = id;
+ request.retryCount = retryCount;
+ request.errorMessages = [...errorMessages];
+ request.loadedUrl = loadedUrl;
+ // `apify-client` parses `*At` fields into `Date`s, whatever the schema says.
+ request.handledAt =
+ (handledAt as unknown) instanceof Date ? (handledAt as unknown as Date).toISOString() : handledAt;
+ return request;
}
/**
@@ -319,59 +299,6 @@ class CrawleeRequest<UserData extends Dictionary = Dictionary> {
});
}
- /**
- * Tells the crawler processing this request to skip the navigation and process the request directly.
- *
- * When this is set to `true`, the crawling context will not contain the results of the navigation
- * (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
- * Accessing these properties will throw a {@apilink NavigationSkippedError} at runtime.
- */
- get skipNavigation(): boolean {
- return this.userData.__crawlee?.skipNavigation ?? false;
- }
-
- /**
- * Tells the crawler processing this request to skip the navigation and process the request directly.
- *
- * When this is set to `true`, the crawling context will not contain the results of the navigation
- * (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
- * Accessing these properties will throw a {@apilink NavigationSkippedError} at runtime.
- */
- set skipNavigation(value: boolean) {
- if (!this.userData.__crawlee) {
- (this.userData as Dictionary).__crawlee = { skipNavigation: value };
- } else {
- this.userData.__crawlee.skipNavigation = value;
- }
- }
-
- /**
- * Depth of the request in the current crawl tree.
- * Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
- */
- get crawlDepth(): number {
- return this.userData.__crawlee?.crawlDepth ?? 0;
- }
-
- /**
- * Depth of the request in the current crawl tree.
- * Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
- */
- set crawlDepth(value: number) {
- (this.userData as Dictionary).__crawlee ??= {};
- this.userData.__crawlee.crawlDepth = value;
- }
-
- /** ID of a session to use for this request. When set, the crawler will fetch this session from the session pool instead of creating a new one. */
- get sessionId(): string | undefined {
- return this.userData.__crawlee?.sessionId;
- }
-
- set sessionId(value: string | undefined) {
- (this.userData as Dictionary).__crawlee ??= {};
- this.userData.__crawlee.sessionId = value;
- }
-
/** shortcut for getting `request.userData.label` */
get label(): string | undefined {
return this.userData.label;
@@ -382,109 +309,6 @@ class CrawleeRequest<UserData extends Dictionary = Dictionary> {
(this.userData as Dictionary).label = value;
}
- /** Maximum number of retries for this request. Allows to override the global `maxRequestRetries` option of `BasicCrawler`. */
- get maxRetries(): number | undefined {
- return this.userData.__crawlee?.maxRetries;
- }
-
- /** Maximum number of retries for this request. Allows to override the global `maxRequestRetries` option of `BasicCrawler`. */
- set maxRetries(value: number | undefined) {
- if (!this.userData.__crawlee) {
- (this.userData as Dictionary).__crawlee = { maxRetries: value };
- } else {
- this.userData.__crawlee.maxRetries = value;
- }
- }
-
- /** Describes the request's current lifecycle state. */
- get state(): RequestState {
- return this.userData.__crawlee?.state ?? RequestState.UNPROCESSED;
- }
-
- /** Describes the request's current lifecycle state. */
- set state(value: RequestState) {
- if (!this.userData.__crawlee) {
- (this.userData as Dictionary).__crawlee = { state: value };
- } else {
- this.userData.__crawlee.state = value;
- }
- }
-
- /**
- * Reason for skipping this request.
- */
- get skippedReason(): SkippedRequestReason | undefined {
- return this.userData.__crawlee?.skippedReason;
- }
-
- /**
- * Reason for skipping this request.
- */
- set skippedReason(value: SkippedRequestReason | undefined) {
- if (!this.userData.__crawlee) {
- (this.userData as Dictionary).__crawlee = { skippedReason: value };
- } else {
- this.userData.__crawlee.skippedReason = value;
- }
- }
-
- private get enqueueStrategy(): EnqueueStrategyOption | undefined {
- return this.userData.__crawlee?.enqueueStrategy;
- }
-
- private set enqueueStrategy(value: EnqueueStrategyOption | undefined) {
- if (!this.userData.__crawlee) {
- (this.userData as Dictionary).__crawlee = { enqueueStrategy: value };
- } else {
- this.userData.__crawlee.enqueueStrategy = value;
- }
- }
-
- /**
- * Stores information about an error that occurred during processing of this request.
- *
- * You should always use Error instances when throwing errors in JavaScript.
- *
- * Nevertheless, to improve the debugging experience when using third party libraries
- * that may not always throw an Error instance, the function performs a type
- * inspection of the passed argument and attempts to extract as much information
- * as possible, since just throwing a bad type error makes any debugging rather difficult.
- *
- * @param errorOrMessage Error object or error message to be stored in the request.
- * @param [options]
- */
- pushErrorMessage(errorOrMessage: unknown, options: PushErrorMessageOptions = {}): void {
- const { omitStack } = options;
- let message;
- const type = typeof errorOrMessage;
- if (type === 'object') {
- if (!errorOrMessage) {
- message = 'null';
- } else if (errorOrMessage instanceof Error) {
- message = omitStack
- ? errorOrMessage.message
- : // .stack includes the message
- errorOrMessage.stack;
- } else if (Reflect.has(Object(errorOrMessage), 'message')) {
- message = Reflect.get(Object(errorOrMessage), 'message');
- } else if ((errorOrMessage as string).toString() !== '[object Object]') {
- message = (errorOrMessage as string).toString();
- } else {
- try {
- message = util.inspect(errorOrMessage);
- } catch (err) {
- message = 'Unable to extract any message from the received object.';
- }
- }
- } else if (type === 'undefined') {
- message = 'undefined';
- } else {
- message = (errorOrMessage as string).toString();
- }
-
- this.errorMessages.push(message);
- }
-
/** @internal */
static computeUniqueKey({
url,
@@ -640,39 +464,15 @@ export interface RequestOptions<UserData extends Dictionary = Dictionary> {
*/
crawlDepth?: number;
- /**
- * Reason for skipping this request.
- * This is used to provide more information about why the request was skipped.
- * @internal
- */
- skippedReason?: SkippedRequestReason;
-
/**
* Maximum number of retries for this request. Allows to override the global `maxRequestRetries` option of `BasicCrawler`.
*/
maxRetries?: number;
- /** @internal */
- id?: string;
-
- /** @internal */
- handledAt?: string;
-
- /** @internal */
- lockExpiresAt?: Date;
-
/** @internal */
enqueueStrategy?: EnqueueStrategyOption;
}
-export interface PushErrorMessageOptions {
- /**
- * Only push the error message without stack trace when true.
- * @default false
- */
- omitStack?: boolean;
-}
-
interface ComputeUniqueKeyOptions {
url: string;
method: AllowedHttpMethods;
diff --git a/packages/core/src/storages/request_queue.ts b/packages/core/src/storages/request_queue.ts
index 0ce947116..2dbcb5ea9 100644
--- a/packages/core/src/storages/request_queue.ts
+++ b/packages/core/src/storages/request_queue.ts
@@ -9,6 +9,7 @@ import type {
QueueOperationInfo,
RequestQueueBackend,
RequestQueueInfo,
+ RequestSchema,
} from '@crawlee/types';
import { isAsyncIterable, isIterable } from '@crawlee/utils/internal';
import { downloadListOfUrls } from '@crawlee/utils';
@@ -303,8 +304,8 @@ export class RequestQueue implements IStorage, IRequestManager {
* The requests buffered by the given transaction for this queue, keyed by `uniqueKey` — a dedup
* index derived from the transaction journal.
*/
- private bufferedRequests(transaction: StorageTransaction): Map<string, Dictionary> {
- const buffered = new Map<string, Dictionary>();
+ private bufferedRequests(transaction: StorageTransaction): Map<string, RequestSchema> {
+ const buffered = new Map<string, RequestSchema>();
// Only `deferred` records snapshots, so scanning the journal under `writeThrough` never finds any.
if (transaction.policy.requestQueue !== 'deferred') return buffered;
@@ -381,7 +382,7 @@ export class RequestQueue implements IStorage, IRequestManager {
return await this.addRequest(request, { forefront });
}
- const snapshot = JSON.parse(JSON.stringify(request)) as Dictionary;
+ const snapshot = JSON.parse(JSON.stringify(request)) as RequestSchema;
// Strip-list, not allow-list: every user-facing field flows through, including ones added to
// `Request` in the future. The exceptions are `id` and `handledAt`, the two backend-owned
@@ -418,7 +419,7 @@ export class RequestQueue implements IStorage, IRequestManager {
? // Requests without a snapshot were deduplicated or written through; nothing to replay.
entry.requests
.filter((journaled) => journaled.snapshot !== undefined)
- .map((journaled) => new Request(journaled.snapshot as unknown as RequestOptions))
+ .map((journaled) => Request.fromSchema(journaled.snapshot!))
: [],
);
@@ -600,7 +601,7 @@ export class RequestQueue implements IStorage, IRequestManager {
);
}
- if (opts.id !== undefined) {
+ if ('id' in opts && opts.id !== undefined) {
throw new Error(
`Request options are not valid, the 'id' property must not be present. Input: ${inspect(opts)}`,
);
@@ -679,13 +680,13 @@ export class RequestQueue implements IStorage, IRequestManager {
// Requests buffered by the active transaction (under the `deferred` write policy) are visible to it.
const buffered = transaction && this.bufferedRequests(transaction).get(uniqueKey);
if (buffered) {
- return new Request(buffered as unknown as RequestOptions);
+ return Request.fromSchema<T>(buffered);
}
- const requestOptions = await this.backend.getRequest(uniqueKey);
- if (!requestOptions) return null;
+ const schema = await this.backend.getRequest(uniqueKey);
+ if (!schema) return null;
- return new Request(requestOptions as unknown as RequestOptions);
+ return Request.fromSchema<T>(schema);
}
/**
@@ -716,10 +717,10 @@ export class RequestQueue implements IStorage, IRequestManager {
}
this.#statsTracker.add('headItemReadCount');
- const requestOptions = await this.backend.fetchNextRequest();
- if (!requestOptions) return null;
+ const schema = await this.backend.fetchNextRequest();
+ if (!schema) return null;
- return new Request(requestOptions as unknown as RequestOptions);
+ return Request.fromSchema<T>(schema);
}
/**
@@ -871,7 +872,6 @@ export class RequestQueue implements IStorage, IRequestManager {
isHandled: queueOperationInfo.wasAlreadyHandled,
uniqueKey: queueOperationInfo.uniqueKey,
hydrated: null,
- lockExpiresAt: null,
forefront: queueOperationInfo.forefront,
});
}
@@ -1085,7 +1085,6 @@ interface RequestLruItem {
isHandled: boolean;
id: string;
hydrated: Request | null;
- lockExpiresAt: number | null;
forefront: boolean;
}
diff --git a/packages/core/src/storages/transaction.ts b/packages/core/src/storages/transaction.ts
index ca6771c45..a3eaf2993 100644
--- a/packages/core/src/storages/transaction.ts
+++ b/packages/core/src/storages/transaction.ts
@@ -1,6 +1,6 @@
import { AsyncLocalStorage } from 'node:async_hooks';
-import type { Awaitable, Dictionary } from '@crawlee/types';
+import type { Awaitable, Dictionary, RequestSchema } from '@crawlee/types';
import { addTimeoutToPromise, storage as timeoutStorage, tryCancel } from '@apify/timeout';
@@ -90,7 +90,7 @@ export interface JournaledRequest {
* A full JSON snapshot of the request for the commit replay. Only present for buffered additions —
* deduplicated and write-through ones are journaled for introspection only.
*/
- snapshot?: Dictionary;
+ snapshot?: RequestSchema;
}
/**
diff --git a/packages/http-crawler/src/internals/file-download.ts b/packages/http-crawler/src/internals/file-download.ts
index 02cbd25a9..ca7966d69 100644
--- a/packages/http-crawler/src/internals/file-download.ts
+++ b/packages/http-crawler/src/internals/file-download.ts
@@ -1,8 +1,13 @@
import { Transform } from 'node:stream';
-import type { BasicCrawlerOptions, ContextPipeline, CrawlingContext, LoadedRequest } from '@crawlee/basic';
+import type {
+ BasicCrawlerOptions,
+ ContextPipeline,
+ CrawlingContext,
+ CrawlingRequest,
+ LoadedRequest,
+} from '@crawlee/basic';
import { BasicCrawler } from '@crawlee/basic';
-import type { Request } from '@crawlee/core';
import { ResponseWithUrl } from '@crawlee/http-client';
import type { Dictionary } from '@crawlee/types';
@@ -33,7 +38,7 @@ export type FileDownloadHook<
export interface FileDownloadCrawlingContext<
UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
> extends CrawlingContext<UserData> {
- request: LoadedRequest<Request<UserData>>;
+ request: LoadedRequest<CrawlingRequest<UserData>>;
response: Response;
contentType: { type: string; encoding: BufferEncoding };
}
@@ -203,7 +208,7 @@ export class FileDownload extends BasicCrawler<FileDownloadCrawlingContext> {
const { response: trackedResponse, bodyDrained } = trackBodyConsumption(response);
const contextExtension = {
- request: context.request as LoadedRequest<Request>,
+ request: context.request as LoadedRequest<CrawlingRequest>,
response: trackedResponse,
contentType: { type, encoding },
[kBodyDrained]: bodyDrained,
diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.ts
index 275e63f97..f3049ed46 100644
--- a/packages/http-crawler/src/internals/http-crawler.ts
+++ b/packages/http-crawler/src/internals/http-crawler.ts
@@ -10,7 +10,7 @@ import type {
ErrorHandler,
GetUserDataFromRequest,
LoadedRequest,
- Request as CrawleeRequest,
+ CrawlingRequest,
RequestHandler,
RequireContextPipeline,
RouterHandler,
@@ -201,7 +201,7 @@ interface CrawlingContextWithResponse<
/**
* The request object that was successfully loaded and navigated to, including the {@apilink Request.loadedUrl|`loadedUrl`} property.
*/
- request: LoadedRequest<CrawleeRequest<UserData>>;
+ request: LoadedRequest<CrawlingRequest<UserData>>;
/**
* The HTTP response object containing status code, headers, and other response metadata.
@@ -511,7 +511,7 @@ export class HttpCrawler<
}
return Reflect.get(target, propertyName, receiver);
},
- }) as LoadedRequest<CrawleeRequest>,
+ }) as LoadedRequest<CrawlingRequest>,
get response(): InternalHttpCrawlingContext['response'] {
throw new NavigationSkippedError(
'The `response` property is not available - `skipNavigation` was used',
@@ -545,7 +545,7 @@ export class HttpCrawler<
request.loadedUrl = httpResponse?.url;
request.state = RequestState.AFTER_NAV;
- return { request: request as LoadedRequest<CrawleeRequest>, response: httpResponse };
+ return { request: request as LoadedRequest<CrawlingRequest>, response: httpResponse };
}
private async processHttpResponse(
@@ -720,7 +720,7 @@ export class HttpCrawler<
/**
* Encodes and parses response according to the provided content type
*/
- private async parseResponse(request: CrawleeRequest, response: Response) {
+ private async parseResponse(request: CrawlingRequest, response: Response) {
const { status } = response;
const { type, charset } = parseContentTypeFromResponse(response);
const { response: reencodedResponse, encoding } = this.encodeResponse(request, response, charset);
@@ -772,7 +772,7 @@ export class HttpCrawler<
/**
* Combines the provided `requestOptions` with mandatory (non-overridable) values.
*/
- private getRequestOptions(request: CrawleeRequest, session: ISession, proxyUrl?: string) {
+ private getRequestOptions(request: CrawlingRequest, session: ISession, proxyUrl?: string) {
const requestOptions = {
url: request.url,
method: request.method,
@@ -794,7 +794,7 @@ export class HttpCrawler<
}
private encodeResponse(
- request: CrawleeRequest,
+ request: CrawlingRequest,
response: Response,
encoding: BufferEncoding,
): {
@@ -867,7 +867,7 @@ export class HttpCrawler<
throw new Error(`Request timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
}
- private abortDownloadOfBody(request: CrawleeRequest, response: Response) {
+ private abortDownloadOfBody(request: CrawlingRequest, response: Response) {
const { status } = response;
const { type } = parseContentTypeFromResponse(response);
@@ -925,7 +925,7 @@ export class HttpCrawler<
}
interface RequestFunctionOptions {
- request: CrawleeRequest;
+ request: CrawlingRequest;
session: ISession;
proxyUrl?: string;
}
diff --git a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts
index 648bdda4e..b1e9df28e 100644
--- a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts
+++ b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts
@@ -1,7 +1,7 @@
import type {
BrowserHook,
LoadedRequest,
- Request,
+ CrawlingRequest,
RouterHandler,
RouteSchemas,
RoutesFromSchemas,
@@ -93,7 +93,7 @@ export const adaptivePlaywrightCrawlerStatisticState = {
export interface AdaptivePlaywrightCrawlerContext<
UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
> extends CrawlingContext<UserData> {
- request: LoadedRequest<Request<UserData>>;
+ request: LoadedRequest<CrawlingRequest<UserData>>;
/**
* The HTTP response, either from the HTTP client or from the initial request from playwright's navigation.
*/
@@ -150,14 +150,14 @@ export interface AdaptivePlaywrightCrawlerContext<
interface AdaptiveHookContext extends Pick<AdaptivePlaywrightCrawlerContext, 'id' | 'session' | 'proxyInfo' | 'log'> {
page?: Page;
- request: Request;
+ request: CrawlingRequest;
gotoOptions?: PlaywrightGotoOptions;
}
type AdaptiveHook<ContextExtension = Dictionary<never>> = BrowserHook<AdaptiveHookContext, ContextExtension>;
type AdaptivePostNavigationHook<ContextExtension = Dictionary<never>> = BrowserHook<
- Omit<AdaptiveHookContext, 'request'> & { request: LoadedRequest<Request> },
+ Omit<AdaptiveHookContext, 'request'> & { request: LoadedRequest<CrawlingRequest> },
ContextExtension
>;
@@ -518,8 +518,8 @@ export class AdaptivePlaywrightCrawler<
return super.buildContextPipeline().compose({
action: async ({ request }) => ({
- get request(): LoadedRequest<Request<Dictionary>> {
- return request as LoadedRequest<Request<Dictionary>>;
+ get request(): LoadedRequest<CrawlingRequest<Dictionary>> {
+ return request as LoadedRequest<CrawlingRequest<Dictionary>>;
},
get response(): Response {
throw new Error(errorMessage('response'));
diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.ts
index 2c9fc6c89..3c73f8e9b 100644
--- a/test/core/crawlers/basic_crawler.test.ts
+++ b/test/core/crawlers/basic_crawler.test.ts
@@ -13,11 +13,11 @@ import type {
RequestHandler,
RequestOptions,
Session,
- Source,
} from '@crawlee/basic';
import {
AfterCommitError,
BasicCrawler,
+ CrawlingRequest,
ConcurrencySystem,
Configuration,
CriticalError,
@@ -34,6 +34,7 @@ import {
RequestManagerTandem,
STATE_PERSISTENCE_KEY,
RequestQueue,
+ RequestState,
RequestValidationError,
Router,
serviceLocator,
@@ -41,7 +42,7 @@ import {
Statistics,
ThrottlingRequestManager,
} from '@crawlee/basic';
-import { MemoryStorageBackend, RequestState } from '@crawlee/core';
+import { MemoryStorageBackend } from '@crawlee/core';
import { BaseHttpClient } from '@crawlee/http-client';
import type { Dictionary, ISession, ProxyInfo } from '@crawlee/types';
import { RobotsTxtFile, sleep } from '@crawlee/utils';
@@ -645,8 +646,8 @@ describe('BasicCrawler', () => {
await crawler.addRequests(urls, options);
expect(drainedRequests).toHaveLength(2);
- expect(drainedRequests[0]).toMatchObject({ url: 'https://example.com/1/', crawlDepth: 3 });
- expect(drainedRequests[1]).toMatchObject({ url: 'https://example.com/2/', crawlDepth: 3 });
+ expect(drainedRequests.map((r) => r.url)).toEqual(['https://example.com/1/', 'https://example.com/2/']);
+ expect(drainedRequests.map((r) => CrawlingRequest.fromSchema(r).crawlDepth)).toEqual([3, 3]);
expect(onSkippedRequestMock).not.toBeCalled();
});
@@ -676,8 +677,8 @@ describe('BasicCrawler', () => {
expect(transformRequestFunction).toHaveBeenCalled();
- expect(drainedRequests).toHaveLength(2);
- expect(drainedRequests[0]).toMatchObject({ url: 'https://example.com/1/', crawlDepth: 3 });
+ expect(drainedRequests.map((r) => r.url)).toEqual(['https://example.com/1/', 'https://example.com/2/']);
+ expect(CrawlingRequest.fromSchema(drainedRequests[0]).crawlDepth).toBe(3);
});
it.each([null, undefined, false] as const)(
@@ -874,9 +875,9 @@ describe('BasicCrawler', () => {
const newCrawlDepth = 4;
const addRequestsGenerator = crawler.exposedAddCrawlDepthRequestGenerator(requests, newCrawlDepth);
- const generatedRequests: Source[] = [];
+ const generatedRequests: RequestOptions[] = [];
for await (const generatedRequest of addRequestsGenerator) {
- generatedRequests.push(generatedRequest);
+ generatedRequests.push(generatedRequest as RequestOptions);
}
expect(generatedRequests).toHaveLength(2);
@@ -1517,18 +1518,22 @@ describe('BasicCrawler', () => {
.mockResolvedValueOnce({ requestId: 'id-1' } as any)
.mockResolvedValueOnce({ requestId: 'id-2' } as any);
- const request0 = new Request({ id: 'id-0', ...sources[0] });
- const request1 = new Request({ id: 'id-1', ...sources[1] });
- const request2 = new Request({ id: 'id-2', ...sources[2] });
+ const request0 = Object.assign(new Request(sources[0]), { id: 'id-0' });
+ const request1 = Object.assign(new Request(sources[1]), { id: 'id-1' });
+ const request2 = Object.assign(new Request(sources[2]), { id: 'id-2' });
- const queueContent = [request0, request1, request2, request1, request1, request1];
+ // A queue re-serves whatever was reclaimed into it, so the mock has to do the same for retries to accumulate.
+ const queueContent: Request[] = [request0, request1, request2];
vitest.spyOn(requestQueue, 'fetchNextRequest').mockImplementation(async () => queueContent.shift() ?? null);
const markReqHandled = vitest
.spyOn(requestQueue, 'markRequestAsHandled')
.mockReturnValue(Promise.resolve() as any);
- const reclaimReq = vitest.spyOn(requestQueue, 'reclaimRequest').mockReturnValue(Promise.resolve() as any);
+ const reclaimReq = vitest.spyOn(requestQueue, 'reclaimRequest').mockImplementation(async (request) => {
+ queueContent.push(request);
+ return Promise.resolve() as any;
+ });
// The first probe reporting `finished` is masked by the request list, which still has requests to
// transfer into the queue at that point.
@@ -1539,10 +1544,7 @@ describe('BasicCrawler', () => {
await basicCrawler.run();
- // 1st try
-
- expect(reclaimReq).toBeCalledWith(request1, expect.objectContaining({}));
- expect(reclaimReq).toBeCalledTimes(3);
+ expect(reclaimReq.mock.calls.map(([request]) => request.url)).toEqual(Array(3).fill('http://example.com/1'));
expect(processed['http://example.com/0'].userData.foo).toBe('bar');
expect(processed['http://example.com/0'].errorMessages).toEqual([]);
@@ -1637,8 +1639,7 @@ describe('BasicCrawler', () => {
expect(isFinishedFunctionCalled).toBe(true);
expect(isTaskReadyFunctionCalled).toBe(true);
- // TODO: see why the request1 was passed as a second parameter to includes
- expect(processed.includes(request0)).toBe(true);
+ expect(processed.map((request) => request.url)).toContain(request0.url);
vitest.restoreAllMocks();
});
@@ -1685,8 +1686,7 @@ describe('BasicCrawler', () => {
expect(markRequestAsHandled).toBeCalledWith(request0);
expect(markRequestAsHandled).toBeCalledWith(request1);
- // TODO: see why the request1 was passed as a second parameter to includes
- expect(processed.includes(request0)).toBe(true);
+ expect(processed.map((request) => request.url)).toContain(request0.url);
vitest.restoreAllMocks();
});
diff --git a/test/core/enqueue_links/enqueue_links.test.ts b/test/core/enqueue_links/enqueue_links.test.ts
index 19be0b3c5..dad5ce206 100644
--- a/test/core/enqueue_links/enqueue_links.test.ts
+++ b/test/core/enqueue_links/enqueue_links.test.ts
@@ -1,5 +1,6 @@
import {
CheerioCrawler,
+ CrawlingRequest,
EnqueueStrategy,
MemoryStorageBackend,
Request,
@@ -384,7 +385,7 @@ describe('enqueueLinks()', () => {
expect(enqueued).toHaveLength(2);
for (const request of enqueued) {
- expect(request.skipNavigation).toBe(true);
+ expect(CrawlingRequest.fromSchema(request as Request).skipNavigation).toBe(true);
}
});
diff --git a/test/core/router.test.ts b/test/core/router.test.ts
index 29b300852..6e49bec64 100644
--- a/test/core/router.test.ts
+++ b/test/core/router.test.ts
@@ -1,6 +1,6 @@
import type { CrawlingContext } from '@crawlee/basic';
-import { BasicCrawler, defaultRoute, MissingRouteError, Router } from '@crawlee/basic';
-import { Request, RequestValidationError } from '@crawlee/core';
+import { BasicCrawler, CrawlingRequest, defaultRoute, MissingRouteError, Router } from '@crawlee/basic';
+import { RequestValidationError } from '@crawlee/core';
import {
CheerioCrawler,
type CheerioCrawlingContext,
@@ -339,7 +339,11 @@ describe('Router', () => {
// a real Request keeps `label` inside `userData` and `crawlDepth` inside the non-enumerable `__crawlee`;
// both must survive the schema replacing `userData` with the parsed (label-less) value.
- const request = new Request({ url: 'https://example.com/p', label: 'PRODUCT', userData: { sku: 'A1' } });
+ const request = new CrawlingRequest({
+ url: 'https://example.com/p',
+ label: 'PRODUCT',
+ userData: { sku: 'A1' },
+ });
request.crawlDepth = 3;
await router({ request, log } as any);
diff --git a/test/core/storages/request_queue.test.ts b/test/core/storages/request_queue.test.ts
index 6d8a336d7..692902d6a 100644
--- a/test/core/storages/request_queue.test.ts
+++ b/test/core/storages/request_queue.test.ts
@@ -1,5 +1,6 @@
/* eslint-disable dot-notation */
+import { CrawlingRequest } from '@crawlee/basic';
import { MemoryStorageBackend, ProxyConfiguration, Request, RequestQueue, serviceLocator } from '@crawlee/core';
import { BaseHttpClient } from '@crawlee/http-client';
import { sleep } from '@crawlee/utils';
@@ -326,17 +327,17 @@ describe('RequestQueue remote', () => {
test('Request.userData.__crawlee internal object is non-enumerable and always defined', async () => {
const url = 'http://example.com';
const method = 'POST';
- const r1 = new Request({
+ const r1 = new CrawlingRequest({
url,
method,
userData: { __crawlee: { skipNavigation: true, maxRetries: 10, foo: 123, bar: true, crawlDepth: 10 } },
});
- const r2 = new Request({
+ const r2 = new CrawlingRequest({
url,
method,
userData: {} as any,
});
- const r3 = new Request({
+ const r3 = new CrawlingRequest({
url,
method,
});
@@ -355,7 +356,7 @@ describe('RequestQueue remote', () => {
});
// Re-wrapping userData that comes from another Request instance (where `__crawlee` is
// non-enumerable) must preserve the internal state instead of dropping it via the spread.
- const r4 = new Request({ url, method, userData: r1.userData });
+ const r4 = new CrawlingRequest({ url, method, userData: r1.userData });
expect(r4.skipNavigation).toBe(true);
expect(r4.maxRetries).toBe(5);
expect(r4.crawlDepth).toBe(10);
diff --git a/test/core/storages/storage_transaction.test.ts b/test/core/storages/storage_transaction.test.ts
index a9547a5eb..63ed3f135 100644
--- a/test/core/storages/storage_transaction.test.ts
+++ b/test/core/storages/storage_transaction.test.ts
@@ -783,8 +783,9 @@ describe('RequestQueue in a transaction', () => {
const addBatchSpy = vitest.spyOn(queue.backend, 'addBatchOfRequests');
// Looks like it came from a backend: foreign id, already handled.
- const reenqueued = new Request({
+ const reenqueued = Request.fromSchema({
url: 'https://example.com/old',
+ uniqueKey: 'https://example.com/old',
id: 'foreign-backend-id',
handledAt: new Date().toISOString(),
});