Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

chore: use `userData` for the `Request.state`

Martin Adámek97989df6de9f
diff --git a/packages/core/src/request.ts b/packages/core/src/request.ts
index 6104301a6..9b0814d4a 100644
--- a/packages/core/src/request.ts
+++ b/packages/core/src/request.ts
@@ -122,9 +122,6 @@ export class Request<UserData extends Dictionary = Dictionary> {
*/
handledAt?: string;
- /** Describes the request's current lifecycle stage. */
- state: RequestState = RequestState.UNPROCESSED;
-
/**
* `Request` parameters including the URL, HTTP method and headers, and others.
*/
@@ -201,13 +198,11 @@ export class Request<UserData extends Dictionary = Dictionary> {
set: (value: Record<string, any>) => {
Object.defineProperties(value, {
__crawlee: {
- // eslint-disable-next-line no-underscore-dangle
value: this._userData.__crawlee,
enumerable: false,
writable: true,
},
toJSON: {
- // eslint-disable-next-line no-underscore-dangle
value: () => {
if (Object.keys(this._userData.__crawlee).length > 0) {
return ({
@@ -236,14 +231,12 @@ export class Request<UserData extends Dictionary = Dictionary> {
/** Tells the crawler processing this request to skip the navigation and process the request directly. */
get skipNavigation(): boolean {
- // eslint-disable-next-line no-underscore-dangle
return this.userData.__crawlee?.skipNavigation ?? false;
}
+ /** Tells the crawler processing this request to skip the navigation and process the request directly. */
set skipNavigation(value: boolean) {
- // eslint-disable-next-line no-underscore-dangle
if (!this.userData.__crawlee) (this.userData as Dictionary).__crawlee = { skipNavigation: value };
- // eslint-disable-next-line no-underscore-dangle
else this.userData.__crawlee.skipNavigation = value;
}
@@ -257,6 +250,17 @@ export class Request<UserData extends Dictionary = Dictionary> {
(this.userData as Dictionary).label = value;
}
+ /** Describes the request's current lifecycle state. */
+ get state(): RequestState {
+ return this.userData.__crawlee?.state ?? RequestState.UNPROCESSED;
+ }
+
+ /** Describes the request's current lifecycle state. */
+ set state(value: RequestState) {
+ if (!this.userData.__crawlee) (this.userData as Dictionary).__crawlee = { state: value };
+ else this.userData.__crawlee.state = value;
+ }
+
/**
* Stores information about an error that occurred during processing of this request.
*