Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
chore: use `userData` for the `Request.state`
Martin Adámek97989df6de9f
diff --git a/packages/core/src/request.ts b/packages/core/src/request.tsindex 6104301a6..9b0814d4a 100644--- a/packages/core/src/request.ts+++ b/packages/core/src/request.ts@@ -122,9 +122,6 @@ export class Request<UserData extends Dictionary = Dictionary> {*/handledAt?: string;- /** Describes the request's current lifecycle stage. */- state: RequestState = RequestState.UNPROCESSED;-/*** `Request` parameters including the URL, HTTP method and headers, and others.*/@@ -201,13 +198,11 @@ export class Request<UserData extends Dictionary = Dictionary> {set: (value: Record<string, any>) => {Object.defineProperties(value, {__crawlee: {- // eslint-disable-next-line no-underscore-danglevalue: this._userData.__crawlee,enumerable: false,writable: true,},toJSON: {- // eslint-disable-next-line no-underscore-danglevalue: () => {if (Object.keys(this._userData.__crawlee).length > 0) {return ({@@ -236,14 +231,12 @@ export class Request<UserData extends Dictionary = Dictionary> {/** Tells the crawler processing this request to skip the navigation and process the request directly. */get skipNavigation(): boolean {- // eslint-disable-next-line no-underscore-danglereturn this.userData.__crawlee?.skipNavigation ?? false;}+ /** Tells the crawler processing this request to skip the navigation and process the request directly. */set skipNavigation(value: boolean) {- // eslint-disable-next-line no-underscore-dangleif (!this.userData.__crawlee) (this.userData as Dictionary).__crawlee = { skipNavigation: value };- // eslint-disable-next-line no-underscore-dangleelse this.userData.__crawlee.skipNavigation = value;}@@ -257,6 +250,17 @@ export class Request<UserData extends Dictionary = Dictionary> {(this.userData as Dictionary).label = value;}+ /** Describes the request's current lifecycle state. */+ get state(): RequestState {+ return this.userData.__crawlee?.state ?? RequestState.UNPROCESSED;+ }++ /** Describes the request's current lifecycle state. */+ set state(value: RequestState) {+ if (!this.userData.__crawlee) (this.userData as Dictionary).__crawlee = { state: value };+ else this.userData.__crawlee.state = value;+ }+/*** Stores information about an error that occurred during processing of this request.*