Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

More updates to Session docs

Ondra Urban14207b463f2c
diff --git a/docs/api/Session.md b/docs/api/Session.md
index d7c6f697c..6417a185b 100644
--- a/docs/api/Session.md
+++ b/docs/api/Session.md
@@ -11,7 +11,6 @@ user data for example some authorization tokens and specific headers in general.
- [Session](session)
- [`new exports.Session(options)`](#new_Session_new)
- - [`.cookieJar`](#Session+cookieJar) : `CookieJar`
- [`.isBlocked()`](#Session+isBlocked) ⇒ `boolean`
- [`.isExpired()`](#Session+isExpired) ⇒ `boolean`
- [`.isMaxUsageCountReached()`](#Session+isMaxUsageCountReached) ⇒ `boolean`
@@ -24,7 +23,7 @@ user data for example some authorization tokens and specific headers in general.
- [`.setCookiesFromResponse(response)`](#Session+setCookiesFromResponse)
- [`.setPuppeteerCookies(cookies, url)`](#Session+setPuppeteerCookies)
- [`.getPuppeteerCookies(url)`](#Session+getPuppeteerCookies) ⇒ `Array<PuppeteerCookie>`
- - [`.getCookieString(url)`](#Session+getCookieString) ⇒ `String`
+ - [`.getCookieString(url)`](#Session+getCookieString) ⇒ `string`
<a name="new_Session_new"></a>
@@ -45,10 +44,6 @@ Session configuration.
<tr>
</tr></tbody>
</table>
-<a name="Session+cookieJar"></a>
-
-## `session.cookieJar` : `CookieJar`
-
<a name="Session+isBlocked"></a>
## `session.isBlocked()` ⇒ `boolean`
@@ -120,12 +115,12 @@ by retiring the session when such code is received. Optionally the default statu
</thead>
<tbody>
<tr>
-<td><code>statusCode</code></td><td><code>Number</code></td>
+<td><code>statusCode</code></td><td><code>number</code></td>
</tr>
<tr>
<td colspan="3"><p>HTTP status code</p>
</td></tr><tr>
-<td><code>[blockedStatusCodes]</code></td><td><code>Array<Number></code></td>
+<td><code>[blockedStatusCodes]</code></td><td><code>Array<number></code></td>
</tr>
<tr>
<td colspan="3"><p>Custom HTTP status codes that means blocking on particular website.</p>
@@ -180,7 +175,7 @@ manually:
</tr>
<tr>
</tr><tr>
-<td><code>url</code></td><td><code>String</code></td>
+<td><code>url</code></td><td><code>string</code></td>
</tr>
<tr>
</tr></tbody>
@@ -207,11 +202,11 @@ Returns cookies in a format compatible with puppeteer and ready to be used with
</table>
<a name="Session+getCookieString"></a>
-## `session.getCookieString(url)` ⇒ `String`
+## `session.getCookieString(url)` ⇒ `string`
Returns cookies saved with the session in the typical key1=value1; key2=value2 format, ready to be used in a cookie header or elsewhere.
-**Returns**: `String` - - represents `Cookie` header.
+**Returns**: `string` - - represents `Cookie` header.
<table>
<thead>
@@ -221,7 +216,7 @@ Returns cookies saved with the session in the typical key1=value1; key2=value2 f
</thead>
<tbody>
<tr>
-<td><code>url</code></td><td><code>String</code></td>
+<td><code>url</code></td><td><code>string</code></td>
</tr>
<tr>
</tr></tbody>
diff --git a/docs/typedefs/SessionOptions.md b/docs/typedefs/SessionOptions.md
index 1accce233..15f5543ca 100644
--- a/docs/typedefs/SessionOptions.md
+++ b/docs/typedefs/SessionOptions.md
@@ -15,12 +15,12 @@ title: SessionOptions
</thead>
<tbody>
<tr>
-<td><code>[id]</code></td><td><code>String</code></td><td></td>
+<td><code>[id]</code></td><td><code>string</code></td><td></td>
</tr>
<tr>
<td colspan="3"><p>Id of session used for generating fingerprints. It is used as proxy session name.</p>
</td></tr><tr>
-<td><code>[maxAgeSecs]</code></td><td><code>Number</code></td><td><code>3000</code></td>
+<td><code>[maxAgeSecs]</code></td><td><code>number</code></td><td><code>3000</code></td>
</tr>
<tr>
<td colspan="3"><p>Number of seconds after which the session is considered as expired.</p>
@@ -55,17 +55,17 @@ title: SessionOptions
<tr>
<td colspan="3"><p>Date of expiration.</p>
</td></tr><tr>
-<td><code>[usageCount]</code></td><td><code>Number</code></td><td><code>0</code></td>
+<td><code>[usageCount]</code></td><td><code>number</code></td><td><code>0</code></td>
</tr>
<tr>
<td colspan="3"><p>Indicates how many times the session has been used.</p>
</td></tr><tr>
-<td><code>[errorCount]</code></td><td><code>Number</code></td><td><code>0</code></td>
+<td><code>[errorCount]</code></td><td><code>number</code></td><td><code>0</code></td>
</tr>
<tr>
<td colspan="3"><p>Indicates how many times the session is marked bad.</p>
</td></tr><tr>
-<td><code>[maxUsageCount]</code></td><td><code>Number</code></td><td><code>50</code></td>
+<td><code>[maxUsageCount]</code></td><td><code>number</code></td><td><code>50</code></td>
</tr>
<tr>
<td colspan="3"><p>Session should be used only a limited amount of times.
diff --git a/docs/typedefs/SessionState.md b/docs/typedefs/SessionState.md
index dc7f4860f..cb83d2355 100644
--- a/docs/typedefs/SessionState.md
+++ b/docs/typedefs/SessionState.md
@@ -5,7 +5,7 @@ title: SessionState
<a name="SessionState"></a>
-Persistable {Session} state.
+Persistable [`Session`](session) state.
**Properties**
@@ -17,11 +17,7 @@ Persistable {Session} state.
</thead>
<tbody>
<tr>
-<td><code>id</code></td><td><code>String</code></td>
-</tr>
-<tr>
-</tr><tr>
-<td><code>cookies</code></td><td><code>Object</code></td>
+<td><code>id</code></td><td><code>string</code></td>
</tr>
<tr>
</tr><tr>
@@ -33,27 +29,27 @@ Persistable {Session} state.
</tr>
<tr>
</tr><tr>
-<td><code>maxErrorScore</code></td><td><code>Number</code></td>
+<td><code>errorScore</code></td><td><code>number</code></td>
</tr>
<tr>
</tr><tr>
-<td><code>errorScoreDecrement</code></td><td><code>Number</code></td>
+<td><code>maxErrorScore</code></td><td><code>number</code></td>
</tr>
<tr>
</tr><tr>
-<td><code>expiresAt</code></td><td><code>String</code></td>
+<td><code>errorScoreDecrement</code></td><td><code>number</code></td>
</tr>
<tr>
</tr><tr>
-<td><code>createdAt</code></td><td><code>String</code></td>
+<td><code>usageCount</code></td><td><code>number</code></td>
</tr>
<tr>
</tr><tr>
-<td><code>usageCount</code></td><td><code>Number</code></td>
+<td><code>expiresAt</code></td><td><code>string</code></td>
</tr>
<tr>
</tr><tr>
-<td><code>errorScore</code></td><td><code>Number</code></td>
+<td><code>createdAt</code></td><td><code>string</code></td>
</tr>
<tr>
</tr></tbody>
diff --git a/src/session_pool/session.js b/src/session_pool/session.js
index 7b8bd4504..0f6e12ce5 100644
--- a/src/session_pool/session.js
+++ b/src/session_pool/session.js
@@ -13,24 +13,23 @@ import { Cookie as PuppeteerCookie } from 'puppeteer';
/* eslint-enable no-unused-vars,import/named,import/no-duplicates,import/order,import/no-cycle */
/**
- * Persistable {Session} state.
+ * Persistable {@link Session} state.
* @typedef {Object} SessionState
- * @property {String} id
- * @property {Object} cookies
+ * @property {string} id
* @property {Object} cookieJar
* @property {Object} userData
- * @property {Number} maxErrorScore
- * @property {Number} errorScoreDecrement
- * @property {String} expiresAt
- * @property {String} createdAt
- * @property {Number} usageCount
- * @property {Number} errorScore
+ * @property {number} errorScore
+ * @property {number} maxErrorScore
+ * @property {number} errorScoreDecrement
+ * @property {number} usageCount
+ * @property {string} expiresAt
+ * @property {string} createdAt
*/
/**
* @typedef {Object} SessionOptions
- * @property {String} [id] - Id of session used for generating fingerprints. It is used as proxy session name.
- * @property {Number} [maxAgeSecs=3000] - Number of seconds after which the session is considered as expired.
+ * @property {string} [id] - Id of session used for generating fingerprints. It is used as proxy session name.
+ * @property {number} [maxAgeSecs=3000] - Number of seconds after which the session is considered as expired.
* @property {Object} userData - Object where custom user data can be stored. For example custom headers.
* @property {number} [maxErrorScore=3] - Maximum number of marking session as blocked usage.
* If the `errorScore` reaches the `maxErrorScore` session is marked as block and it is thrown away.
@@ -41,9 +40,9 @@ import { Cookie as PuppeteerCookie } from 'puppeteer';
* number.
* @property {Date} [createdAt] - Date of creation.
* @property {Date} [expiresAt] - Date of expiration.
- * @property {Number} [usageCount=0] - Indicates how many times the session has been used.
- * @property {Number} [errorCount=0] - Indicates how many times the session is marked bad.
- * @property {Number} [maxUsageCount=50] - Session should be used only a limited amount of times.
+ * @property {number} [usageCount=0] - Indicates how many times the session has been used.
+ * @property {number} [errorCount=0] - Indicates how many times the session is marked bad.
+ * @property {number} [maxUsageCount=50] - Session should be used only a limited amount of times.
* This number indicates how many times the session is going to be used, before it is thrown away.
* @property {SessionPool} sessionPool - SessionPool instance. Session will emit the `sessionRetired` event on this instance.
*/
@@ -60,7 +59,6 @@ export class Session {
constructor(options = {}) {
const {
id = `session_${cryptoRandomObjectId(10)}`,
- cookies = [], // @TODO: Delete, deprecate or leave it as custom cookie persistance?
cookieJar = new CookieJar(),
maxAgeSecs = 3000,
userData = {},
@@ -93,10 +91,12 @@ export class Session {
}
// Configurable
- this.id = id;
- this.cookies = cookies;
- /** @type CookieJar */
+ /**
+ * @type CookieJar
+ * @private
+ * */
this.cookieJar = cookieJar.setCookie ? cookieJar : CookieJar.fromJSON(JSON.stringify(cookieJar));
+ this.id = id;
this.maxAgeSecs = maxAgeSecs;
this.userData = userData;
this.maxErrorScore = maxErrorScore;
@@ -209,8 +209,8 @@ export class Session {
* that the target website is blocking us. This function helps to do this conveniently
* by retiring the session when such code is received. Optionally the default status
* codes can be extended in the second parameter.
- * @param statusCode {Number} - HTTP status code
- * @param [blockedStatusCodes] {Array<Number>} - Custom HTTP status codes that means blocking on particular website.
+ * @param statusCode {number} - HTTP status code
+ * @param [blockedStatusCodes] {number[]} - Custom HTTP status codes that means blocking on particular website.
* @return {boolean} whether the session was retired.
*/
retireOnBlockedStatusCodes(statusCode, blockedStatusCodes = []) {
@@ -254,7 +254,7 @@ export class Session {
* ```
*
* @param cookies {PuppeteerCookie[]}
- * @param url {String}
+ * @param url {string}
*/
setPuppeteerCookies(cookies, url) {
try {
@@ -280,8 +280,8 @@ export class Session {
* Returns cookies saved with the session in the typical
* key1=value1; key2=value2 format, ready to be used in
* a cookie header or elsewhere.
- * @param {String} url
- * @return {String} - represents `Cookie` header.
+ * @param {string} url
+ * @return {string} - represents `Cookie` header.
*/
getCookieString(url) {
return this.cookieJar.getCookieStringSync(url, {});
@@ -327,7 +327,7 @@ export class Session {
/**
* Sets cookies.
* @param {Cookie[]} cookies
- * @param {String} url
+ * @param {string} url
* @private
*/
_setCookies(cookies, url) {
diff --git a/types/session_pool/session.d.ts b/types/session_pool/session.d.ts
index 33ef3fe08..6ecb6f136 100644
--- a/types/session_pool/session.d.ts
+++ b/types/session_pool/session.d.ts
@@ -1,21 +1,20 @@
/**
- * Persistable {Session} state.
+ * Persistable {@link Session} state.
* @typedef {Object} SessionState
- * @property {String} id
- * @property {Object} cookies
+ * @property {string} id
* @property {Object} cookieJar
* @property {Object} userData
- * @property {Number} maxErrorScore
- * @property {Number} errorScoreDecrement
- * @property {String} expiresAt
- * @property {String} createdAt
- * @property {Number} usageCount
- * @property {Number} errorScore
+ * @property {number} errorScore
+ * @property {number} maxErrorScore
+ * @property {number} errorScoreDecrement
+ * @property {number} usageCount
+ * @property {string} expiresAt
+ * @property {string} createdAt
*/
/**
* @typedef {Object} SessionOptions
- * @property {String} [id] - Id of session used for generating fingerprints. It is used as proxy session name.
- * @property {Number} [maxAgeSecs=3000] - Number of seconds after which the session is considered as expired.
+ * @property {string} [id] - Id of session used for generating fingerprints. It is used as proxy session name.
+ * @property {number} [maxAgeSecs=3000] - Number of seconds after which the session is considered as expired.
* @property {Object} userData - Object where custom user data can be stored. For example custom headers.
* @property {number} [maxErrorScore=3] - Maximum number of marking session as blocked usage.
* If the `errorScore` reaches the `maxErrorScore` session is marked as block and it is thrown away.
@@ -26,9 +25,9 @@
* number.
* @property {Date} [createdAt] - Date of creation.
* @property {Date} [expiresAt] - Date of expiration.
- * @property {Number} [usageCount=0] - Indicates how many times the session has been used.
- * @property {Number} [errorCount=0] - Indicates how many times the session is marked bad.
- * @property {Number} [maxUsageCount=50] - Session should be used only a limited amount of times.
+ * @property {number} [usageCount=0] - Indicates how many times the session has been used.
+ * @property {number} [errorCount=0] - Indicates how many times the session is marked bad.
+ * @property {number} [maxUsageCount=50] - Session should be used only a limited amount of times.
* This number indicates how many times the session is going to be used, before it is thrown away.
* @property {SessionPool} sessionPool - SessionPool instance. Session will emit the `sessionRetired` event on this instance.
*/
@@ -43,10 +42,12 @@ export class Session {
* @param {SessionOptions} options
*/
constructor(options?: SessionOptions);
- id: string;
- cookies: any;
- /** @type CookieJar */
+ /**
+ * @type CookieJar
+ * @private
+ * */
cookieJar: CookieJar;
+ id: string;
maxAgeSecs: number;
userData: any;
maxErrorScore: number;
@@ -110,8 +111,8 @@ export class Session {
* that the target website is blocking us. This function helps to do this conveniently
* by retiring the session when such code is received. Optionally the default status
* codes can be extended in the second parameter.
- * @param statusCode {Number} - HTTP status code
- * @param [blockedStatusCodes] {Array<Number>} - Custom HTTP status codes that means blocking on particular website.
+ * @param statusCode {number} - HTTP status code
+ * @param [blockedStatusCodes] {number[]} - Custom HTTP status codes that means blocking on particular website.
* @return {boolean} whether the session was retired.
*/
retireOnBlockedStatusCodes(statusCode: number, blockedStatusCodes?: number[]): boolean;
@@ -140,7 +141,7 @@ export class Session {
* ```
*
* @param cookies {PuppeteerCookie[]}
- * @param url {String}
+ * @param url {string}
*/
setPuppeteerCookies(cookies: PuppeteerCookie[], url: string): void;
/**
@@ -153,8 +154,8 @@ export class Session {
* Returns cookies saved with the session in the typical
* key1=value1; key2=value2 format, ready to be used in
* a cookie header or elsewhere.
- * @param {String} url
- * @return {String} - represents `Cookie` header.
+ * @param {string} url
+ * @return {string} - represents `Cookie` header.
*/
getCookieString(url: string): string;
/**
@@ -174,25 +175,24 @@ export class Session {
/**
* Sets cookies.
* @param {Cookie[]} cookies
- * @param {String} url
+ * @param {string} url
* @private
*/
_setCookies(cookies: Cookie[], url: string): void;
}
/**
- * Persistable {Session} state.
+ * Persistable {@link Session} state.
*/
export type SessionState = {
id: string;
- cookies: any;
cookieJar: any;
userData: any;
+ errorScore: number;
maxErrorScore: number;
errorScoreDecrement: number;
+ usageCount: number;
expiresAt: string;
createdAt: string;
- usageCount: number;
- errorScore: number;
};
export type SessionOptions = {
/**
diff --git a/website/i18n/en.json b/website/i18n/en.json
index aaa3f4171..4114c250f 100644
--- a/website/i18n/en.json
+++ b/website/i18n/en.json
@@ -119,12 +119,12 @@
"guides/sessionmanagement": {
"title": "Session Management"
},
- "guides/whatisanactor": {
- "title": "What is an Actor"
- },
"guides/typescriptactor": {
"title": "Typescript Actors"
},
+ "guides/whatisanactor": {
+ "title": "What is an Actor"
+ },
"readme/introduction": {
"title": "readme/introduction"
},
@@ -158,6 +158,9 @@
"typedefs/cheeriohandlepageinputs": {
"title": "CheerioHandlePageInputs"
},
+ "typedefs/createsession": {
+ "title": "CreateSession"
+ },
"typedefs/datasetconsumer": {
"title": "DatasetConsumer"
},
@@ -278,6 +281,7 @@
"Puppeteer User-Functions": "Puppeteer User-Functions",
"Dataset User-Functions": "Dataset User-Functions",
"KeyValueStore User-Functions": "KeyValueStore User-Functions",
+ "SessionPool User-Functions": "SessionPool User-Functions",
"Utility User-Functions": "Utility User-Functions"
}
},