Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

docs: improve `maxRequestRetries` documentation (#2957)

Jindřich Bär6d636c03c7e2
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
index 973ce0557..8d6d77ac4 100644
--- a/packages/basic-crawler/src/internals/basic-crawler.ts
+++ b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -235,7 +235,12 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCraw
handleFailedRequestFunction?: ErrorHandler<Context>;
/**
- * Indicates how many times the request is retried if {@apilink BasicCrawlerOptions.requestHandler|`requestHandler`} fails.
+ * Specifies the maximum number of retries allowed for a request if its processing fails.
+ * This includes retries due to navigation errors or errors thrown from user-supplied functions
+ * (`requestHandler`, `preNavigationHooks`, `postNavigationHooks`).
+ *
+ * This limit does not apply to retries triggered by session rotation
+ * (see {@apilink BasicCrawlerOptions.maxSessionRotations|`maxSessionRotations`}).
* @default 3
*/
maxRequestRetries?: number;
diff --git a/website/versioned_docs/version-3.13/api-typedoc.json b/website/versioned_docs/version-3.13/api-typedoc.json
index ce0d512c1..4685fc260 100644
--- a/website/versioned_docs/version-3.13/api-typedoc.json
+++ b/website/versioned_docs/version-3.13/api-typedoc.json
@@ -4566,16 +4566,40 @@
"summary": [
{
"kind": "text",
- "text": "Indicates how many times the request is retried if "
+ "text": "Specifies the maximum number of retries allowed for a request if its processing fails.\nThis includes retries due to navigation errors or errors thrown from user-supplied functions\n("
+ },
+ {
+ "kind": "code",
+ "text": "`requestHandler`"
+ },
+ {
+ "kind": "text",
+ "text": ", "
+ },
+ {
+ "kind": "code",
+ "text": "`preNavigationHooks`"
+ },
+ {
+ "kind": "text",
+ "text": ", "
+ },
+ {
+ "kind": "code",
+ "text": "`postNavigationHooks`"
+ },
+ {
+ "kind": "text",
+ "text": ").\n\nThis limit does not apply to retries triggered by session rotation\n(see "
},
{
"kind": "inline-tag",
"tag": "@apilink",
- "text": "BasicCrawlerOptions.requestHandler|`requestHandler`"
+ "text": "BasicCrawlerOptions.maxSessionRotations|`maxSessionRotations`"
},
{
"kind": "text",
- "text": " fails."
+ "text": ")."
}
],
"blockTags": [
@@ -15576,16 +15600,40 @@
"summary": [
{
"kind": "text",
- "text": "Indicates how many times the request is retried if "
+ "text": "Specifies the maximum number of retries allowed for a request if its processing fails.\nThis includes retries due to navigation errors or errors thrown from user-supplied functions \n("
+ },
+ {
+ "kind": "code",
+ "text": "`requestHandler`"
+ },
+ {
+ "kind": "text",
+ "text": ", "
+ },
+ {
+ "kind": "code",
+ "text": "`preNavigationHooks`"
+ },
+ {
+ "kind": "text",
+ "text": ", "
+ },
+ {
+ "kind": "code",
+ "text": "`postNavigationHooks`"
+ },
+ {
+ "kind": "text",
+ "text": ").\n\nThis limit does not apply to retries triggered by session rotation \n(see "
},
{
"kind": "inline-tag",
"tag": "@apilink",
- "text": "BasicCrawlerOptions.requestHandler|`requestHandler`"
+ "text": "BasicCrawlerOptions.maxSessionRotations|`maxSessionRotations`"
},
{
"kind": "text",
- "text": " fails."
+ "text": ")."
}
],
"blockTags": [
@@ -117496,16 +117544,40 @@
"summary": [
{
"kind": "text",
- "text": "Indicates how many times the request is retried if "
+ "text": "Specifies the maximum number of retries allowed for a request if its processing fails.\nThis includes retries due to navigation errors or errors thrown from user-supplied functions \n("
+ },
+ {
+ "kind": "code",
+ "text": "`requestHandler`"
+ },
+ {
+ "kind": "text",
+ "text": ", "
+ },
+ {
+ "kind": "code",
+ "text": "`preNavigationHooks`"
+ },
+ {
+ "kind": "text",
+ "text": ", "
+ },
+ {
+ "kind": "code",
+ "text": "`postNavigationHooks`"
+ },
+ {
+ "kind": "text",
+ "text": ").\n\nThis limit does not apply to retries triggered by session rotation \n(see "
},
{
"kind": "inline-tag",
"tag": "@apilink",
- "text": "BasicCrawlerOptions.requestHandler|`requestHandler`"
+ "text": "BasicCrawlerOptions.maxSessionRotations|`maxSessionRotations`"
},
{
"kind": "text",
- "text": " fails."
+ "text": ")."
}
],
"blockTags": [
@@ -214231,16 +214303,40 @@
"summary": [
{
"kind": "text",
- "text": "Indicates how many times the request is retried if "
+ "text": "Specifies the maximum number of retries allowed for a request if its processing fails.\nThis includes retries due to navigation errors or errors thrown from user-supplied functions \n("
+ },
+ {
+ "kind": "code",
+ "text": "`requestHandler`"
+ },
+ {
+ "kind": "text",
+ "text": ", "
+ },
+ {
+ "kind": "code",
+ "text": "`preNavigationHooks`"
+ },
+ {
+ "kind": "text",
+ "text": ", "
+ },
+ {
+ "kind": "code",
+ "text": "`postNavigationHooks`"
+ },
+ {
+ "kind": "text",
+ "text": ").\n\nThis limit does not apply to retries triggered by session rotation \n(see "
},
{
"kind": "inline-tag",
"tag": "@apilink",
- "text": "BasicCrawlerOptions.requestHandler|`requestHandler`"
+ "text": "BasicCrawlerOptions.maxSessionRotations|`maxSessionRotations`"
},
{
"kind": "text",
- "text": " fails."
+ "text": ")."
}
],
"blockTags": [
@@ -225190,16 +225286,40 @@
"summary": [
{
"kind": "text",
- "text": "Indicates how many times the request is retried if "
+ "text": "Specifies the maximum number of retries allowed for a request if its processing fails.\nThis includes retries due to navigation errors or errors thrown from user-supplied functions \n("
+ },
+ {
+ "kind": "code",
+ "text": "`requestHandler`"
+ },
+ {
+ "kind": "text",
+ "text": ", "
+ },
+ {
+ "kind": "code",
+ "text": "`preNavigationHooks`"
+ },
+ {
+ "kind": "text",
+ "text": ", "
+ },
+ {
+ "kind": "code",
+ "text": "`postNavigationHooks`"
+ },
+ {
+ "kind": "text",
+ "text": ").\n\nThis limit does not apply to retries triggered by session rotation \n(see "
},
{
"kind": "inline-tag",
"tag": "@apilink",
- "text": "BasicCrawlerOptions.requestHandler|`requestHandler`"
+ "text": "BasicCrawlerOptions.maxSessionRotations|`maxSessionRotations`"
},
{
"kind": "text",
- "text": " fails."
+ "text": ")."
}
],
"blockTags": [
@@ -236497,16 +236617,40 @@
"summary": [
{
"kind": "text",
- "text": "Indicates how many times the request is retried if "
+ "text": "Specifies the maximum number of retries allowed for a request if its processing fails.\nThis includes retries due to navigation errors or errors thrown from user-supplied functions \n("
+ },
+ {
+ "kind": "code",
+ "text": "`requestHandler`"
+ },
+ {
+ "kind": "text",
+ "text": ", "
+ },
+ {
+ "kind": "code",
+ "text": "`preNavigationHooks`"
+ },
+ {
+ "kind": "text",
+ "text": ", "
+ },
+ {
+ "kind": "code",
+ "text": "`postNavigationHooks`"
+ },
+ {
+ "kind": "text",
+ "text": ").\n\nThis limit does not apply to retries triggered by session rotation \n(see "
},
{
"kind": "inline-tag",
"tag": "@apilink",
- "text": "BasicCrawlerOptions.requestHandler|`requestHandler`"
+ "text": "BasicCrawlerOptions.maxSessionRotations|`maxSessionRotations`"
},
{
"kind": "text",
- "text": " fails."
+ "text": ")."
}
],
"blockTags": [
@@ -284148,16 +284292,40 @@
"summary": [
{
"kind": "text",
- "text": "Indicates how many times the request is retried if "
+ "text": "Specifies the maximum number of retries allowed for a request if its processing fails.\nThis includes retries due to navigation errors or errors thrown from user-supplied functions \n("
+ },
+ {
+ "kind": "code",
+ "text": "`requestHandler`"
+ },
+ {
+ "kind": "text",
+ "text": ", "
+ },
+ {
+ "kind": "code",
+ "text": "`preNavigationHooks`"
+ },
+ {
+ "kind": "text",
+ "text": ", "
+ },
+ {
+ "kind": "code",
+ "text": "`postNavigationHooks`"
+ },
+ {
+ "kind": "text",
+ "text": ").\n\nThis limit does not apply to retries triggered by session rotation \n(see "
},
{
"kind": "inline-tag",
"tag": "@apilink",
- "text": "BasicCrawlerOptions.requestHandler|`requestHandler`"
+ "text": "BasicCrawlerOptions.maxSessionRotations|`maxSessionRotations`"
},
{
"kind": "text",
- "text": " fails."
+ "text": ")."
}
],
"blockTags": [
@@ -293759,16 +293927,40 @@
"summary": [
{
"kind": "text",
- "text": "Indicates how many times the request is retried if "
+ "text": "Specifies the maximum number of retries allowed for a request if its processing fails.\nThis includes retries due to navigation errors or errors thrown from user-supplied functions \n("
+ },
+ {
+ "kind": "code",
+ "text": "`requestHandler`"
+ },
+ {
+ "kind": "text",
+ "text": ", "
+ },
+ {
+ "kind": "code",
+ "text": "`preNavigationHooks`"
+ },
+ {
+ "kind": "text",
+ "text": ", "
+ },
+ {
+ "kind": "code",
+ "text": "`postNavigationHooks`"
+ },
+ {
+ "kind": "text",
+ "text": ").\n\nThis limit does not apply to retries triggered by session rotation \n(see "
},
{
"kind": "inline-tag",
"tag": "@apilink",
- "text": "BasicCrawlerOptions.requestHandler|`requestHandler`"
+ "text": "BasicCrawlerOptions.maxSessionRotations|`maxSessionRotations`"
},
{
"kind": "text",
- "text": " fails."
+ "text": ")."
}
],
"blockTags": [
@@ -314395,16 +314587,40 @@
"summary": [
{
"kind": "text",
- "text": "Indicates how many times the request is retried if "
+ "text": "Specifies the maximum number of retries allowed for a request if its processing fails.\nThis includes retries due to navigation errors or errors thrown from user-supplied functions \n("
+ },
+ {
+ "kind": "code",
+ "text": "`requestHandler`"
+ },
+ {
+ "kind": "text",
+ "text": ", "
+ },
+ {
+ "kind": "code",
+ "text": "`preNavigationHooks`"
+ },
+ {
+ "kind": "text",
+ "text": ", "
+ },
+ {
+ "kind": "code",
+ "text": "`postNavigationHooks`"
+ },
+ {
+ "kind": "text",
+ "text": ").\n\nThis limit does not apply to retries triggered by session rotation \n(see "
},
{
"kind": "inline-tag",
"tag": "@apilink",
- "text": "BasicCrawlerOptions.requestHandler|`requestHandler`"
+ "text": "BasicCrawlerOptions.maxSessionRotations|`maxSessionRotations`"
},
{
"kind": "text",
- "text": " fails."
+ "text": ")."
}
],
"blockTags": [