Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
docs: improve `maxRequestRetries` documentation (#2957)
Jindřich Bär6d636c03c7e2
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex 973ce0557..8d6d77ac4 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -235,7 +235,12 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCrawhandleFailedRequestFunction?: ErrorHandler<Context>;/**- * Indicates how many times the request is retried if {@apilink BasicCrawlerOptions.requestHandler|`requestHandler`} fails.+ * Specifies the maximum number of retries allowed for a request if its processing fails.+ * This includes retries due to navigation errors or errors thrown from user-supplied functions+ * (`requestHandler`, `preNavigationHooks`, `postNavigationHooks`).+ *+ * This limit does not apply to retries triggered by session rotation+ * (see {@apilink BasicCrawlerOptions.maxSessionRotations|`maxSessionRotations`}).* @default 3*/maxRequestRetries?: number;diff --git a/website/versioned_docs/version-3.13/api-typedoc.json b/website/versioned_docs/version-3.13/api-typedoc.jsonindex ce0d512c1..4685fc260 100644--- a/website/versioned_docs/version-3.13/api-typedoc.json+++ b/website/versioned_docs/version-3.13/api-typedoc.json@@ -4566,16 +4566,40 @@"summary": [{"kind": "text",- "text": "Indicates how many times the request is retried if "+ "text": "Specifies the maximum number of retries allowed for a request if its processing fails.\nThis includes retries due to navigation errors or errors thrown from user-supplied functions\n("+ },+ {+ "kind": "code",+ "text": "`requestHandler`"+ },+ {+ "kind": "text",+ "text": ", "+ },+ {+ "kind": "code",+ "text": "`preNavigationHooks`"+ },+ {+ "kind": "text",+ "text": ", "+ },+ {+ "kind": "code",+ "text": "`postNavigationHooks`"+ },+ {+ "kind": "text",+ "text": ").\n\nThis limit does not apply to retries triggered by session rotation\n(see "},{"kind": "inline-tag","tag": "@apilink",- "text": "BasicCrawlerOptions.requestHandler|`requestHandler`"+ "text": "BasicCrawlerOptions.maxSessionRotations|`maxSessionRotations`"},{"kind": "text",- "text": " fails."+ "text": ")."}],"blockTags": [@@ -15576,16 +15600,40 @@"summary": [{"kind": "text",- "text": "Indicates how many times the request is retried if "+ "text": "Specifies the maximum number of retries allowed for a request if its processing fails.\nThis includes retries due to navigation errors or errors thrown from user-supplied functions \n("+ },+ {+ "kind": "code",+ "text": "`requestHandler`"+ },+ {+ "kind": "text",+ "text": ", "+ },+ {+ "kind": "code",+ "text": "`preNavigationHooks`"+ },+ {+ "kind": "text",+ "text": ", "+ },+ {+ "kind": "code",+ "text": "`postNavigationHooks`"+ },+ {+ "kind": "text",+ "text": ").\n\nThis limit does not apply to retries triggered by session rotation \n(see "},{"kind": "inline-tag","tag": "@apilink",- "text": "BasicCrawlerOptions.requestHandler|`requestHandler`"+ "text": "BasicCrawlerOptions.maxSessionRotations|`maxSessionRotations`"},{"kind": "text",- "text": " fails."+ "text": ")."}],"blockTags": [@@ -117496,16 +117544,40 @@"summary": [{"kind": "text",- "text": "Indicates how many times the request is retried if "+ "text": "Specifies the maximum number of retries allowed for a request if its processing fails.\nThis includes retries due to navigation errors or errors thrown from user-supplied functions \n("+ },+ {+ "kind": "code",+ "text": "`requestHandler`"+ },+ {+ "kind": "text",+ "text": ", "+ },+ {+ "kind": "code",+ "text": "`preNavigationHooks`"+ },+ {+ "kind": "text",+ "text": ", "+ },+ {+ "kind": "code",+ "text": "`postNavigationHooks`"+ },+ {+ "kind": "text",+ "text": ").\n\nThis limit does not apply to retries triggered by session rotation \n(see "},{"kind": "inline-tag","tag": "@apilink",- "text": "BasicCrawlerOptions.requestHandler|`requestHandler`"+ "text": "BasicCrawlerOptions.maxSessionRotations|`maxSessionRotations`"},{"kind": "text",- "text": " fails."+ "text": ")."}],"blockTags": [@@ -214231,16 +214303,40 @@"summary": [{"kind": "text",- "text": "Indicates how many times the request is retried if "+ "text": "Specifies the maximum number of retries allowed for a request if its processing fails.\nThis includes retries due to navigation errors or errors thrown from user-supplied functions \n("+ },+ {+ "kind": "code",+ "text": "`requestHandler`"+ },+ {+ "kind": "text",+ "text": ", "+ },+ {+ "kind": "code",+ "text": "`preNavigationHooks`"+ },+ {+ "kind": "text",+ "text": ", "+ },+ {+ "kind": "code",+ "text": "`postNavigationHooks`"+ },+ {+ "kind": "text",+ "text": ").\n\nThis limit does not apply to retries triggered by session rotation \n(see "},{"kind": "inline-tag","tag": "@apilink",- "text": "BasicCrawlerOptions.requestHandler|`requestHandler`"+ "text": "BasicCrawlerOptions.maxSessionRotations|`maxSessionRotations`"},{"kind": "text",- "text": " fails."+ "text": ")."}],"blockTags": [@@ -225190,16 +225286,40 @@"summary": [{"kind": "text",- "text": "Indicates how many times the request is retried if "+ "text": "Specifies the maximum number of retries allowed for a request if its processing fails.\nThis includes retries due to navigation errors or errors thrown from user-supplied functions \n("+ },+ {+ "kind": "code",+ "text": "`requestHandler`"+ },+ {+ "kind": "text",+ "text": ", "+ },+ {+ "kind": "code",+ "text": "`preNavigationHooks`"+ },+ {+ "kind": "text",+ "text": ", "+ },+ {+ "kind": "code",+ "text": "`postNavigationHooks`"+ },+ {+ "kind": "text",+ "text": ").\n\nThis limit does not apply to retries triggered by session rotation \n(see "},{"kind": "inline-tag","tag": "@apilink",- "text": "BasicCrawlerOptions.requestHandler|`requestHandler`"+ "text": "BasicCrawlerOptions.maxSessionRotations|`maxSessionRotations`"},{"kind": "text",- "text": " fails."+ "text": ")."}],"blockTags": [@@ -236497,16 +236617,40 @@"summary": [{"kind": "text",- "text": "Indicates how many times the request is retried if "+ "text": "Specifies the maximum number of retries allowed for a request if its processing fails.\nThis includes retries due to navigation errors or errors thrown from user-supplied functions \n("+ },+ {+ "kind": "code",+ "text": "`requestHandler`"+ },+ {+ "kind": "text",+ "text": ", "+ },+ {+ "kind": "code",+ "text": "`preNavigationHooks`"+ },+ {+ "kind": "text",+ "text": ", "+ },+ {+ "kind": "code",+ "text": "`postNavigationHooks`"+ },+ {+ "kind": "text",+ "text": ").\n\nThis limit does not apply to retries triggered by session rotation \n(see "},{"kind": "inline-tag","tag": "@apilink",- "text": "BasicCrawlerOptions.requestHandler|`requestHandler`"+ "text": "BasicCrawlerOptions.maxSessionRotations|`maxSessionRotations`"},{"kind": "text",- "text": " fails."+ "text": ")."}],"blockTags": [@@ -284148,16 +284292,40 @@"summary": [{"kind": "text",- "text": "Indicates how many times the request is retried if "+ "text": "Specifies the maximum number of retries allowed for a request if its processing fails.\nThis includes retries due to navigation errors or errors thrown from user-supplied functions \n("+ },+ {+ "kind": "code",+ "text": "`requestHandler`"+ },+ {+ "kind": "text",+ "text": ", "+ },+ {+ "kind": "code",+ "text": "`preNavigationHooks`"+ },+ {+ "kind": "text",+ "text": ", "+ },+ {+ "kind": "code",+ "text": "`postNavigationHooks`"+ },+ {+ "kind": "text",+ "text": ").\n\nThis limit does not apply to retries triggered by session rotation \n(see "},{"kind": "inline-tag","tag": "@apilink",- "text": "BasicCrawlerOptions.requestHandler|`requestHandler`"+ "text": "BasicCrawlerOptions.maxSessionRotations|`maxSessionRotations`"},{"kind": "text",- "text": " fails."+ "text": ")."}],"blockTags": [@@ -293759,16 +293927,40 @@"summary": [{"kind": "text",- "text": "Indicates how many times the request is retried if "+ "text": "Specifies the maximum number of retries allowed for a request if its processing fails.\nThis includes retries due to navigation errors or errors thrown from user-supplied functions \n("+ },+ {+ "kind": "code",+ "text": "`requestHandler`"+ },+ {+ "kind": "text",+ "text": ", "+ },+ {+ "kind": "code",+ "text": "`preNavigationHooks`"+ },+ {+ "kind": "text",+ "text": ", "+ },+ {+ "kind": "code",+ "text": "`postNavigationHooks`"+ },+ {+ "kind": "text",+ "text": ").\n\nThis limit does not apply to retries triggered by session rotation \n(see "},{"kind": "inline-tag","tag": "@apilink",- "text": "BasicCrawlerOptions.requestHandler|`requestHandler`"+ "text": "BasicCrawlerOptions.maxSessionRotations|`maxSessionRotations`"},{"kind": "text",- "text": " fails."+ "text": ")."}],"blockTags": [@@ -314395,16 +314587,40 @@"summary": [{"kind": "text",- "text": "Indicates how many times the request is retried if "+ "text": "Specifies the maximum number of retries allowed for a request if its processing fails.\nThis includes retries due to navigation errors or errors thrown from user-supplied functions \n("+ },+ {+ "kind": "code",+ "text": "`requestHandler`"+ },+ {+ "kind": "text",+ "text": ", "+ },+ {+ "kind": "code",+ "text": "`preNavigationHooks`"+ },+ {+ "kind": "text",+ "text": ", "+ },+ {+ "kind": "code",+ "text": "`postNavigationHooks`"+ },+ {+ "kind": "text",+ "text": ").\n\nThis limit does not apply to retries triggered by session rotation \n(see "},{"kind": "inline-tag","tag": "@apilink",- "text": "BasicCrawlerOptions.requestHandler|`requestHandler`"+ "text": "BasicCrawlerOptions.maxSessionRotations|`maxSessionRotations`"},{"kind": "text",- "text": " fails."+ "text": ")."}],"blockTags": [