Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

feat(core): add `Request.maxRetries` to allow overriding the `maxRequestRetries` (#1925)

Martin Adámekc5592db0f809
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
index 915a8ef7d..bbb72f584 100644
--- a/packages/basic-crawler/src/internals/basic-crawler.ts
+++ b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -1207,7 +1207,8 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
}
// Ensure there are more retries available for the request
- return request.retryCount < this.maxRequestRetries;
+ const maxRequestRetries = request.maxRetries ?? this.maxRequestRetries;
+ return request.retryCount < maxRequestRetries;
}
protected _augmentContextWithDeprecatedError(context: Context, error: Error) {
diff --git a/packages/core/src/request.ts b/packages/core/src/request.ts
index 54f8fdacd..7fc500be9 100644
--- a/packages/core/src/request.ts
+++ b/packages/core/src/request.ts
@@ -20,6 +20,7 @@ const requestOptionalPredicates = {
payload: ow.optional.any(ow.string, ow.buffer),
noRetry: ow.optional.boolean,
retryCount: ow.optional.number,
+ maxRetries: ow.optional.number,
errorMessages: ow.optional.array.ofType(ow.string),
headers: ow.optional.object,
userData: ow.optional.object,
@@ -153,6 +154,7 @@ export class Request<UserData extends Dictionary = Dictionary> {
payload,
noRetry = false,
retryCount = 0,
+ maxRetries,
errorMessages = [],
headers = {},
userData = {},
@@ -161,7 +163,7 @@ export class Request<UserData extends Dictionary = Dictionary> {
keepUrlFragment = false,
useExtendedUniqueKey = false,
skipNavigation,
- } = options as RequestOptions & { loadedUrl?: string; retryCount?: number; errorMessages?: string[]; handledAt?: string | Date };
+ } = options as RequestOptions & { loadedUrl?: string; retryCount?: number; maxRetries?: number; errorMessages?: string[]; handledAt?: string | Date };
let {
method = 'GET',
@@ -227,6 +229,7 @@ export class Request<UserData extends Dictionary = Dictionary> {
this.userData = userData;
if (skipNavigation != null) this.skipNavigation = skipNavigation;
+ if (maxRetries != null) this.maxRetries = maxRetries;
}
/** Tells the crawler processing this request to skip the navigation and process the request directly. */
@@ -236,8 +239,11 @@ export class Request<UserData extends Dictionary = Dictionary> {
/** Tells the crawler processing this request to skip the navigation and process the request directly. */
set skipNavigation(value: boolean) {
- if (!this.userData.__crawlee) (this.userData as Dictionary).__crawlee = { skipNavigation: value };
- else this.userData.__crawlee.skipNavigation = value;
+ if (!this.userData.__crawlee) {
+ (this.userData as Dictionary).__crawlee = { skipNavigation: value };
+ } else {
+ this.userData.__crawlee.skipNavigation = value;
+ }
}
/** shortcut for getting `request.userData.label` */
@@ -250,6 +256,20 @@ export class Request<UserData extends Dictionary = Dictionary> {
(this.userData as Dictionary).label = value;
}
+ /** Maximum number of retries for this request. Allows to override the global `maxRequestRetries` option of `BasicCrawler`. */
+ get maxRetries(): number | undefined {
+ return this.userData.__crawlee?.maxRetries;
+ }
+
+ /** Maximum number of retries for this request. Allows to override the global `maxRequestRetries` option of `BasicCrawler`. */
+ set maxRetries(value: number | undefined) {
+ if (!this.userData.__crawlee) {
+ (this.userData as Dictionary).__crawlee = { maxRetries: value };
+ } else {
+ this.userData.__crawlee.maxRetries = value;
+ }
+ }
+
/** Describes the request's current lifecycle state. */
get state(): RequestState {
return this.userData.__crawlee?.state ?? RequestState.UNPROCESSED;
@@ -257,8 +277,11 @@ export class Request<UserData extends Dictionary = Dictionary> {
/** Describes the request's current lifecycle state. */
set state(value: RequestState) {
- if (!this.userData.__crawlee) (this.userData as Dictionary).__crawlee = { state: value };
- else this.userData.__crawlee.state = value;
+ if (!this.userData.__crawlee) {
+ (this.userData as Dictionary).__crawlee = { state: value };
+ } else {
+ this.userData.__crawlee.state = value;
+ }
}
/**
diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.ts
index d5097c9d3..3cfdcd5b9 100644
--- a/test/core/crawlers/basic_crawler.test.ts
+++ b/test/core/crawlers/basic_crawler.test.ts
@@ -317,6 +317,41 @@ describe('BasicCrawler', () => {
expect(await requestList.isEmpty()).toBe(true);
});
+ test('should retry failed requests based on `request.maxRetries`', async () => {
+ const sources = [
+ { url: 'http://example.com/1', maxRetries: 10 },
+ { url: 'http://example.com/2', maxRetries: 5 },
+ { url: 'http://example.com/3', maxRetries: 1 },
+ ];
+ const processed: Dictionary<Request> = {};
+ const requestList = await RequestList.open(null, sources);
+
+ const requestHandler: RequestHandler = async ({ request }) => {
+ await sleep(10);
+ processed[request.url] = request;
+ throw Error(`This is ${request.retryCount}th error!`);
+ };
+
+ const basicCrawler = new BasicCrawler({
+ requestList,
+ minConcurrency: 3,
+ maxConcurrency: 3,
+ requestHandler,
+ });
+
+ await basicCrawler.run();
+
+ expect(processed['http://example.com/1'].errorMessages).toHaveLength(11);
+ expect(processed['http://example.com/1'].retryCount).toBe(10);
+ expect(processed['http://example.com/2'].errorMessages).toHaveLength(6);
+ expect(processed['http://example.com/2'].retryCount).toBe(5);
+ expect(processed['http://example.com/3'].errorMessages).toHaveLength(2);
+ expect(processed['http://example.com/3'].retryCount).toBe(1);
+
+ expect(await requestList.isFinished()).toBe(true);
+ expect(await requestList.isEmpty()).toBe(true);
+ });
+
test('should not retry requests with noRetry set to true ', async () => {
const noRetryRequest = new Request({ url: 'http://example.com/3' });
noRetryRequest.noRetry = true;
diff --git a/test/core/storages/request_queue.test.ts b/test/core/storages/request_queue.test.ts
index 391ddee8d..021d9f2a9 100644
--- a/test/core/storages/request_queue.test.ts
+++ b/test/core/storages/request_queue.test.ts
@@ -685,7 +685,7 @@ describe('RequestQueue remote', () => {
const r1 = new Request({
url,
method,
- userData: { __crawlee: { skipNavigation: true, foo: 123, bar: true } },
+ userData: { __crawlee: { skipNavigation: true, maxRetries: 10, foo: 123, bar: true } },
});
const r2 = new Request({
url,
@@ -699,13 +699,19 @@ describe('RequestQueue remote', () => {
const desc1 = Object.getOwnPropertyDescriptor(r1.userData, '__crawlee');
expect(desc1.enumerable).toBe(false);
expect(r1.skipNavigation).toBe(true);
- expect(r1.userData.__crawlee).toMatchObject({ skipNavigation: true, foo: 123, bar: true });
+ expect(r1.maxRetries).toBe(10);
+ r1.maxRetries = 5;
+ expect(r1.userData.__crawlee).toMatchObject({ skipNavigation: true, maxRetries: 5, foo: 123, bar: true });
const desc2 = Object.getOwnPropertyDescriptor(r2.userData, '__crawlee');
expect(desc2.enumerable).toBe(false);
+ expect(r2.maxRetries).toBeUndefined();
expect(r2.userData.__crawlee).toEqual({});
const desc3 = Object.getOwnPropertyDescriptor(r3.userData, '__crawlee');
expect(desc3.enumerable).toBe(false);
+ expect(r3.maxRetries).toBeUndefined();
expect(r3.userData.__crawlee).toEqual({});
+ r3.maxRetries = 2;
+ expect(r3.userData.__crawlee).toEqual({ maxRetries: 2 });
});
});