Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: Correctly track the number of requests handled by a crawler (#3410)

Jan Bucharb2296cea708f
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
index 1dc5efea4..3ff8e2c63 100644
--- a/packages/basic-crawler/src/internals/basic-crawler.ts
+++ b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -550,7 +550,19 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
protected domainAccessedTime: Map<string, number>;
protected maxSessionRotations: number;
protected maxRequestsPerCrawl?: number;
- protected handledRequestsCount = 0;
+
+ protected get handledRequestsCount(): number {
+ return this.stats.state.requestsFinished + this.stats.state.requestsFailed;
+ }
+
+ /** @deprecated Setting `handledRequestsCount` directly is no longer supported. The count is now derived from `this.stats`. */
+ protected set handledRequestsCount(_value: number) {
+ throw new Error(
+ 'Setting `handledRequestsCount` directly is no longer supported. ' +
+ 'The count is now derived from `this.stats.state.requestsFinished` and `this.stats.state.requestsFailed`.',
+ );
+ }
+
protected statusMessageLoggingInterval: number;
protected statusMessageCallback?: StatusMessageCallback;
protected sessionPoolOptions: SessionPoolOptions;
@@ -983,7 +995,6 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
this.requestQueue = await this._getRequestQueue();
this.requestManager = undefined;
await this.initializeRequestManager();
- this.handledRequestsCount = 0; // This would've been reset by this._init() further down below, but at that point `handledRequestsCount` could prevent `addRequests` from adding the initial requests
}
this.stats.reset();
@@ -1348,9 +1359,6 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
this._closeEvents = true;
}
- // Initialize AutoscaledPool before awaiting _loadHandledRequestCount(),
- // so that the caller can get a reference to it before awaiting the promise returned from run()
- // (otherwise there would be no way)
this.autoscaledPool = new AutoscaledPool(this.autoscaledPoolOptions, this.config);
if (this.useSessionPool) {
@@ -1360,7 +1368,6 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
}
await this.initializeRequestManager();
- await this._loadHandledRequestCount();
}
protected async _runRequestHandler(crawlingContext: Context): Promise<void> {
@@ -1632,7 +1639,6 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
isRequestLocked = false; // markRequestHandled succeeded and unlocked the request
this.stats.finishJob(statisticsId, request.retryCount);
- this.handledRequestsCount++;
// reclaim session if request finishes successfully
request.state = RequestState.DONE;
@@ -1863,7 +1869,6 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
// If we get here, the request is either not retryable
// or failed more than retryCount times and will not be retried anymore.
// Mark the request as failed and do not retry.
- this.handledRequestsCount++;
await source.markRequestHandled(request);
this.stats.failJob(request.id || request.uniqueKey, request.retryCount);
@@ -1952,15 +1957,6 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
return context as LoadedContext<Context>;
}
- /**
- * Updates handledRequestsCount from possibly stored counts, usually after worker migration.
- */
- protected async _loadHandledRequestCount(): Promise<void> {
- if (this.requestManager) {
- this.handledRequestsCount = await this.requestManager.handledCount();
- }
- }
-
protected async _executeHooks<HookLike extends (...args: any[]) => Awaitable<void>>(
hooks: HookLike[],
...args: Parameters<HookLike>
diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.ts
index cd160f0d0..222b18d47 100644
--- a/test/core/crawlers/basic_crawler.test.ts
+++ b/test/core/crawlers/basic_crawler.test.ts
@@ -1095,7 +1095,7 @@ describe('BasicCrawler', () => {
expect(await requestList.isEmpty()).toBe(false);
});
- test('should load handledRequestCount from storages', async () => {
+ test('should derive handledRequestCount from Statistics', async () => {
const requestQueue = new RequestQueue({ id: 'id', client: Configuration.getStorageClient() });
requestQueue.isEmpty = async () => false;
requestQueue.isFinished = async () => false;
@@ -1104,53 +1104,12 @@ describe('BasicCrawler', () => {
// @ts-expect-error Overriding the method for testing purposes
requestQueue.markRequestHandled = async () => {};
- const requestQueueStub = vitest.spyOn(requestQueue, 'handledCount').mockResolvedValue(33);
+ // Even though the request queue reports 33 handled requests (e.g. from a previous crawler run),
+ // the crawler should use its own Statistics to track the count and process all 40 requests.
+ vitest.spyOn(requestQueue, 'handledCount').mockResolvedValue(33);
let count = 0;
- let crawler = new BasicCrawler({
- requestQueue,
- maxConcurrency: 1,
- requestHandler: async () => {
- await sleep(1);
- count++;
- },
- maxRequestsPerCrawl: 40,
- });
-
- await crawler.run();
- expect(requestQueueStub).toBeCalled();
- expect(count).toBe(7);
- vitest.restoreAllMocks();
-
- const sources = Array.from(Array(10).keys(), (x) => x + 1).map((i) => ({ url: `http://example.com/${i}` }));
- const sourcesCopy = JSON.parse(JSON.stringify(sources));
- let requestList = await RequestList.open({ sources });
- const requestListStub = vitest.spyOn(requestList, 'handledCount').mockReturnValue(33);
-
- count = 0;
- crawler = new BasicCrawler({
- requestList,
- maxConcurrency: 1,
- requestHandler: async () => {
- await sleep(1);
- count++;
- },
- maxRequestsPerCrawl: 40,
- });
-
- await crawler.run();
- expect(requestListStub).toBeCalled();
- expect(count).toBe(7);
- vitest.restoreAllMocks();
-
- requestList = await RequestList.open({ sources: sourcesCopy });
- const listStub = vitest.spyOn(requestList, 'handledCount').mockReturnValue(20);
- const queueStub = vitest.spyOn(requestQueue, 'handledCount').mockResolvedValue(33);
- const addRequestStub = vitest.spyOn(requestQueue, 'addRequest').mockReturnValue(Promise.resolve() as any);
-
- count = 0;
- crawler = new BasicCrawler({
- requestList,
+ const crawler = new BasicCrawler({
requestQueue,
maxConcurrency: 1,
requestHandler: async () => {
@@ -1161,12 +1120,8 @@ describe('BasicCrawler', () => {
});
await crawler.run();
-
- expect(queueStub).toBeCalled();
- expect(listStub).not.toBeCalled();
- expect(addRequestStub).toBeCalledTimes(7);
- expect(count).toBe(7);
-
+ // The crawler should have processed 40 requests (its own limit), not 7 (40 - 33).
+ expect(count).toBe(40);
vitest.restoreAllMocks();
});
@@ -1441,12 +1396,6 @@ describe('BasicCrawler', () => {
failedRequestHandler: async () => {},
});
- // @ts-expect-error Accessing private prop
- crawler._loadHandledRequestCount = () => {
- expect(crawler.sessionPool).toBeDefined();
- expect(events.listenerCount(EventType.PERSIST_STATE)).toEqual(1);
- };
-
await crawler.run();
expect(events.listenerCount(EventType.PERSIST_STATE)).toEqual(0);
// @ts-expect-error private symbol
@@ -1605,7 +1554,7 @@ describe('BasicCrawler', () => {
requestHandler: async () => {},
});
- crawler['handledRequestsCount'] = 2; // eslint-disable-line dot-notation
+ crawler.stats.state.requestsFinished = 2;
// Try to add 6 requests - should only add 3 due to limit
const requestsToAdd = [
@@ -1638,7 +1587,7 @@ describe('BasicCrawler', () => {
requestHandler: async () => {},
});
- crawler['handledRequestsCount'] = 1; // eslint-disable-line dot-notation
+ crawler.stats.state.requestsFinished = 1;
// First call - should add 2 requests (2 more slots to go)
await crawler.addRequests(['http://example.com/1', 'http://example.com/2']);
@@ -1687,7 +1636,7 @@ describe('BasicCrawler', () => {
requestHandler: async () => {},
});
- crawler['handledRequestsCount'] = 0; // eslint-disable-line dot-notation
+ crawler.stats.state.requestsFinished = 0;
// Mock robots.txt checking to disallow some URLs
vitest.spyOn(crawler as any, 'isAllowedBasedOnRobotsTxtFile').mockImplementation(async (url) => {
@@ -1794,7 +1743,7 @@ describe('BasicCrawler', () => {
return;
}
- crawler['handledRequestsCount'] = 2; // eslint-disable-line dot-notation
+ crawler.stats.state.requestsFinished = 2;
await context.enqueueLinks({ urls: requestsToAdd, label: 'not-undefined' });
},