Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: Correctly track the number of requests handled by a crawler (#3410)
Jan Bucharb2296cea708f
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex 1dc5efea4..3ff8e2c63 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -550,7 +550,19 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextprotected domainAccessedTime: Map<string, number>;protected maxSessionRotations: number;protected maxRequestsPerCrawl?: number;- protected handledRequestsCount = 0;++ protected get handledRequestsCount(): number {+ return this.stats.state.requestsFinished + this.stats.state.requestsFailed;+ }++ /** @deprecated Setting `handledRequestsCount` directly is no longer supported. The count is now derived from `this.stats`. */+ protected set handledRequestsCount(_value: number) {+ throw new Error(+ 'Setting `handledRequestsCount` directly is no longer supported. ' ++ 'The count is now derived from `this.stats.state.requestsFinished` and `this.stats.state.requestsFailed`.',+ );+ }+protected statusMessageLoggingInterval: number;protected statusMessageCallback?: StatusMessageCallback;protected sessionPoolOptions: SessionPoolOptions;@@ -983,7 +995,6 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextthis.requestQueue = await this._getRequestQueue();this.requestManager = undefined;await this.initializeRequestManager();- this.handledRequestsCount = 0; // This would've been reset by this._init() further down below, but at that point `handledRequestsCount` could prevent `addRequests` from adding the initial requests}this.stats.reset();@@ -1348,9 +1359,6 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextthis._closeEvents = true;}- // Initialize AutoscaledPool before awaiting _loadHandledRequestCount(),- // so that the caller can get a reference to it before awaiting the promise returned from run()- // (otherwise there would be no way)this.autoscaledPool = new AutoscaledPool(this.autoscaledPoolOptions, this.config);if (this.useSessionPool) {@@ -1360,7 +1368,6 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext}await this.initializeRequestManager();- await this._loadHandledRequestCount();}protected async _runRequestHandler(crawlingContext: Context): Promise<void> {@@ -1632,7 +1639,6 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextisRequestLocked = false; // markRequestHandled succeeded and unlocked the requestthis.stats.finishJob(statisticsId, request.retryCount);- this.handledRequestsCount++;// reclaim session if request finishes successfullyrequest.state = RequestState.DONE;@@ -1863,7 +1869,6 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext// If we get here, the request is either not retryable// or failed more than retryCount times and will not be retried anymore.// Mark the request as failed and do not retry.- this.handledRequestsCount++;await source.markRequestHandled(request);this.stats.failJob(request.id || request.uniqueKey, request.retryCount);@@ -1952,15 +1957,6 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextreturn context as LoadedContext<Context>;}- /**- * Updates handledRequestsCount from possibly stored counts, usually after worker migration.- */- protected async _loadHandledRequestCount(): Promise<void> {- if (this.requestManager) {- this.handledRequestsCount = await this.requestManager.handledCount();- }- }-protected async _executeHooks<HookLike extends (...args: any[]) => Awaitable<void>>(hooks: HookLike[],...args: Parameters<HookLike>diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.tsindex cd160f0d0..222b18d47 100644--- a/test/core/crawlers/basic_crawler.test.ts+++ b/test/core/crawlers/basic_crawler.test.ts@@ -1095,7 +1095,7 @@ describe('BasicCrawler', () => {expect(await requestList.isEmpty()).toBe(false);});- test('should load handledRequestCount from storages', async () => {+ test('should derive handledRequestCount from Statistics', async () => {const requestQueue = new RequestQueue({ id: 'id', client: Configuration.getStorageClient() });requestQueue.isEmpty = async () => false;requestQueue.isFinished = async () => false;@@ -1104,53 +1104,12 @@ describe('BasicCrawler', () => {// @ts-expect-error Overriding the method for testing purposesrequestQueue.markRequestHandled = async () => {};- const requestQueueStub = vitest.spyOn(requestQueue, 'handledCount').mockResolvedValue(33);+ // Even though the request queue reports 33 handled requests (e.g. from a previous crawler run),+ // the crawler should use its own Statistics to track the count and process all 40 requests.+ vitest.spyOn(requestQueue, 'handledCount').mockResolvedValue(33);let count = 0;- let crawler = new BasicCrawler({- requestQueue,- maxConcurrency: 1,- requestHandler: async () => {- await sleep(1);- count++;- },- maxRequestsPerCrawl: 40,- });-- await crawler.run();- expect(requestQueueStub).toBeCalled();- expect(count).toBe(7);- vitest.restoreAllMocks();-- const sources = Array.from(Array(10).keys(), (x) => x + 1).map((i) => ({ url: `http://example.com/${i}` }));- const sourcesCopy = JSON.parse(JSON.stringify(sources));- let requestList = await RequestList.open({ sources });- const requestListStub = vitest.spyOn(requestList, 'handledCount').mockReturnValue(33);-- count = 0;- crawler = new BasicCrawler({- requestList,- maxConcurrency: 1,- requestHandler: async () => {- await sleep(1);- count++;- },- maxRequestsPerCrawl: 40,- });-- await crawler.run();- expect(requestListStub).toBeCalled();- expect(count).toBe(7);- vitest.restoreAllMocks();-- requestList = await RequestList.open({ sources: sourcesCopy });- const listStub = vitest.spyOn(requestList, 'handledCount').mockReturnValue(20);- const queueStub = vitest.spyOn(requestQueue, 'handledCount').mockResolvedValue(33);- const addRequestStub = vitest.spyOn(requestQueue, 'addRequest').mockReturnValue(Promise.resolve() as any);-- count = 0;- crawler = new BasicCrawler({- requestList,+ const crawler = new BasicCrawler({requestQueue,maxConcurrency: 1,requestHandler: async () => {@@ -1161,12 +1120,8 @@ describe('BasicCrawler', () => {});await crawler.run();-- expect(queueStub).toBeCalled();- expect(listStub).not.toBeCalled();- expect(addRequestStub).toBeCalledTimes(7);- expect(count).toBe(7);-+ // The crawler should have processed 40 requests (its own limit), not 7 (40 - 33).+ expect(count).toBe(40);vitest.restoreAllMocks();});@@ -1441,12 +1396,6 @@ describe('BasicCrawler', () => {failedRequestHandler: async () => {},});- // @ts-expect-error Accessing private prop- crawler._loadHandledRequestCount = () => {- expect(crawler.sessionPool).toBeDefined();- expect(events.listenerCount(EventType.PERSIST_STATE)).toEqual(1);- };-await crawler.run();expect(events.listenerCount(EventType.PERSIST_STATE)).toEqual(0);// @ts-expect-error private symbol@@ -1605,7 +1554,7 @@ describe('BasicCrawler', () => {requestHandler: async () => {},});- crawler['handledRequestsCount'] = 2; // eslint-disable-line dot-notation+ crawler.stats.state.requestsFinished = 2;// Try to add 6 requests - should only add 3 due to limitconst requestsToAdd = [@@ -1638,7 +1587,7 @@ describe('BasicCrawler', () => {requestHandler: async () => {},});- crawler['handledRequestsCount'] = 1; // eslint-disable-line dot-notation+ crawler.stats.state.requestsFinished = 1;// First call - should add 2 requests (2 more slots to go)await crawler.addRequests(['http://example.com/1', 'http://example.com/2']);@@ -1687,7 +1636,7 @@ describe('BasicCrawler', () => {requestHandler: async () => {},});- crawler['handledRequestsCount'] = 0; // eslint-disable-line dot-notation+ crawler.stats.state.requestsFinished = 0;// Mock robots.txt checking to disallow some URLsvitest.spyOn(crawler as any, 'isAllowedBasedOnRobotsTxtFile').mockImplementation(async (url) => {@@ -1794,7 +1743,7 @@ describe('BasicCrawler', () => {return;}- crawler['handledRequestsCount'] = 2; // eslint-disable-line dot-notation+ crawler.stats.state.requestsFinished = 2;await context.enqueueLinks({ urls: requestsToAdd, label: 'not-undefined' });},