Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: do not drop statistics on migration/resurrection/resume (#2462)

Jan Buchar8ce7dd4ae6a3
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
index 1b3ff9dda..6778e7e31 100644
--- a/packages/basic-crawler/src/internals/basic-crawler.ts
+++ b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -470,7 +470,8 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
*/
readonly router: RouterHandler<Context> = Router.create<Context>();
- running?: boolean;
+ running = false;
+ hasFinishedBefore = false;
protected log: Log;
protected requestHandler!: RequestHandler<Context>;
@@ -829,19 +830,22 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
const purgeRequestQueue = options?.purgeRequestQueue ?? true;
- // When executing the run method for the second time explicitly (the first time `this.running` is `undefined`),
- // we need to purge the default RQ to allow processing the same requests again - this is important so users can
- // pass in failed requests back to the `crawler.run()`, otherwise they would be considered as handled and
- // ignored - as a failed requests is still handled.
- if (this.running === false && this.requestQueue?.name === 'default' && purgeRequestQueue) {
- await this.requestQueue.drop();
- this.requestQueue = await this._getRequestQueue();
+ if (this.hasFinishedBefore) {
+ // When executing the run method for the second time explicitly,
+ // we need to purge the default RQ to allow processing the same requests again - this is important so users can
+ // pass in failed requests back to the `crawler.run()`, otherwise they would be considered as handled and
+ // ignored - as a failed requests is still handled.
+ if (this.requestQueue?.name === 'default' && purgeRequestQueue) {
+ await this.requestQueue.drop();
+ this.requestQueue = await this._getRequestQueue();
+ }
+
+ this.stats.reset();
+ await this.stats.resetStore();
+ await this.sessionPool?.resetStore();
}
this.running = true;
- this.stats.reset();
- await this.stats.resetStore();
- await this.sessionPool?.resetStore();
await purgeDefaultStorages({ onlyPurgeOnce: true });
@@ -913,6 +917,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
// eslint-disable-next-line max-len
await this.setStatusMessage(`Finished! Total ${this.stats.state.requestsFinished + this.stats.state.requestsFailed} requests: ${this.stats.state.requestsFinished} succeeded, ${this.stats.state.requestsFailed} failed.`, { isStatusMessageTerminal: true, level: 'INFO' });
this.running = false;
+ this.hasFinishedBefore = true;
return stats;
}
diff --git a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts
index 607fecfd1..15a9c1ba9 100644
--- a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts
+++ b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts
@@ -1,6 +1,13 @@
import { addTimeoutToPromise } from '@apify/timeout';
import { extractUrlsFromPage } from '@crawlee/browser';
-import type { RestrictedCrawlingContext, StatisticState, StatisticPersistedState, GetUserDataFromRequest, RouterRoutes } from '@crawlee/core';
+import type {
+ RestrictedCrawlingContext,
+ StatisticState,
+ StatisticsOptions,
+ StatisticPersistedState,
+ GetUserDataFromRequest,
+ RouterRoutes,
+} from '@crawlee/core';
import { Configuration, RequestHandlerResult, Router, Statistics, withCheckedStorageAccess } from '@crawlee/core';
import type { Awaitable, Dictionary } from '@crawlee/types';
import { extractUrlsFromCheerio } from '@crawlee/utils';
@@ -28,6 +35,11 @@ interface AdaptivePlaywrightCrawlerPersistedStatisticState extends StatisticPers
class AdaptivePlaywrightCrawlerStatistics extends Statistics {
override state: AdaptivePlaywrightCrawlerStatisticState = null as any; // this needs to be assigned for a valid override, but the initialization is done by a reset() call from the parent constructor
+ constructor(options: StatisticsOptions = {}) {
+ super(options);
+ this.reset();
+ }
+
override reset(): void {
super.reset();
this.state.httpOnlyRequestHandlerRuns = 0;