Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: do not drop statistics on migration/resurrection/resume (#2462)
Jan Buchar8ce7dd4ae6a3
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex 1b3ff9dda..6778e7e31 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -470,7 +470,8 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext*/readonly router: RouterHandler<Context> = Router.create<Context>();- running?: boolean;+ running = false;+ hasFinishedBefore = false;protected log: Log;protected requestHandler!: RequestHandler<Context>;@@ -829,19 +830,22 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextconst purgeRequestQueue = options?.purgeRequestQueue ?? true;- // When executing the run method for the second time explicitly (the first time `this.running` is `undefined`),- // we need to purge the default RQ to allow processing the same requests again - this is important so users can- // pass in failed requests back to the `crawler.run()`, otherwise they would be considered as handled and- // ignored - as a failed requests is still handled.- if (this.running === false && this.requestQueue?.name === 'default' && purgeRequestQueue) {- await this.requestQueue.drop();- this.requestQueue = await this._getRequestQueue();+ if (this.hasFinishedBefore) {+ // When executing the run method for the second time explicitly,+ // we need to purge the default RQ to allow processing the same requests again - this is important so users can+ // pass in failed requests back to the `crawler.run()`, otherwise they would be considered as handled and+ // ignored - as a failed requests is still handled.+ if (this.requestQueue?.name === 'default' && purgeRequestQueue) {+ await this.requestQueue.drop();+ this.requestQueue = await this._getRequestQueue();+ }++ this.stats.reset();+ await this.stats.resetStore();+ await this.sessionPool?.resetStore();}this.running = true;- this.stats.reset();- await this.stats.resetStore();- await this.sessionPool?.resetStore();await purgeDefaultStorages({ onlyPurgeOnce: true });@@ -913,6 +917,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext// eslint-disable-next-line max-lenawait this.setStatusMessage(`Finished! Total ${this.stats.state.requestsFinished + this.stats.state.requestsFailed} requests: ${this.stats.state.requestsFinished} succeeded, ${this.stats.state.requestsFailed} failed.`, { isStatusMessageTerminal: true, level: 'INFO' });this.running = false;+ this.hasFinishedBefore = true;return stats;}diff --git a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.tsindex 607fecfd1..15a9c1ba9 100644--- a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts+++ b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts@@ -1,6 +1,13 @@import { addTimeoutToPromise } from '@apify/timeout';import { extractUrlsFromPage } from '@crawlee/browser';-import type { RestrictedCrawlingContext, StatisticState, StatisticPersistedState, GetUserDataFromRequest, RouterRoutes } from '@crawlee/core';+import type {+ RestrictedCrawlingContext,+ StatisticState,+ StatisticsOptions,+ StatisticPersistedState,+ GetUserDataFromRequest,+ RouterRoutes,+} from '@crawlee/core';import { Configuration, RequestHandlerResult, Router, Statistics, withCheckedStorageAccess } from '@crawlee/core';import type { Awaitable, Dictionary } from '@crawlee/types';import { extractUrlsFromCheerio } from '@crawlee/utils';@@ -28,6 +35,11 @@ interface AdaptivePlaywrightCrawlerPersistedStatisticState extends StatisticPersclass AdaptivePlaywrightCrawlerStatistics extends Statistics {override state: AdaptivePlaywrightCrawlerStatisticState = null as any; // this needs to be assigned for a valid override, but the initialization is done by a reset() call from the parent constructor+ constructor(options: StatisticsOptions = {}) {+ super(options);+ this.reset();+ }+override reset(): void {super.reset();this.state.httpOnlyRequestHandlerRuns = 0;