Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: graceful `BasicCrawler` tidy-up on `CriticalError` (#2817)
Jindřich Bär53331e82ee66
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex b562ff4f6..168282c8f 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -918,6 +918,8 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextthis.events.on(EventType.MIGRATING, boundPauseOnMigration);this.events.on(EventType.ABORTING, boundPauseOnMigration);+ let stats = {} as FinalStatistics;+try {await this.autoscaledPool!.run();} finally {@@ -927,50 +929,51 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextprocess.off('SIGINT', sigintHandler);this.events.off(EventType.MIGRATING, boundPauseOnMigration);this.events.off(EventType.ABORTING, boundPauseOnMigration);- }-- const finalStats = this.stats.calculate();- const stats = {- requestsFinished: this.stats.state.requestsFinished,- requestsFailed: this.stats.state.requestsFailed,- retryHistogram: this.stats.requestRetryHistogram,- ...finalStats,- };- this.log.info('Final request statistics:', stats);- if (this.stats.errorTracker.total !== 0) {- const prettify = ([count, info]: [number, string[]]) => `${count}x: ${info.at(-1)!.trim()} (${info[0]})`;+ const finalStats = this.stats.calculate();+ stats = {+ requestsFinished: this.stats.state.requestsFinished,+ requestsFailed: this.stats.state.requestsFailed,+ retryHistogram: this.stats.requestRetryHistogram,+ ...finalStats,+ };+ this.log.info('Final request statistics:', stats);++ if (this.stats.errorTracker.total !== 0) {+ const prettify = ([count, info]: [number, string[]]) =>+ `${count}x: ${info.at(-1)!.trim()} (${info[0]})`;++ this.log.info(`Error analysis:`, {+ totalErrors: this.stats.errorTracker.total,+ uniqueErrors: this.stats.errorTracker.getUniqueErrorCount(),+ mostCommonErrors: this.stats.errorTracker.getMostPopularErrors(3).map(prettify),+ });+ }- this.log.info(`Error analysis:`, {- totalErrors: this.stats.errorTracker.total,- uniqueErrors: this.stats.errorTracker.getUniqueErrorCount(),- mostCommonErrors: this.stats.errorTracker.getMostPopularErrors(3).map(prettify),- });- }+ const client = this.config.getStorageClient();- const client = this.config.getStorageClient();+ if (client.teardown) {+ let finished = false;+ setTimeout(() => {+ if (!finished) {+ this.log.info('Waiting for the storage to write its state to file system.');+ }+ }, 1000);+ await client.teardown();+ finished = true;+ }- if (client.teardown) {- let finished = false;- setTimeout(() => {- if (!finished) {- this.log.info('Waiting for the storage to write its state to file system.');- }- }, 1000);- await client.teardown();- finished = true;+ periodicLogger.stop();+ await this.setStatusMessage(+ `Finished! Total ${this.stats.state.requestsFinished + this.stats.state.requestsFailed} requests: ${+ this.stats.state.requestsFinished+ } succeeded, ${this.stats.state.requestsFailed} failed.`,+ { isStatusMessageTerminal: true, level: 'INFO' },+ );+ this.running = false;+ this.hasFinishedBefore = true;}- periodicLogger.stop();- await this.setStatusMessage(- `Finished! Total ${this.stats.state.requestsFinished + this.stats.state.requestsFailed} requests: ${- this.stats.state.requestsFinished- } succeeded, ${this.stats.state.requestsFailed} failed.`,- { isStatusMessageTerminal: true, level: 'INFO' },- );- this.running = false;- this.hasFinishedBefore = true;-return stats;}