Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: make crawler terminal status message reliably delivered (#3733)
Lukáš Křivka7aed264f1cd9
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex 54fd7ef66..adbe9b62f 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -55,7 +55,7 @@ import {validators,} from '@crawlee/core';import type { Awaitable, BatchAddRequestsResult, Dictionary, SetStatusMessageOptions } from '@crawlee/types';-import { getObjectType, isAsyncIterable, isIterable, RobotsTxtFile, ROTATE_PROXY_ERRORS } from '@crawlee/utils';+import { getObjectType, isAsyncIterable, isIterable, RobotsTxtFile, ROTATE_PROXY_ERRORS, sleep } from '@crawlee/utils';import { stringify } from 'csv-stringify/sync';import { ensureDir, writeFile, writeJSON } from 'fs-extra';import ow, { ArgumentError } from 'ow';@@ -1082,13 +1082,17 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext}periodicLogger.stop();- // Don't await, we don't want to block the execution- void this.setStatusMessage(- `Finished! Total ${this.stats.state.requestsFinished + this.stats.state.requestsFailed} requests: ${- this.stats.state.requestsFinished- } succeeded, ${this.stats.state.requestsFailed} failed.`,- { isStatusMessageTerminal: true, level: 'INFO' },- );+ // Give the event loop a single tick to flush the HTTP+ // 1ms is enough because we already have a keep-alive connection to the API+ await Promise.race([+ this.setStatusMessage(+ `Finished! Total ${this.stats.state.requestsFinished + this.stats.state.requestsFailed} requests: ${+ this.stats.state.requestsFinished+ } succeeded, ${this.stats.state.requestsFailed} failed.`,+ { isStatusMessageTerminal: true, level: 'INFO' },+ ),+ sleep(1),+ ]);this.running = false;this.hasFinishedBefore = true;