Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: start status message logger after the crawl actually starts
Martin Adámek5d1df7aae00d
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex fb112f62d..29e3a29c0 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -661,7 +661,6 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextawait this._init();await this.stats.startCapturing();const periodicLogger = this.getPeriodicLogger();- await periodicLogger.log();const sigintHandler = async () => {this.log.warning('Pausing... Press CTRL+C again to force exit. To resume, do: CRAWLEE_PURGE_ON_START=0 npm start');@@ -677,6 +676,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContexttry {this.log.info('Starting the crawl');+ await periodicLogger.log();await this.autoscaledPool!.run();} finally {await this.teardown();