Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat: add `keyValueStore` option to `Statistics` class (#1345)
Martin Adámek898cc43fc66d
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 03147b0eb..64af307c6 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -1,7 +1,8 @@-2.3.1 / TBA+2.3.1 / 2022/05/02====================- fix: `utils.apifyClient` early instantiation (#1330)-- feat: `playwright.injectJQuery()` (#1337)+- feat: `utils.playwright.injectJQuery()` (#1337)+- feat: add `keyValueStore` option to `Statistics` class (#1345)2.3.0 / 2022/04/07====================diff --git a/src/crawlers/statistics.js b/src/crawlers/statistics.jsindex 66aabe50f..9a2c41a6a 100644--- a/src/crawlers/statistics.js+++ b/src/crawlers/statistics.js@@ -55,17 +55,19 @@ class Statistics {ow(options, ow.object.exactShape({logIntervalSecs: ow.optional.number,logMessage: ow.optional.string,+ keyValueStore: ow.optional.object,}));const {logIntervalSecs = 60,logMessage = 'Statistics',+ keyValueStore,} = options;this.log = defaultLog.child({ prefix: 'Statistics' });this.logIntervalMillis = logIntervalSecs * 1000;this.logMessage = logMessage;- this.keyValueStore = null;+ this.keyValueStore = keyValueStore;// assign an id while incrementing so it can be saved/restored from KVthis.id = Statistics.id++;this.persistStateKey = `SDK_CRAWLER_STATISTICS_${this.id}`;@@ -184,7 +186,7 @@ class Statistics {* displaying the current state in predefined intervals*/async startCapturing() {- this.keyValueStore = await openKeyValueStore();+ this.keyValueStore ??= await openKeyValueStore();await this._maybeLoadStatistics();@@ -334,6 +336,7 @@ export default Statistics;* @typedef StatisticsOptions* @property {number} [logIntervalSecs]* @property {string} [logMessage]+ * @property {KeyValueStore} [keyValueStore]*//**