Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

feat: add `keyValueStore` option to `Statistics` class (#1345)

Martin Adámek898cc43fc66d
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 03147b0eb..64af307c6 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,7 +1,8 @@
-2.3.1 / TBA
+2.3.1 / 2022/05/02
====================
- fix: `utils.apifyClient` early instantiation (#1330)
-- feat: `playwright.injectJQuery()` (#1337)
+- feat: `utils.playwright.injectJQuery()` (#1337)
+- feat: add `keyValueStore` option to `Statistics` class (#1345)
2.3.0 / 2022/04/07
====================
diff --git a/src/crawlers/statistics.js b/src/crawlers/statistics.js
index 66aabe50f..9a2c41a6a 100644
--- a/src/crawlers/statistics.js
+++ b/src/crawlers/statistics.js
@@ -55,17 +55,19 @@ class Statistics {
ow(options, ow.object.exactShape({
logIntervalSecs: ow.optional.number,
logMessage: ow.optional.string,
+ keyValueStore: ow.optional.object,
}));
const {
logIntervalSecs = 60,
logMessage = 'Statistics',
+ keyValueStore,
} = options;
this.log = defaultLog.child({ prefix: 'Statistics' });
this.logIntervalMillis = logIntervalSecs * 1000;
this.logMessage = logMessage;
- this.keyValueStore = null;
+ this.keyValueStore = keyValueStore;
// assign an id while incrementing so it can be saved/restored from KV
this.id = Statistics.id++;
this.persistStateKey = `SDK_CRAWLER_STATISTICS_${this.id}`;
@@ -184,7 +186,7 @@ class Statistics {
* displaying the current state in predefined intervals
*/
async startCapturing() {
- this.keyValueStore = await openKeyValueStore();
+ this.keyValueStore ??= await openKeyValueStore();
await this._maybeLoadStatistics();
@@ -334,6 +336,7 @@ export default Statistics;
* @typedef StatisticsOptions
* @property {number} [logIntervalSecs]
* @property {string} [logMessage]
+ * @property {KeyValueStore} [keyValueStore]
*/
/**