Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Changed arg order, moved size into object

Jan Novotny484248a657cb
diff --git a/src/key_value_store.js b/src/key_value_store.js
index fbc985180..d76695e4b 100644
--- a/src/key_value_store.js
+++ b/src/key_value_store.js
@@ -294,7 +294,7 @@ export class KeyValueStore {
/**
* Iterates over key value store keys, yielding each in turn to an `iteratee` function.
- * Each invocation of `iteratee` is called with three arguments: `(item, size, index)`.
+ * Each invocation of `iteratee` is called with three arguments: `(item, index, info)`.
*
* If the `iteratee` function returns a Promise then it is awaited before the next call.
* If it throws an error, the iteration is aborted and the `forEachKey` function throws the error.
@@ -302,8 +302,8 @@ export class KeyValueStore {
* **Example usage**
* ```javascript
* const keyValueStore = await Apify.openKeyValueStore();
- * keyValueStore.forEachKey(async (key, size, index) => {
- * console.log(`Key at ${index}: ${key} has size ${size}`);
+ * keyValueStore.forEachKey(async (key, index, info) => {
+ * console.log(`Key at ${index}: ${key} has size ${info.size}`);
* });
* ```
*
@@ -322,7 +322,7 @@ export class KeyValueStore {
const response = await keyValueStores.listKeys({ storeId: this.storeId, exclusiveStartKey });
const { nextExclusiveStartKey, isTruncated, items } = response;
for (const item of items) {
- await iteratee(item.key, item.size, index++);
+ await iteratee(item.key, index++, { size: item.size });
}
return isTruncated
? this.forEachKey(iteratee, { exclusiveStartKey: nextExclusiveStartKey }, index)
@@ -411,10 +411,17 @@ export class KeyValueStoreLocal {
const files = await readdirPromised(this.localStoragePath);
let keys = [];
for (const file of files) {
- keys.push({
- key: path.parse(file).name,
- size: (await statPromised(this._getPath(file))).size,
- });
+ try {
+ const { size } = await statPromised(this._getPath(file));
+ keys.push({
+ key: path.parse(file).name,
+ info: { size },
+ });
+ } catch (e) {
+ if (e.code !== 'ENOENT') {
+ throw e;
+ }
+ }
}
keys = keys.sort((a, b) => {
@@ -428,7 +435,7 @@ export class KeyValueStoreLocal {
if (keyPos !== -1) keys = keys.slice(keyPos + 1);
}
for (const item of keys) {
- await iteratee(item.key, item.size, index++);
+ await iteratee(item.key, index++, item.info);
}
}
diff --git a/test/key_value_store.js b/test/key_value_store.js
index 3e9a979a3..47f0614be 100644
--- a/test/key_value_store.js
+++ b/test/key_value_store.js
@@ -414,14 +414,14 @@ describe('KeyValueStore', () => {
});
const results = [];
- await store.forEachKey(async (key, size, index) => {
- results.push([key, size, index]);
+ await store.forEachKey(async (key, index, info) => {
+ results.push([key, index, info]);
}, { exclusiveStartKey: 'key0' });
expect(results).to.have.lengthOf(5);
results.forEach((r, i) => {
- expect(r[2]).to.be.eql(i);
- expect(r[1]).to.be.eql(i + 1);
+ expect(r[2]).to.be.eql({ size: i + 1 });
+ expect(r[1]).to.be.eql(i);
expect(r[0]).to.be.eql(`key${i + 1}`);
});
@@ -437,14 +437,14 @@ describe('KeyValueStore', () => {
}
const results = [];
- await store.forEachKey((key, size, index) => {
- results.push([key, size, index]);
+ await store.forEachKey((key, index, info) => {
+ results.push([key, index, info]);
}, { exclusiveStartKey: 'key3' });
expect(results).to.have.lengthOf(6);
results.forEach((r, i) => {
- expect(r[2]).to.be.eql(i);
- expect(r[1]).to.be.eql(2);
+ expect(r[2]).to.be.eql({ size: 2 });
+ expect(r[1]).to.be.eql(i);
expect(r[0]).to.be.eql(`key${i + 4}`);
});