Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Fixed forEachKey incosistency
Jan Novotny5f774e36660c
diff --git a/src/key_value_store.js b/src/key_value_store.jsindex cee00b3ea..fbc985180 100644--- a/src/key_value_store.js+++ b/src/key_value_store.js@@ -21,6 +21,7 @@ const readFilePromised = Promise.promisify(fs.readFile);const readdirPromised = Promise.promisify(fs.readdir);const writeFilePromised = Promise.promisify(fs.writeFile);const unlinkPromised = Promise.promisify(fs.unlink);+const statPromised = Promise.promisify(fs.stat);const emptyDirPromised = Promise.promisify(fsExtra.emptyDir);const { keyValueStores } = apifyClient;@@ -293,7 +294,7 @@ export class KeyValueStore {/*** Iterates over key value store keys, yielding each in turn to an `iteratee` function.- * Each invocation of `iteratee` is called with two arguments: `(item, index)`.+ * Each invocation of `iteratee` is called with three arguments: `(item, size, index)`.** If the `iteratee` function returns a Promise then it is awaited before the next call.* If it throws an error, the iteration is aborted and the `forEachKey` function throws the error.@@ -301,8 +302,8 @@ export class KeyValueStore {* **Example usage*** ```javascript* const keyValueStore = await Apify.openKeyValueStore();- * keyValueStore.forEachKey(async (key, index) => {- * console.log(`Key at ${index}: ${key}`);+ * keyValueStore.forEachKey(async (key, size, index) => {+ * console.log(`Key at ${index}: ${key} has size ${size}`);* });* ```*@@ -320,8 +321,8 @@ export class KeyValueStore {const response = await keyValueStores.listKeys({ storeId: this.storeId, exclusiveStartKey });const { nextExclusiveStartKey, isTruncated, items } = response;- for (const key of items) {- await iteratee(key, index++);+ for (const item of items) {+ await iteratee(item.key, item.size, index++);}return isTruncated? this.forEachKey(iteratee, { exclusiveStartKey: nextExclusiveStartKey }, index)@@ -408,13 +409,26 @@ export class KeyValueStoreLocal {checkParamOrThrow(index, 'index', 'Number');const files = await readdirPromised(this.localStoragePath);- let keys = files.map(file => path.parse(file).name).sort(); // Array is sorted to emulate API.+ let keys = [];+ for (const file of files) {+ keys.push({+ key: path.parse(file).name,+ size: (await statPromised(this._getPath(file))).size,+ });+ }++ keys = keys.sort((a, b) => {+ if (a.key < b.key) return -1;+ if (a.key > b.key) return 1;+ return 0;+ }); // Array is sorted to emulate API.+if (exclusiveStartKey) {- const keyPos = keys.indexOf(exclusiveStartKey);+ const keyPos = keys.findIndex(item => item.key === exclusiveStartKey);if (keyPos !== -1) keys = keys.slice(keyPos + 1);}- for (const key of keys) {- await iteratee(key, index++);+ for (const item of keys) {+ await iteratee(item.key, item.size, index++);}}diff --git a/test/key_value_store.js b/test/key_value_store.jsindex 71c8e8640..3e9a979a3 100644--- a/test/key_value_store.js+++ b/test/key_value_store.js@@ -382,7 +382,10 @@ describe('KeyValueStore', () => {.resolves({isTruncated: true,nextExclusiveStartKey: 'key2',- items: ['key1', 'key2'],+ items: [+ { key: 'key1', size: 1 },+ { key: 'key2', size: 2 },+ ],});mock.expects('listKeys').once()@@ -393,7 +396,10 @@ describe('KeyValueStore', () => {.resolves({isTruncated: true,nextExclusiveStartKey: 'key4',- items: ['key3', 'key4'],+ items: [+ { key: 'key3', size: 3 },+ { key: 'key4', size: 4 },+ ],});mock.expects('listKeys').once()@@ -404,17 +410,18 @@ describe('KeyValueStore', () => {.resolves({isTruncated: false,nextExclusiveStartKey: null,- items: ['key5'],+ items: [{ key: 'key5', size: 5 }],});const results = [];- await store.forEachKey(async (key, index) => {- results.push([key, index]);+ await store.forEachKey(async (key, size, index) => {+ results.push([key, size, index]);}, { exclusiveStartKey: 'key0' });expect(results).to.have.lengthOf(5);results.forEach((r, i) => {- expect(r[1]).to.be.eql(i);+ expect(r[2]).to.be.eql(i);+ expect(r[1]).to.be.eql(i + 1);expect(r[0]).to.be.eql(`key${i + 1}`);});@@ -430,13 +437,14 @@ describe('KeyValueStore', () => {}const results = [];- await store.forEachKey((key, index) => {- results.push([key, index]);+ await store.forEachKey((key, size, index) => {+ results.push([key, size, index]);}, { exclusiveStartKey: 'key3' });expect(results).to.have.lengthOf(6);results.forEach((r, i) => {- expect(r[1]).to.be.eql(i);+ expect(r[2]).to.be.eql(i);+ expect(r[1]).to.be.eql(2);expect(r[0]).to.be.eql(`key${i + 4}`);});