Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

feat: support `KVS.listKeys()` `prefix` and `collection` parameters (#3001)

Jindřich Bär5c4726df96e3
diff --git a/packages/core/src/storages/key_value_store.ts b/packages/core/src/storages/key_value_store.ts
index c3247a944..161d4fa29 100644
--- a/packages/core/src/storages/key_value_store.ts
+++ b/packages/core/src/storages/key_value_store.ts
@@ -460,22 +460,24 @@ export class KeyValueStore {
options: KeyValueStoreIteratorOptions = {},
index = 0,
): Promise<void> {
- const { exclusiveStartKey } = options;
+ const { exclusiveStartKey, prefix, collection } = options;
ow(iteratee, ow.function);
ow(
options,
ow.object.exactShape({
exclusiveStartKey: ow.optional.string,
+ prefix: ow.optional.string,
+ collection: ow.optional.string,
}),
);
- const response = await this.client.listKeys({ exclusiveStartKey });
+ const response = await this.client.listKeys({ exclusiveStartKey, prefix, collection });
const { nextExclusiveStartKey, isTruncated, items } = response;
for (const item of items) {
await iteratee(item.key, index++, { size: item.size });
}
return isTruncated
- ? this._forEachKey(iteratee, { exclusiveStartKey: nextExclusiveStartKey }, index)
+ ? this._forEachKey(iteratee, { exclusiveStartKey: nextExclusiveStartKey, prefix, collection }, index)
: undefined; // [].forEach() returns undefined.
}
@@ -758,4 +760,12 @@ export interface KeyValueStoreIteratorOptions {
* All keys up to this one (including) are skipped from the result.
*/
exclusiveStartKey?: string;
+ /**
+ * If set, only keys that start with this prefix are returned.
+ */
+ prefix?: string;
+ /**
+ * Collection name to use for listing keys.
+ */
+ collection?: string;
}
diff --git a/packages/memory-storage/src/resource-clients/key-value-store.ts b/packages/memory-storage/src/resource-clients/key-value-store.ts
index 986961de0..1a3af0432 100644
--- a/packages/memory-storage/src/resource-clients/key-value-store.ts
+++ b/packages/memory-storage/src/resource-clients/key-value-store.ts
@@ -119,10 +119,16 @@ export class KeyValueStoreClient extends BaseClient {
}
async listKeys(options: storage.KeyValueStoreClientListOptions = {}): Promise<storage.KeyValueStoreClientListData> {
- const { limit = DEFAULT_API_PARAM_LIMIT, exclusiveStartKey } = s
+ const {
+ limit = DEFAULT_API_PARAM_LIMIT,
+ exclusiveStartKey,
+ prefix,
+ } = s
.object({
limit: s.number.greaterThan(0).optional,
exclusiveStartKey: s.string.optional,
+ collection: s.string.optional, // This is ignored, but kept for validation consistency with API client.
+ prefix: s.string.optional,
})
.parse(options);
@@ -151,23 +157,25 @@ export class KeyValueStoreClient extends BaseClient {
return a.key.localeCompare(b.key);
});
- let truncatedItems = items;
+ const filteredItems = items.filter((item) => !prefix || item.key.startsWith(prefix));
+
+ let truncatedItems = filteredItems;
if (exclusiveStartKey) {
- const keyPos = items.findIndex((item) => item.key === exclusiveStartKey);
- if (keyPos !== -1) truncatedItems = items.slice(keyPos + 1);
+ const keyPos = filteredItems.findIndex((item) => item.key === exclusiveStartKey);
+ if (keyPos !== -1) truncatedItems = filteredItems.slice(keyPos + 1);
}
const limitedItems = truncatedItems.slice(0, limit);
- const lastItemInStore = items[items.length - 1];
- const lastSelectedItem = limitedItems[limitedItems.length - 1];
+ const lastItemInStore = filteredItems.at(-1);
+ const lastSelectedItem = limitedItems.at(-1);
const isLastSelectedItemAbsolutelyLast = lastItemInStore === lastSelectedItem;
- const nextExclusiveStartKey = isLastSelectedItemAbsolutelyLast ? undefined : lastSelectedItem.key;
+ const nextExclusiveStartKey = isLastSelectedItemAbsolutelyLast ? undefined : lastSelectedItem?.key;
existingStoreById.updateTimestamps(false);
return {
- count: items.length,
+ count: limitedItems.length,
limit,
exclusiveStartKey,
isTruncated: !isLastSelectedItemAbsolutelyLast,
diff --git a/packages/types/src/storages.ts b/packages/types/src/storages.ts
index 425d55ac3..ba9247626 100644
--- a/packages/types/src/storages.ts
+++ b/packages/types/src/storages.ts
@@ -136,6 +136,8 @@ export interface KeyValueStoreClientUpdateOptions {
export interface KeyValueStoreClientListOptions {
limit?: number;
exclusiveStartKey?: string;
+ collection?: string;
+ prefix?: string;
}
export interface KeyValueStoreItemData {
diff --git a/test/core/storages/key_value_store.test.ts b/test/core/storages/key_value_store.test.ts
index 0d8bfb97c..2476b114a 100644
--- a/test/core/storages/key_value_store.test.ts
+++ b/test/core/storages/key_value_store.test.ts
@@ -509,6 +509,39 @@ describe('KeyValueStore', () => {
});
describe('forEachKey', () => {
+ test('should work with prefixes', async () => {
+ const store = await KeyValueStore.open();
+
+ for (const [key, value] of Object.entries({
+ 'img-key1': 'PAYLOAD',
+ 'img-key2': 'PAYLOAD',
+ 'txt-key1': 'PAYLOAD',
+ 'txt-key2': 'PAYLOAD',
+ })) {
+ await store.setValue(key, value);
+ }
+
+ const imgKeys: string[] = [];
+ const txtKeys: string[] = [];
+
+ await store.forEachKey(
+ (key) => {
+ imgKeys.push(key);
+ },
+ { prefix: 'img-' },
+ );
+
+ await store.forEachKey(
+ (key) => {
+ txtKeys.push(key);
+ },
+ { prefix: 'txt-' },
+ );
+
+ expect(imgKeys).toEqual(['img-key1', 'img-key2']);
+ expect(txtKeys).toEqual(['txt-key1', 'txt-key2']);
+ });
+
test('should work remotely', async () => {
const store = new KeyValueStore({
id: 'my-store-id-1',
@@ -555,13 +588,13 @@ describe('KeyValueStore', () => {
async (key, index, info) => {
results.push([key, index, info]);
},
- { exclusiveStartKey: 'key0' },
+ { exclusiveStartKey: 'key0', prefix: 'img/' },
);
expect(mockListKeys).toBeCalledTimes(3);
- expect(mockListKeys).toHaveBeenNthCalledWith(1, { exclusiveStartKey: 'key0' });
- expect(mockListKeys).toHaveBeenNthCalledWith(2, { exclusiveStartKey: 'key2' });
- expect(mockListKeys).toHaveBeenNthCalledWith(3, { exclusiveStartKey: 'key4' });
+ expect(mockListKeys).toHaveBeenNthCalledWith(1, { exclusiveStartKey: 'key0', prefix: 'img/' });
+ expect(mockListKeys).toHaveBeenNthCalledWith(2, { exclusiveStartKey: 'key2', prefix: 'img/' });
+ expect(mockListKeys).toHaveBeenNthCalledWith(3, { exclusiveStartKey: 'key4', prefix: 'img/' });
expect(results).toHaveLength(5);
results.forEach((r, i) => {