Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

refactor: set storage object on KV store open (#2837)

Daniil Poletaev0bae010ed970
diff --git a/packages/core/src/storages/key_value_store.ts b/packages/core/src/storages/key_value_store.ts
index 59dfd3bdb..466aae925 100644
--- a/packages/core/src/storages/key_value_store.ts
+++ b/packages/core/src/storages/key_value_store.ts
@@ -106,6 +106,7 @@ export const maybeStringify = <T>(value: T, options: { contentType?: string }) =
export class KeyValueStore {
readonly id: string;
readonly name?: string;
+ readonly storageObject?: Record<string, unknown>;
private readonly client: KeyValueStoreClient;
private persistStateEventStarted = false;
@@ -121,6 +122,7 @@ export class KeyValueStore {
) {
this.id = options.id;
this.name = options.name;
+ this.storageObject = options.storageObject;
this.client = options.client.keyValueStore(this.id);
}
@@ -708,6 +710,7 @@ export interface KeyValueStoreOptions {
id: string;
name?: string;
client: StorageClient;
+ storageObject?: Record<string, unknown>;
}
export interface RecordOptions {
diff --git a/packages/core/src/storages/storage_manager.ts b/packages/core/src/storages/storage_manager.ts
index 86eacb7dc..78c51cb80 100644
--- a/packages/core/src/storages/storage_manager.ts
+++ b/packages/core/src/storages/storage_manager.ts
@@ -84,6 +84,7 @@ export class StorageManager<T extends IStorage = IStorage> {
storage = new this.StorageConstructor({
id: storageObject.id,
name: storageObject.name,
+ storageObject,
client,
});
diff --git a/test/e2e/kv-open-return-storage-object/actor/.actor/actor.json b/test/e2e/kv-open-return-storage-object/actor/.actor/actor.json
new file mode 100644
index 000000000..7faf158a7
--- /dev/null
+++ b/test/e2e/kv-open-return-storage-object/actor/.actor/actor.json
@@ -0,0 +1,7 @@
+{
+ "actorSpecification": 1,
+ "name": "test-kv-open-return-storage-object",
+ "version": "0.0",
+ "buildTag": "latest",
+ "env": null
+}
diff --git a/test/e2e/kv-open-return-storage-object/actor/.gitignore b/test/e2e/kv-open-return-storage-object/actor/.gitignore
new file mode 100644
index 000000000..ced7cbfc5
--- /dev/null
+++ b/test/e2e/kv-open-return-storage-object/actor/.gitignore
@@ -0,0 +1,7 @@
+.idea
+.DS_Store
+node_modules
+package-lock.json
+apify_storage
+crawlee_storage
+storage
diff --git a/test/e2e/kv-open-return-storage-object/actor/Dockerfile b/test/e2e/kv-open-return-storage-object/actor/Dockerfile
new file mode 100644
index 000000000..36afd80b9
--- /dev/null
+++ b/test/e2e/kv-open-return-storage-object/actor/Dockerfile
@@ -0,0 +1,16 @@
+FROM apify/actor-node:20-beta
+
+COPY packages ./packages
+COPY package*.json ./
+
+RUN npm --quiet set progress=false \
+ && npm install --only=prod --no-optional --no-audit \
+ && npm update --no-audit \
+ && echo "Installed NPM packages:" \
+ && (npm list --only=prod --no-optional --all || true) \
+ && echo "Node.js version:" \
+ && node --version \
+ && echo "NPM version:" \
+ && npm --version
+
+COPY . ./
diff --git a/test/e2e/kv-open-return-storage-object/actor/main.js b/test/e2e/kv-open-return-storage-object/actor/main.js
new file mode 100644
index 000000000..9aeb49e47
--- /dev/null
+++ b/test/e2e/kv-open-return-storage-object/actor/main.js
@@ -0,0 +1,14 @@
+import { Actor, KeyValueStore } from 'apify';
+
+const mainOptions = {
+ exit: Actor.isAtHome(),
+ storage:
+ process.env.STORAGE_IMPLEMENTATION === 'LOCAL'
+ ? new (await import('@apify/storage-local')).ApifyStorageLocal()
+ : undefined,
+};
+
+await Actor.main(async () => {
+ const kv = await KeyValueStore.open();
+ kv.setValue('storageObject', { storeObject: kv.storageObject });
+}, mainOptions);
diff --git a/test/e2e/kv-open-return-storage-object/actor/package.json b/test/e2e/kv-open-return-storage-object/actor/package.json
new file mode 100644
index 000000000..f40826ba0
--- /dev/null
+++ b/test/e2e/kv-open-return-storage-object/actor/package.json
@@ -0,0 +1,25 @@
+{
+ "name": "test-kv-open-return-storage-object",
+ "version": "0.0.1",
+ "description": "Key-Value Store - Return storage object on open",
+ "dependencies": {
+ "apify": "next",
+ "@apify/storage-local": "^2.1.3",
+ "@crawlee/basic": "file:./packages/basic-crawler",
+ "@crawlee/core": "file:./packages/core",
+ "@crawlee/memory-storage": "file:./packages/memory-storage",
+ "@crawlee/types": "file:./packages/types",
+ "@crawlee/utils": "file:./packages/utils"
+ },
+ "overrides": {
+ "apify": {
+ "@crawlee/core": "file:./packages/core",
+ "@crawlee/utils": "file:./packages/utils"
+ }
+ },
+ "scripts": {
+ "start": "node main.js"
+ },
+ "type": "module",
+ "license": "ISC"
+}
diff --git a/test/e2e/kv-open-return-storage-object/test.mjs b/test/e2e/kv-open-return-storage-object/test.mjs
new file mode 100644
index 000000000..83de7b80d
--- /dev/null
+++ b/test/e2e/kv-open-return-storage-object/test.mjs
@@ -0,0 +1,25 @@
+import { initialize, expect, getActorTestDir, runActor } from '../tools.mjs';
+
+/* This test verifies that the storageObject is correctly returned when the KeyValueStore is opened.
+ * The storageObject is the result of the KeyValueStoreClient.get() method,
+ * containing properties such as name, id, and other custom attributes.
+ */
+
+const testActorDirname = getActorTestDir(import.meta.url);
+await initialize(testActorDirname);
+
+const { defaultKeyValueStoreItems: items } = await runActor(testActorDirname);
+
+await expect(items !== undefined, 'Key value store exists');
+
+const item = items.find((kvItem) => kvItem.name === 'storageObject');
+
+const parsed = JSON.parse(item.raw.toString());
+
+await expect(
+ typeof parsed === 'object' && parsed !== null,
+ 'Key-value contains key "storeObject" and it\'s value is a non-nullable object',
+);
+await expect(parsed.id !== null, 'storeObject contains id');
+await expect(parsed.name !== null, 'storeObject contains name');
+await expect(parsed.userId !== null, 'storeObject contains userId');