Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
refactor: set storage object on KV store open (#2837)
Daniil Poletaev0bae010ed970
diff --git a/packages/core/src/storages/key_value_store.ts b/packages/core/src/storages/key_value_store.tsindex 59dfd3bdb..466aae925 100644--- a/packages/core/src/storages/key_value_store.ts+++ b/packages/core/src/storages/key_value_store.ts@@ -106,6 +106,7 @@ export const maybeStringify = <T>(value: T, options: { contentType?: string }) =export class KeyValueStore {readonly id: string;readonly name?: string;+ readonly storageObject?: Record<string, unknown>;private readonly client: KeyValueStoreClient;private persistStateEventStarted = false;@@ -121,6 +122,7 @@ export class KeyValueStore {) {this.id = options.id;this.name = options.name;+ this.storageObject = options.storageObject;this.client = options.client.keyValueStore(this.id);}@@ -708,6 +710,7 @@ export interface KeyValueStoreOptions {id: string;name?: string;client: StorageClient;+ storageObject?: Record<string, unknown>;}export interface RecordOptions {diff --git a/packages/core/src/storages/storage_manager.ts b/packages/core/src/storages/storage_manager.tsindex 86eacb7dc..78c51cb80 100644--- a/packages/core/src/storages/storage_manager.ts+++ b/packages/core/src/storages/storage_manager.ts@@ -84,6 +84,7 @@ export class StorageManager<T extends IStorage = IStorage> {storage = new this.StorageConstructor({id: storageObject.id,name: storageObject.name,+ storageObject,client,});diff --git a/test/e2e/kv-open-return-storage-object/actor/.actor/actor.json b/test/e2e/kv-open-return-storage-object/actor/.actor/actor.jsonnew file mode 100644index 000000000..7faf158a7--- /dev/null+++ b/test/e2e/kv-open-return-storage-object/actor/.actor/actor.json@@ -0,0 +1,7 @@+{+ "actorSpecification": 1,+ "name": "test-kv-open-return-storage-object",+ "version": "0.0",+ "buildTag": "latest",+ "env": null+}diff --git a/test/e2e/kv-open-return-storage-object/actor/.gitignore b/test/e2e/kv-open-return-storage-object/actor/.gitignorenew file mode 100644index 000000000..ced7cbfc5--- /dev/null+++ b/test/e2e/kv-open-return-storage-object/actor/.gitignore@@ -0,0 +1,7 @@+.idea+.DS_Store+node_modules+package-lock.json+apify_storage+crawlee_storage+storagediff --git a/test/e2e/kv-open-return-storage-object/actor/Dockerfile b/test/e2e/kv-open-return-storage-object/actor/Dockerfilenew file mode 100644index 000000000..36afd80b9--- /dev/null+++ b/test/e2e/kv-open-return-storage-object/actor/Dockerfile@@ -0,0 +1,16 @@+FROM apify/actor-node:20-beta++COPY packages ./packages+COPY package*.json ./++RUN npm --quiet set progress=false \+ && npm install --only=prod --no-optional --no-audit \+ && npm update --no-audit \+ && echo "Installed NPM packages:" \+ && (npm list --only=prod --no-optional --all || true) \+ && echo "Node.js version:" \+ && node --version \+ && echo "NPM version:" \+ && npm --version++COPY . ./diff --git a/test/e2e/kv-open-return-storage-object/actor/main.js b/test/e2e/kv-open-return-storage-object/actor/main.jsnew file mode 100644index 000000000..9aeb49e47--- /dev/null+++ b/test/e2e/kv-open-return-storage-object/actor/main.js@@ -0,0 +1,14 @@+import { Actor, KeyValueStore } from 'apify';++const mainOptions = {+ exit: Actor.isAtHome(),+ storage:+ process.env.STORAGE_IMPLEMENTATION === 'LOCAL'+ ? new (await import('@apify/storage-local')).ApifyStorageLocal()+ : undefined,+};++await Actor.main(async () => {+ const kv = await KeyValueStore.open();+ kv.setValue('storageObject', { storeObject: kv.storageObject });+}, mainOptions);diff --git a/test/e2e/kv-open-return-storage-object/actor/package.json b/test/e2e/kv-open-return-storage-object/actor/package.jsonnew file mode 100644index 000000000..f40826ba0--- /dev/null+++ b/test/e2e/kv-open-return-storage-object/actor/package.json@@ -0,0 +1,25 @@+{+ "name": "test-kv-open-return-storage-object",+ "version": "0.0.1",+ "description": "Key-Value Store - Return storage object on open",+ "dependencies": {+ "apify": "next",+ "@apify/storage-local": "^2.1.3",+ "@crawlee/basic": "file:./packages/basic-crawler",+ "@crawlee/core": "file:./packages/core",+ "@crawlee/memory-storage": "file:./packages/memory-storage",+ "@crawlee/types": "file:./packages/types",+ "@crawlee/utils": "file:./packages/utils"+ },+ "overrides": {+ "apify": {+ "@crawlee/core": "file:./packages/core",+ "@crawlee/utils": "file:./packages/utils"+ }+ },+ "scripts": {+ "start": "node main.js"+ },+ "type": "module",+ "license": "ISC"+}diff --git a/test/e2e/kv-open-return-storage-object/test.mjs b/test/e2e/kv-open-return-storage-object/test.mjsnew file mode 100644index 000000000..83de7b80d--- /dev/null+++ b/test/e2e/kv-open-return-storage-object/test.mjs@@ -0,0 +1,25 @@+import { initialize, expect, getActorTestDir, runActor } from '../tools.mjs';++/* This test verifies that the storageObject is correctly returned when the KeyValueStore is opened.+ * The storageObject is the result of the KeyValueStoreClient.get() method,+ * containing properties such as name, id, and other custom attributes.+ */++const testActorDirname = getActorTestDir(import.meta.url);+await initialize(testActorDirname);++const { defaultKeyValueStoreItems: items } = await runActor(testActorDirname);++await expect(items !== undefined, 'Key value store exists');++const item = items.find((kvItem) => kvItem.name === 'storageObject');++const parsed = JSON.parse(item.raw.toString());++await expect(+ typeof parsed === 'object' && parsed !== null,+ 'Key-value contains key "storeObject" and it\'s value is a non-nullable object',+);+await expect(parsed.id !== null, 'storeObject contains id');+await expect(parsed.name !== null, 'storeObject contains name');+await expect(parsed.userId !== null, 'storeObject contains userId');