Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

refactor: set storage object on dataset open (#3116)

Daniil Poletaevd4dae7fc58ac
diff --git a/packages/core/src/storages/dataset.ts b/packages/core/src/storages/dataset.ts
index ec2331492..676c59a20 100644
--- a/packages/core/src/storages/dataset.ts
+++ b/packages/core/src/storages/dataset.ts
@@ -232,6 +232,7 @@ export class Dataset<Data extends Dictionary = Dictionary> {
id: string;
name?: string;
client: DatasetClient<Data>;
+ readonly storageObject?: Record<string, unknown>;
log: Log = log.child({ prefix: 'Dataset' });
/**
@@ -244,6 +245,7 @@ export class Dataset<Data extends Dictionary = Dictionary> {
this.id = options.id;
this.name = options.name;
this.client = options.client.dataset(this.id) as DatasetClient<Data>;
+ this.storageObject = options.storageObject;
}
/**
@@ -733,6 +735,7 @@ export interface DatasetOptions {
id: string;
name?: string;
client: StorageClient;
+ storageObject?: Record<string, unknown>;
}
export interface DatasetContent<Data> {
diff --git a/test/e2e/kv-open-return-storage-object/test.mjs b/test/e2e/kv-open-return-storage-object/test.mjs
deleted file mode 100644
index 83de7b80d..000000000
--- a/test/e2e/kv-open-return-storage-object/test.mjs
+++ /dev/null
@@ -1,25 +0,0 @@
-import { initialize, expect, getActorTestDir, runActor } from '../tools.mjs';
-
-/* This test verifies that the storageObject is correctly returned when the KeyValueStore is opened.
- * The storageObject is the result of the KeyValueStoreClient.get() method,
- * containing properties such as name, id, and other custom attributes.
- */
-
-const testActorDirname = getActorTestDir(import.meta.url);
-await initialize(testActorDirname);
-
-const { defaultKeyValueStoreItems: items } = await runActor(testActorDirname);
-
-await expect(items !== undefined, 'Key value store exists');
-
-const item = items.find((kvItem) => kvItem.name === 'storageObject');
-
-const parsed = JSON.parse(item.raw.toString());
-
-await expect(
- typeof parsed === 'object' && parsed !== null,
- 'Key-value contains key "storeObject" and it\'s value is a non-nullable object',
-);
-await expect(parsed.id !== null, 'storeObject contains id');
-await expect(parsed.name !== null, 'storeObject contains name');
-await expect(parsed.userId !== null, 'storeObject contains userId');
diff --git a/test/e2e/kv-open-return-storage-object/actor/.actor/actor.json b/test/e2e/storage-open-return-storage-object/actor/.actor/actor.json
similarity index 100%
rename from test/e2e/kv-open-return-storage-object/actor/.actor/actor.json
rename to test/e2e/storage-open-return-storage-object/actor/.actor/actor.json
diff --git a/test/e2e/kv-open-return-storage-object/actor/.gitignore b/test/e2e/storage-open-return-storage-object/actor/.gitignore
similarity index 100%
rename from test/e2e/kv-open-return-storage-object/actor/.gitignore
rename to test/e2e/storage-open-return-storage-object/actor/.gitignore
diff --git a/test/e2e/kv-open-return-storage-object/actor/Dockerfile b/test/e2e/storage-open-return-storage-object/actor/Dockerfile
similarity index 100%
rename from test/e2e/kv-open-return-storage-object/actor/Dockerfile
rename to test/e2e/storage-open-return-storage-object/actor/Dockerfile
diff --git a/test/e2e/kv-open-return-storage-object/actor/main.js b/test/e2e/storage-open-return-storage-object/actor/main.js
similarity index 56%
rename from test/e2e/kv-open-return-storage-object/actor/main.js
rename to test/e2e/storage-open-return-storage-object/actor/main.js
index 72079180a..88ed1479c 100644
--- a/test/e2e/kv-open-return-storage-object/actor/main.js
+++ b/test/e2e/storage-open-return-storage-object/actor/main.js
@@ -1,4 +1,4 @@
-import { Actor, KeyValueStore } from 'apify';
+import { Actor, KeyValueStore, Dataset } from 'apify';
const mainOptions = {
exit: Actor.isAtHome(),
@@ -10,5 +10,9 @@ const mainOptions = {
await Actor.main(async () => {
const kv = await KeyValueStore.open();
- await kv.setValue('storageObject', { storeObject: kv.storageObject });
+ const dataset = await Dataset.open();
+ await kv.setValue('storageObject', {
+ keyValueStorageObject: kv.storageObject,
+ datasetStorageObject: dataset.storageObject,
+ });
}, mainOptions);
diff --git a/test/e2e/kv-open-return-storage-object/actor/package.json b/test/e2e/storage-open-return-storage-object/actor/package.json
similarity index 100%
rename from test/e2e/kv-open-return-storage-object/actor/package.json
rename to test/e2e/storage-open-return-storage-object/actor/package.json
diff --git a/test/e2e/storage-open-return-storage-object/test.mjs b/test/e2e/storage-open-return-storage-object/test.mjs
new file mode 100644
index 000000000..1ff6a89cd
--- /dev/null
+++ b/test/e2e/storage-open-return-storage-object/test.mjs
@@ -0,0 +1,33 @@
+import { initialize, expect, getActorTestDir, runActor } from '../tools.mjs';
+
+/* This test verifies that the storageObject is correctly returned when the KeyValueStore or Dataset is opened.
+ * The storageObject is the result of the KeyValueStoreClient.get() or Dataset.get() methods,
+ * containing properties such as name, id, and other custom attributes.
+ */
+
+const testActorDirname = getActorTestDir(import.meta.url);
+await initialize(testActorDirname);
+
+const { defaultKeyValueStoreItems: items } = await runActor(testActorDirname);
+
+await expect(items !== undefined, 'Key value store exists');
+
+const item = items.find((kvItem) => kvItem.name === 'storageObject');
+
+const parsed = JSON.parse(item.raw.toString());
+
+await expect(
+ typeof parsed === 'object' && parsed !== null,
+ 'Key-value contains key "storageObject" and it\'s value is a non-nullable object',
+);
+
+const datasetStorageObject = parsed.datasetStorageObject;
+const keyValueStorageObject = parsed.keyValueStorageObject;
+
+await expect(datasetStorageObject.id !== null, 'datasetStorageObject contains id');
+await expect(datasetStorageObject.name !== null, 'datasetStorageObject contains name');
+await expect(datasetStorageObject.userId !== null, 'datasetStorageObject contains userId');
+
+await expect(keyValueStorageObject.id !== null, 'keyValueStorageObject contains id');
+await expect(keyValueStorageObject.name !== null, 'keyValueStorageObject contains name');
+await expect(keyValueStorageObject.userId !== null, 'keyValueStorageObject contains userId');