Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
refactor: set storage object on dataset open (#3116)
Daniil Poletaevd4dae7fc58ac
diff --git a/packages/core/src/storages/dataset.ts b/packages/core/src/storages/dataset.tsindex ec2331492..676c59a20 100644--- a/packages/core/src/storages/dataset.ts+++ b/packages/core/src/storages/dataset.ts@@ -232,6 +232,7 @@ export class Dataset<Data extends Dictionary = Dictionary> {id: string;name?: string;client: DatasetClient<Data>;+ readonly storageObject?: Record<string, unknown>;log: Log = log.child({ prefix: 'Dataset' });/**@@ -244,6 +245,7 @@ export class Dataset<Data extends Dictionary = Dictionary> {this.id = options.id;this.name = options.name;this.client = options.client.dataset(this.id) as DatasetClient<Data>;+ this.storageObject = options.storageObject;}/**@@ -733,6 +735,7 @@ export interface DatasetOptions {id: string;name?: string;client: StorageClient;+ storageObject?: Record<string, unknown>;}export interface DatasetContent<Data> {diff --git a/test/e2e/kv-open-return-storage-object/test.mjs b/test/e2e/kv-open-return-storage-object/test.mjsdeleted file mode 100644index 83de7b80d..000000000--- a/test/e2e/kv-open-return-storage-object/test.mjs+++ /dev/null@@ -1,25 +0,0 @@-import { initialize, expect, getActorTestDir, runActor } from '../tools.mjs';--/* This test verifies that the storageObject is correctly returned when the KeyValueStore is opened.- * The storageObject is the result of the KeyValueStoreClient.get() method,- * containing properties such as name, id, and other custom attributes.- */--const testActorDirname = getActorTestDir(import.meta.url);-await initialize(testActorDirname);--const { defaultKeyValueStoreItems: items } = await runActor(testActorDirname);--await expect(items !== undefined, 'Key value store exists');--const item = items.find((kvItem) => kvItem.name === 'storageObject');--const parsed = JSON.parse(item.raw.toString());--await expect(- typeof parsed === 'object' && parsed !== null,- 'Key-value contains key "storeObject" and it\'s value is a non-nullable object',-);-await expect(parsed.id !== null, 'storeObject contains id');-await expect(parsed.name !== null, 'storeObject contains name');-await expect(parsed.userId !== null, 'storeObject contains userId');diff --git a/test/e2e/kv-open-return-storage-object/actor/.actor/actor.json b/test/e2e/storage-open-return-storage-object/actor/.actor/actor.jsonsimilarity index 100%rename from test/e2e/kv-open-return-storage-object/actor/.actor/actor.jsonrename to test/e2e/storage-open-return-storage-object/actor/.actor/actor.jsondiff --git a/test/e2e/kv-open-return-storage-object/actor/.gitignore b/test/e2e/storage-open-return-storage-object/actor/.gitignoresimilarity index 100%rename from test/e2e/kv-open-return-storage-object/actor/.gitignorerename to test/e2e/storage-open-return-storage-object/actor/.gitignorediff --git a/test/e2e/kv-open-return-storage-object/actor/Dockerfile b/test/e2e/storage-open-return-storage-object/actor/Dockerfilesimilarity index 100%rename from test/e2e/kv-open-return-storage-object/actor/Dockerfilerename to test/e2e/storage-open-return-storage-object/actor/Dockerfilediff --git a/test/e2e/kv-open-return-storage-object/actor/main.js b/test/e2e/storage-open-return-storage-object/actor/main.jssimilarity index 56%rename from test/e2e/kv-open-return-storage-object/actor/main.jsrename to test/e2e/storage-open-return-storage-object/actor/main.jsindex 72079180a..88ed1479c 100644--- a/test/e2e/kv-open-return-storage-object/actor/main.js+++ b/test/e2e/storage-open-return-storage-object/actor/main.js@@ -1,4 +1,4 @@-import { Actor, KeyValueStore } from 'apify';+import { Actor, KeyValueStore, Dataset } from 'apify';const mainOptions = {exit: Actor.isAtHome(),@@ -10,5 +10,9 @@ const mainOptions = {await Actor.main(async () => {const kv = await KeyValueStore.open();- await kv.setValue('storageObject', { storeObject: kv.storageObject });+ const dataset = await Dataset.open();+ await kv.setValue('storageObject', {+ keyValueStorageObject: kv.storageObject,+ datasetStorageObject: dataset.storageObject,+ });}, mainOptions);diff --git a/test/e2e/kv-open-return-storage-object/actor/package.json b/test/e2e/storage-open-return-storage-object/actor/package.jsonsimilarity index 100%rename from test/e2e/kv-open-return-storage-object/actor/package.jsonrename to test/e2e/storage-open-return-storage-object/actor/package.jsondiff --git a/test/e2e/storage-open-return-storage-object/test.mjs b/test/e2e/storage-open-return-storage-object/test.mjsnew file mode 100644index 000000000..1ff6a89cd--- /dev/null+++ b/test/e2e/storage-open-return-storage-object/test.mjs@@ -0,0 +1,33 @@+import { initialize, expect, getActorTestDir, runActor } from '../tools.mjs';++/* This test verifies that the storageObject is correctly returned when the KeyValueStore or Dataset is opened.+ * The storageObject is the result of the KeyValueStoreClient.get() or Dataset.get() methods,+ * containing properties such as name, id, and other custom attributes.+ */++const testActorDirname = getActorTestDir(import.meta.url);+await initialize(testActorDirname);++const { defaultKeyValueStoreItems: items } = await runActor(testActorDirname);++await expect(items !== undefined, 'Key value store exists');++const item = items.find((kvItem) => kvItem.name === 'storageObject');++const parsed = JSON.parse(item.raw.toString());++await expect(+ typeof parsed === 'object' && parsed !== null,+ 'Key-value contains key "storageObject" and it\'s value is a non-nullable object',+);++const datasetStorageObject = parsed.datasetStorageObject;+const keyValueStorageObject = parsed.keyValueStorageObject;++await expect(datasetStorageObject.id !== null, 'datasetStorageObject contains id');+await expect(datasetStorageObject.name !== null, 'datasetStorageObject contains name');+await expect(datasetStorageObject.userId !== null, 'datasetStorageObject contains userId');++await expect(keyValueStorageObject.id !== null, 'keyValueStorageObject contains id');+await expect(keyValueStorageObject.name !== null, 'keyValueStorageObject contains name');+await expect(keyValueStorageObject.userId !== null, 'keyValueStorageObject contains userId');