Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat: add `Dataset.exportToValue` (#1553)
Matt Stephensacc6344f0e52
diff --git a/packages/core/src/storages/dataset.ts b/packages/core/src/storages/dataset.tsindex 7ec04739e..8b6d41921 100644--- a/packages/core/src/storages/dataset.ts+++ b/packages/core/src/storages/dataset.ts@@ -7,6 +7,8 @@ import type { Awaitable } from '../typedefs';import type { StorageManagerOptions } from './storage_manager';import { StorageManager } from './storage_manager';import { purgeDefaultStorages } from './utils';+import { KeyValueStore } from './key_value_store';+import type { RecordOptions } from './key_value_store';/** @internal */export const DATASET_ITERATORS_DEFAULT_LIMIT = 10000;@@ -204,6 +206,9 @@ export interface DatasetIteratorOptions extends Omit<DatasetDataOptions, 'offset* { foo: 'bar2', col2: 'val2' },* { col3: 123 },* ]);+ *+ * // Export the entirety of the dataset to one file in the key-value store+ * await dataset.exportToValue('MY-DATA', { contentType: 'text/csv' });* ```* @category Result Stores*/@@ -282,6 +287,19 @@ export class Dataset<Data extends Dictionary = Dictionary> {}}+ /**+ * Save the entirety of the dataset's contents into one file within a key-value store.+ *+ * @param key The name of the value to save the data in, defaults to `OUTPUT`.+ * @param options An optional options object where you can provide a `keyValueStoreName` and a `contentType` for the output.+ */+ async exportToValue(key?: string, options: RecordOptions & { keyValueStoreName?: string } = {}) {+ const { keyValueStoreName, ...recordOptions } = options;+ const kvStore = await KeyValueStore.open(keyValueStoreName ?? null);++ await kvStore.setValue(key || 'OUTPUT', await this.client.listItems(), recordOptions);+ }+/*** Returns an object containing general information about the dataset.*diff --git a/test/core/storages/dataset.test.ts b/test/core/storages/dataset.test.tsindex ecba21c35..89eeaf06a 100644--- a/test/core/storages/dataset.test.ts+++ b/test/core/storages/dataset.test.ts@@ -1,5 +1,5 @@import { MAX_PAYLOAD_SIZE_BYTES } from '@apify/consts';-import { Dataset, checkAndSerialize, chunkBySize, Configuration } from '@crawlee/core';+import { Dataset, checkAndSerialize, chunkBySize, Configuration, KeyValueStore } from '@crawlee/core';import type { Dictionary } from '@crawlee/utils';describe('dataset', () => {@@ -472,5 +472,21 @@ describe('dataset', () => {expect(chunkBySize(triple, (2 * size) + 3)).toEqual([`[${json},${json}]`, chunk]);expect(chunkBySize([...triple, ...triple], (2 * size) + 3)).toEqual([`[${json},${json}]`, `[${json},${json}]`, `[${json},${json}]`]);});++ test('exportToValue', async () => {+ const dataset = await Dataset.open('test-123-abc-123');++ const dataToPush = [{+ hello: 'world',+ }, {+ foo: 'bar',+ }];++ await dataset.pushData(dataToPush);+ await dataset.exportToValue('HELLO');++ const kvData = await KeyValueStore.getValue<{ items: [] }>('HELLO');+ expect(kvData.items).toEqual(dataToPush);+ });});});