Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

feat: add static `Dataset.exportToValue` (#1564)

Matt Stephensa7c17d434559
diff --git a/docs/examples/export_entire_dataset.mdx b/docs/examples/export_entire_dataset.mdx
new file mode 100644
index 000000000..b6344f58a
--- /dev/null
+++ b/docs/examples/export_entire_dataset.mdx
@@ -0,0 +1,14 @@
+---
+id: export-entire-dataset
+title: Export entire dataset to one file
+---
+
+import CodeBlock from '@theme/CodeBlock';
+import ApiLink from '@site/src/components/ApiLink';
+import CrawlSource from '!!raw-loader!./export_entire_dataset.ts';
+
+This `Dataset` example uses the `exportToValue` function to export the entire default dataset to a single CSV file into a key-value store named "my-data".
+
+<CodeBlock className="language-js">
+ {CrawlSource}
+</CodeBlock>
\ No newline at end of file
diff --git a/docs/examples/export_entire_dataset.ts b/docs/examples/export_entire_dataset.ts
new file mode 100644
index 000000000..a0d301036
--- /dev/null
+++ b/docs/examples/export_entire_dataset.ts
@@ -0,0 +1,20 @@
+import { Dataset } from 'crawlee';
+
+// Retrieve or generate two items to be pushed
+const data = [
+ {
+ id: 123,
+ name: 'foo',
+ },
+ {
+ id: 456,
+ name: 'bar'
+ },
+];
+
+// Push the two items to the default dataset
+await Dataset.pushData(data);
+
+// Export the entirety of the dataset to a single file in
+// a key-value store named "my-data" under the key "OUTPUT"
+await Dataset.exportToValue('OUTPUT', { contentType: 'text/csv', keyValueStoreName: 'my-data' });
diff --git a/packages/core/src/storages/dataset.ts b/packages/core/src/storages/dataset.ts
index 8b6d41921..b3d95435e 100644
--- a/packages/core/src/storages/dataset.ts
+++ b/packages/core/src/storages/dataset.ts
@@ -300,6 +300,17 @@ export class Dataset<Data extends Dictionary = Dictionary> {
await kvStore.setValue(key || 'OUTPUT', await this.client.listItems(), recordOptions);
}
+ /**
+ * Save the entirety of the default dataset's contents into one file within a key-value store.
+ *
+ * @param key The name of the value to save the data in, defaults to `OUTPUT`.
+ * @param options An optional options object where you can provide a `keyValueStoreName` and a `contentType` for the output.
+ */
+ static async exportToValue(key?: string, options: RecordOptions & { keyValueStoreName?: string } = {}) {
+ const dataset = await this.open();
+ await dataset.exportToValue(key, options);
+ }
+
/**
* Returns an object containing general information about the dataset.
*
diff --git a/test/core/storages/dataset.test.ts b/test/core/storages/dataset.test.ts
index 89eeaf06a..02562f683 100644
--- a/test/core/storages/dataset.test.ts
+++ b/test/core/storages/dataset.test.ts
@@ -473,20 +473,35 @@ describe('dataset', () => {
expect(chunkBySize([...triple, ...triple], (2 * size) + 3)).toEqual([`[${json},${json}]`, `[${json},${json}]`, `[${json},${json}]`]);
});
- test('exportToValue', async () => {
- const dataset = await Dataset.open('test-123-abc-123');
+ describe('exportToValue', () => {
+ const dataToPush = [
+ {
+ hello: 'world',
+ },
+ {
+ foo: 'bar',
+ },
+ ];
+
+ it('Should work', async () => {
+ const dataset = await Dataset.open(Math.random().toString(36));
+ await dataset.pushData(dataToPush);
+ await dataset.exportToValue('HELLO');
+
+ const kvData = await KeyValueStore.getValue<{ items: [] }>('HELLO');
+
+ expect(kvData.items).toEqual(dataToPush);
+ });
- const dataToPush = [{
- hello: 'world',
- }, {
- foo: 'bar',
- }];
+ it('Should work as a static method for the default dataset', async () => {
+ await Dataset.pushData(dataToPush);
- await dataset.pushData(dataToPush);
- await dataset.exportToValue('HELLO');
+ await Dataset.exportToValue('TEST-123-123');
- const kvData = await KeyValueStore.getValue<{ items: [] }>('HELLO');
- expect(kvData.items).toEqual(dataToPush);
+ const kvData = await KeyValueStore.getValue<{ items: [] }>('TEST-123-123');
+
+ expect(kvData.items).toEqual(dataToPush);
+ });
});
});
});
diff --git a/website/versioned_docs/version-3.0/examples/export_entire_dataset.mdx b/website/versioned_docs/version-3.0/examples/export_entire_dataset.mdx
new file mode 100644
index 000000000..b6344f58a
--- /dev/null
+++ b/website/versioned_docs/version-3.0/examples/export_entire_dataset.mdx
@@ -0,0 +1,14 @@
+---
+id: export-entire-dataset
+title: Export entire dataset to one file
+---
+
+import CodeBlock from '@theme/CodeBlock';
+import ApiLink from '@site/src/components/ApiLink';
+import CrawlSource from '!!raw-loader!./export_entire_dataset.ts';
+
+This `Dataset` example uses the `exportToValue` function to export the entire default dataset to a single CSV file into a key-value store named "my-data".
+
+<CodeBlock className="language-js">
+ {CrawlSource}
+</CodeBlock>
\ No newline at end of file
diff --git a/website/versioned_docs/version-3.0/examples/export_entire_dataset.ts b/website/versioned_docs/version-3.0/examples/export_entire_dataset.ts
new file mode 100644
index 000000000..a0d301036
--- /dev/null
+++ b/website/versioned_docs/version-3.0/examples/export_entire_dataset.ts
@@ -0,0 +1,20 @@
+import { Dataset } from 'crawlee';
+
+// Retrieve or generate two items to be pushed
+const data = [
+ {
+ id: 123,
+ name: 'foo',
+ },
+ {
+ id: 456,
+ name: 'bar'
+ },
+];
+
+// Push the two items to the default dataset
+await Dataset.pushData(data);
+
+// Export the entirety of the dataset to a single file in
+// a key-value store named "my-data" under the key "OUTPUT"
+await Dataset.exportToValue('OUTPUT', { contentType: 'text/csv', keyValueStoreName: 'my-data' });