Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat: add static `Dataset.exportToValue` (#1564)
Matt Stephensa7c17d434559
diff --git a/docs/examples/export_entire_dataset.mdx b/docs/examples/export_entire_dataset.mdxnew file mode 100644index 000000000..b6344f58a--- /dev/null+++ b/docs/examples/export_entire_dataset.mdx@@ -0,0 +1,14 @@+---+id: export-entire-dataset+title: Export entire dataset to one file+---++import CodeBlock from '@theme/CodeBlock';+import ApiLink from '@site/src/components/ApiLink';+import CrawlSource from '!!raw-loader!./export_entire_dataset.ts';++This `Dataset` example uses the `exportToValue` function to export the entire default dataset to a single CSV file into a key-value store named "my-data".++<CodeBlock className="language-js">+ {CrawlSource}+</CodeBlock>\ No newline at end of filediff --git a/docs/examples/export_entire_dataset.ts b/docs/examples/export_entire_dataset.tsnew file mode 100644index 000000000..a0d301036--- /dev/null+++ b/docs/examples/export_entire_dataset.ts@@ -0,0 +1,20 @@+import { Dataset } from 'crawlee';++// Retrieve or generate two items to be pushed+const data = [+ {+ id: 123,+ name: 'foo',+ },+ {+ id: 456,+ name: 'bar'+ },+];++// Push the two items to the default dataset+await Dataset.pushData(data);++// Export the entirety of the dataset to a single file in+// a key-value store named "my-data" under the key "OUTPUT"+await Dataset.exportToValue('OUTPUT', { contentType: 'text/csv', keyValueStoreName: 'my-data' });diff --git a/packages/core/src/storages/dataset.ts b/packages/core/src/storages/dataset.tsindex 8b6d41921..b3d95435e 100644--- a/packages/core/src/storages/dataset.ts+++ b/packages/core/src/storages/dataset.ts@@ -300,6 +300,17 @@ export class Dataset<Data extends Dictionary = Dictionary> {await kvStore.setValue(key || 'OUTPUT', await this.client.listItems(), recordOptions);}+ /**+ * Save the entirety of the default dataset's contents into one file within a key-value store.+ *+ * @param key The name of the value to save the data in, defaults to `OUTPUT`.+ * @param options An optional options object where you can provide a `keyValueStoreName` and a `contentType` for the output.+ */+ static async exportToValue(key?: string, options: RecordOptions & { keyValueStoreName?: string } = {}) {+ const dataset = await this.open();+ await dataset.exportToValue(key, options);+ }+/*** Returns an object containing general information about the dataset.*diff --git a/test/core/storages/dataset.test.ts b/test/core/storages/dataset.test.tsindex 89eeaf06a..02562f683 100644--- a/test/core/storages/dataset.test.ts+++ b/test/core/storages/dataset.test.ts@@ -473,20 +473,35 @@ describe('dataset', () => {expect(chunkBySize([...triple, ...triple], (2 * size) + 3)).toEqual([`[${json},${json}]`, `[${json},${json}]`, `[${json},${json}]`]);});- test('exportToValue', async () => {- const dataset = await Dataset.open('test-123-abc-123');+ describe('exportToValue', () => {+ const dataToPush = [+ {+ hello: 'world',+ },+ {+ foo: 'bar',+ },+ ];++ it('Should work', async () => {+ const dataset = await Dataset.open(Math.random().toString(36));+ await dataset.pushData(dataToPush);+ await dataset.exportToValue('HELLO');++ const kvData = await KeyValueStore.getValue<{ items: [] }>('HELLO');++ expect(kvData.items).toEqual(dataToPush);+ });- const dataToPush = [{- hello: 'world',- }, {- foo: 'bar',- }];+ it('Should work as a static method for the default dataset', async () => {+ await Dataset.pushData(dataToPush);- await dataset.pushData(dataToPush);- await dataset.exportToValue('HELLO');+ await Dataset.exportToValue('TEST-123-123');- const kvData = await KeyValueStore.getValue<{ items: [] }>('HELLO');- expect(kvData.items).toEqual(dataToPush);+ const kvData = await KeyValueStore.getValue<{ items: [] }>('TEST-123-123');++ expect(kvData.items).toEqual(dataToPush);+ });});});});diff --git a/website/versioned_docs/version-3.0/examples/export_entire_dataset.mdx b/website/versioned_docs/version-3.0/examples/export_entire_dataset.mdxnew file mode 100644index 000000000..b6344f58a--- /dev/null+++ b/website/versioned_docs/version-3.0/examples/export_entire_dataset.mdx@@ -0,0 +1,14 @@+---+id: export-entire-dataset+title: Export entire dataset to one file+---++import CodeBlock from '@theme/CodeBlock';+import ApiLink from '@site/src/components/ApiLink';+import CrawlSource from '!!raw-loader!./export_entire_dataset.ts';++This `Dataset` example uses the `exportToValue` function to export the entire default dataset to a single CSV file into a key-value store named "my-data".++<CodeBlock className="language-js">+ {CrawlSource}+</CodeBlock>\ No newline at end of filediff --git a/website/versioned_docs/version-3.0/examples/export_entire_dataset.ts b/website/versioned_docs/version-3.0/examples/export_entire_dataset.tsnew file mode 100644index 000000000..a0d301036--- /dev/null+++ b/website/versioned_docs/version-3.0/examples/export_entire_dataset.ts@@ -0,0 +1,20 @@+import { Dataset } from 'crawlee';++// Retrieve or generate two items to be pushed+const data = [+ {+ id: 123,+ name: 'foo',+ },+ {+ id: 456,+ name: 'bar'+ },+];++// Push the two items to the default dataset+await Dataset.pushData(data);++// Export the entirety of the dataset to a single file in+// a key-value store named "my-data" under the key "OUTPUT"+await Dataset.exportToValue('OUTPUT', { contentType: 'text/csv', keyValueStoreName: 'my-data' });