Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

refactor: finalize DS exports to CSV/JSON (#1589)

Martin Adámekebf5cdd0742f
diff --git a/docs/examples/export_entire_dataset.ts b/docs/examples/export_entire_dataset.ts
index a0d301036..fc43eba98 100644
--- a/docs/examples/export_entire_dataset.ts
+++ b/docs/examples/export_entire_dataset.ts
@@ -8,7 +8,7 @@ const data = [
},
{
id: 456,
- name: 'bar'
+ name: 'bar',
},
];
@@ -17,4 +17,4 @@ await Dataset.pushData(data);
// Export the entirety of the dataset to a single file in
// a key-value store named "my-data" under the key "OUTPUT"
-await Dataset.exportToValue('OUTPUT', { contentType: 'text/csv', keyValueStoreName: 'my-data' });
+await Dataset.exportToCSV('OUTPUT', { toKVS: 'my-data' });
diff --git a/package-lock.json b/package-lock.json
index 492da8406..217203ad5 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -6377,6 +6377,11 @@
"resolved": "https://registry.npmjs.org/cssom/-/cssom-0.3.8.tgz",
"integrity": "sha512-b0tGHbfegbhPJpxpiBPU2sCkigAqtM9O121le6bbOlgyV+NyGyCmVfJ6QW9eRjz8CpNfWEOYBIMIGRYkLwsIYg=="
},
+ "node_modules/csv-stringify": {
+ "version": "6.2.0",
+ "resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-6.2.0.tgz",
+ "integrity": "sha512-dcUbQLRTTDcgQxgEU8V9IctkaCwHZjZfzUZ5ZB3RY8Y+pXtdtl5iVQHfGzANytFFkRKanYzBXrkfpNdGR7eviA=="
+ },
"node_modules/damerau-levenshtein": {
"version": "1.0.8",
"resolved": "https://registry.npmjs.org/damerau-levenshtein/-/damerau-levenshtein-1.0.8.tgz",
@@ -16741,6 +16746,7 @@
"@crawlee/utils": "^3.0.4",
"@types/tough-cookie": "^4.0.2",
"@vladfrangu/async_event_emitter": "^2.0.0",
+ "csv-stringify": "^6.2.0",
"fs-extra": "^10.1.0",
"json5": "^2.2.1",
"minimatch": "^5.1.0",
@@ -18122,6 +18128,7 @@
"@crawlee/utils": "^3.0.4",
"@types/tough-cookie": "^4.0.2",
"@vladfrangu/async_event_emitter": "^2.0.0",
+ "csv-stringify": "^6.2.0",
"fs-extra": "^10.1.0",
"json5": "^2.2.1",
"minimatch": "^5.1.0",
@@ -22620,6 +22627,11 @@
}
}
},
+ "csv-stringify": {
+ "version": "6.2.0",
+ "resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-6.2.0.tgz",
+ "integrity": "sha512-dcUbQLRTTDcgQxgEU8V9IctkaCwHZjZfzUZ5ZB3RY8Y+pXtdtl5iVQHfGzANytFFkRKanYzBXrkfpNdGR7eviA=="
+ },
"damerau-levenshtein": {
"version": "1.0.8",
"resolved": "https://registry.npmjs.org/damerau-levenshtein/-/damerau-levenshtein-1.0.8.tgz",
diff --git a/packages/core/package.json b/packages/core/package.json
index aba2a658a..abcf4a496 100644
--- a/packages/core/package.json
+++ b/packages/core/package.json
@@ -63,6 +63,7 @@
"@crawlee/utils": "^3.0.4",
"@types/tough-cookie": "^4.0.2",
"@vladfrangu/async_event_emitter": "^2.0.0",
+ "csv-stringify": "^6.2.0",
"fs-extra": "^10.1.0",
"json5": "^2.2.1",
"minimatch": "^5.1.0",
diff --git a/packages/core/src/events/event_manager.ts b/packages/core/src/events/event_manager.ts
index bb7833058..184abf1aa 100644
--- a/packages/core/src/events/event_manager.ts
+++ b/packages/core/src/events/event_manager.ts
@@ -25,7 +25,9 @@ export abstract class EventManager {
protected intervals: Intervals = {};
protected log = log.child({ prefix: 'Events' });
- constructor(readonly config = Configuration.getGlobalConfig()) {}
+ constructor(readonly config = Configuration.getGlobalConfig()) {
+ this.events.setMaxListeners(50);
+ }
/**
* Initializes the event manager by creating the `persistState` event interval.
diff --git a/packages/core/src/storages/dataset.ts b/packages/core/src/storages/dataset.ts
index f31d343bc..e3ffb3858 100644
--- a/packages/core/src/storages/dataset.ts
+++ b/packages/core/src/storages/dataset.ts
@@ -1,6 +1,7 @@
import { MAX_PAYLOAD_SIZE_BYTES } from '@apify/consts';
import ow from 'ow';
-import type { DatasetClient, DatasetInfo, StorageClient, Dictionary } from '@crawlee/types';
+import { stringify } from 'csv-stringify/sync';
+import type { DatasetClient, DatasetInfo, Dictionary, StorageClient } from '@crawlee/types';
import { Configuration } from '../configuration';
import { log } from '../log';
import type { Awaitable } from '../typedefs';
@@ -8,7 +9,6 @@ import type { StorageManagerOptions } from './storage_manager';
import { StorageManager } from './storage_manager';
import { purgeDefaultStorages } from './utils';
import { KeyValueStore } from './key_value_store';
-import type { RecordOptions } from './key_value_store';
/** @internal */
export const DATASET_ITERATORS_DEFAULT_LIMIT = 10000;
@@ -161,6 +161,11 @@ export interface DatasetIteratorOptions extends Omit<DatasetDataOptions, 'offset
format?: string;
}
+export interface ExportOptions {
+ fromDataset?: string;
+ toKVS?: string;
+}
+
/**
* The `Dataset` class represents a store for structured data where each object stored has the same attributes,
* such as online store products or real estate offers. You can imagine it as a table,
@@ -208,7 +213,7 @@ export interface DatasetIteratorOptions extends Omit<DatasetDataOptions, 'offset
* ]);
*
* // Export the entirety of the dataset to one file in the key-value store
- * await dataset.exportToValue('MY-DATA', { contentType: 'text/csv' });
+ * await dataset.exportToCSV('MY-DATA');
* ```
* @category Result Stores
*/
@@ -290,25 +295,86 @@ export class Dataset<Data extends Dictionary = Dictionary> {
/**
* Save the entirety of the dataset's contents into one file within a key-value store.
*
- * @param key The name of the value to save the data in, defaults to `OUTPUT`.
- * @param options An optional options object where you can provide a `keyValueStoreName` and a `contentType` for the output.
+ * @param key The name of the value to save the data in.
+ * @param [options] An optional options object where you can provide the dataset and target KVS name.
+ * @param [contentType] Only JSON and CSV are supported currently, defaults to JSON.
*/
- async exportToValue(key?: string, options: RecordOptions & { keyValueStoreName?: string } = {}) {
- const { keyValueStoreName, ...recordOptions } = options;
- const kvStore = await KeyValueStore.open(keyValueStoreName ?? null);
- const value = await this.client.listItems();
- await kvStore.setValue(key ?? 'OUTPUT', value.items, recordOptions);
+ async exportTo(key: string, options?: ExportOptions, contentType?: string): Promise<void> {
+ const kvStore = await KeyValueStore.open(options?.toKVS ?? null);
+ const items: Data[] = [];
+
+ const fetchNextChunk = async (offset = 0): Promise<void> => {
+ const limit = 1000;
+ const value = await this.client.listItems({ offset, limit });
+
+ if (value.count === 0) {
+ return;
+ }
+
+ items.push(...value.items);
+
+ if (value.total > offset + value.count) {
+ return fetchNextChunk(offset + value.count);
+ }
+ };
+
+ await fetchNextChunk();
+
+ if (contentType === 'text/csv') {
+ const value = stringify([
+ Object.keys(items[0]),
+ ...items.map((item) => Object.values(item)),
+ ]);
+ return kvStore.setValue(key, value, { contentType });
+ }
+
+ if (contentType === 'application/json') {
+ return kvStore.setValue(key, items);
+ }
+
+ throw new Error(`Unsupported content type: ${contentType}`);
}
/**
- * Save the entirety of the default dataset's contents into one file within a key-value store.
+ * Save entire default dataset's contents into one JSON file within a key-value store.
*
- * @param key The name of the value to save the data in, defaults to `OUTPUT`.
- * @param options An optional options object where you can provide a `keyValueStoreName` and a `contentType` for the output.
+ * @param key The name of the value to save the data in.
+ * @param [options] An optional options object where you can provide the target KVS name.
*/
- static async exportToValue(key?: string, options: RecordOptions & { keyValueStoreName?: string } = {}) {
- const dataset = await this.open();
- await dataset.exportToValue(key, options);
+ async exportToJSON(key: string, options?: Omit<ExportOptions, 'fromDataset'>) {
+ await this.exportTo(key, options, 'application/json');
+ }
+
+ /**
+ * Save entire default dataset's contents into one CSV file within a key-value store.
+ *
+ * @param key The name of the value to save the data in.
+ * @param [options] An optional options object where you can provide the target KVS name.
+ */
+ async exportToCSV(key: string, options?: Omit<ExportOptions, 'fromDataset'>) {
+ await this.exportTo(key, options, 'text/csv');
+ }
+
+ /**
+ * Save entire default dataset's contents into one JSON file within a key-value store.
+ *
+ * @param key The name of the value to save the data in.
+ * @param [options] An optional options object where you can provide the dataset and target KVS name.
+ */
+ static async exportToJSON(key: string, options?: ExportOptions) {
+ const dataset = await this.open(options?.fromDataset);
+ await dataset.exportToJSON(key, options);
+ }
+
+ /**
+ * Save entire default dataset's contents into one CSV file within a key-value store.
+ *
+ * @param key The name of the value to save the data in.
+ * @param [options] An optional options object where you can provide the dataset and target KVS name.
+ */
+ static async exportToCSV(key: string, options?: ExportOptions) {
+ const dataset = await this.open(options?.fromDataset);
+ await dataset.exportToCSV(key, options);
}
/**
diff --git a/test/core/storages/dataset.test.ts b/test/core/storages/dataset.test.ts
index 7d61abe5b..6ec53e43a 100644
--- a/test/core/storages/dataset.test.ts
+++ b/test/core/storages/dataset.test.ts
@@ -1,6 +1,17 @@
import { MAX_PAYLOAD_SIZE_BYTES } from '@apify/consts';
import { Dataset, checkAndSerialize, chunkBySize, Configuration, KeyValueStore } from '@crawlee/core';
import type { Dictionary } from '@crawlee/utils';
+import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator';
+
+const localStorageEmulator = new MemoryStorageEmulator();
+
+beforeEach(async () => {
+ await localStorageEmulator.init();
+});
+
+afterAll(async () => {
+ await localStorageEmulator.destroy();
+});
describe('dataset', () => {
const storageClient = Configuration.getStorageClient();
@@ -473,7 +484,7 @@ describe('dataset', () => {
expect(chunkBySize([...triple, ...triple], (2 * size) + 3)).toEqual([`[${json},${json}]`, `[${json},${json}]`, `[${json},${json}]`]);
});
- describe('exportToValue', () => {
+ describe('exportToJSON', () => {
const dataToPush = [
{
hello: 'world',
@@ -486,7 +497,7 @@ describe('dataset', () => {
it('Should work', async () => {
const dataset = await Dataset.open(Math.random().toString(36));
await dataset.pushData(dataToPush);
- await dataset.exportToValue('HELLO');
+ await dataset.exportToJSON('HELLO');
const kvData = await KeyValueStore.getValue('HELLO');
expect(kvData).toEqual(dataToPush);
@@ -494,11 +505,41 @@ describe('dataset', () => {
it('Should work as a static method for the default dataset', async () => {
await Dataset.pushData(dataToPush);
- await Dataset.exportToValue('TEST-123-123');
+ await Dataset.exportToJSON('TEST-123-123');
const kvData = await KeyValueStore.getValue('TEST-123-123');
expect(kvData).toEqual(dataToPush);
});
});
+
+ describe('exportToCSV', () => {
+ const dataToPush = [
+ {
+ hello: 'world 1',
+ foo: 'bar 1',
+ },
+ {
+ hello: 'world 2',
+ foo: 'bar 2',
+ },
+ ];
+
+ it('Should work', async () => {
+ const dataset = await Dataset.open(Math.random().toString(36));
+ await dataset.pushData(dataToPush);
+ await dataset.exportToCSV('HELLO-csv');
+
+ const kvData = await KeyValueStore.getValue('HELLO-csv');
+ expect(kvData).toEqual('hello,foo\nworld 1,bar 1\nworld 2,bar 2\n');
+ });
+
+ it('Should work as a static method for the default dataset', async () => {
+ await Dataset.pushData(dataToPush);
+ await Dataset.exportToCSV('TEST-123-123-csv');
+
+ const kvData = await KeyValueStore.getValue('TEST-123-123-csv');
+ expect(kvData).toEqual('hello,foo\nworld 1,bar 1\nworld 2,bar 2\n');
+ });
+ });
});
});