Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat(core): add `crawler.exportData()` helper (#2166)
Martin Adámekc8c09a54a712
diff --git a/.github/workflows/test-ci.yml b/.github/workflows/test-ci.ymlindex 9e450d4ee..c6130d09b 100644--- a/.github/workflows/test-ci.yml+++ b/.github/workflows/test-ci.yml@@ -6,6 +6,9 @@ on:pull_request:branches: [ master ]+env:+ YARN_IGNORE_NODE: 1+jobs:# `yarn install` is done in a separate job and cached to speed up the following jobs.build_and_test:diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex f2a927b75..83c31ddc0 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -1,6 +1,8 @@-import defaultLog, { LogLevel } from '@apify/log';+import { dirname } from 'node:path';+import type { Log } from '@apify/log';-import { addTimeoutToPromise, tryCancel, TimeoutError } from '@apify/timeout';+import defaultLog, { LogLevel } from '@apify/log';+import { addTimeoutToPromise, TimeoutError, tryCancel } from '@apify/timeout';import { cryptoRandomObjectId } from '@apify/utilities';import type {AddRequestsBatchedOptions,@@ -9,6 +11,7 @@ import type {CrawlingContext,EnqueueLinksOptions,EventManager,+ DatasetExportOptions,FinalStatistics,GetUserDataFromRequest,ProxyInfo,@@ -24,28 +27,30 @@ import type {StatisticState,} from '@crawlee/core';import {- Dataset,AutoscaledPool,Configuration,+ CriticalError,+ Dataset,+ enqueueLinks,EventType,KeyValueStore,+ mergeCookies,NonRetryableError,+ purgeDefaultStorages,RequestQueue,RequestQueueV2,RequestState,RetryRequestError,Router,+ SessionError,SessionPool,Statistics,- enqueueLinks,- mergeCookies,- purgeDefaultStorages,validators,- SessionError,- CriticalError,} from '@crawlee/core';-import type { Dictionary, Awaitable, BatchAddRequestsResult, SetStatusMessageOptions } from '@crawlee/types';+import type { Awaitable, BatchAddRequestsResult, Dictionary, SetStatusMessageOptions } from '@crawlee/types';import { ROTATE_PROXY_ERRORS } from '@crawlee/utils';+import { stringify } from 'csv-stringify/sync';+import { ensureDir, writeFile, writeJSON } from 'fs-extra';import type { Method, OptionsInit } from 'got-scraping';import { gotScraping } from 'got-scraping';import ow, { ArgumentError } from 'ow';@@ -935,12 +940,12 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext* Pushes data to the default crawler {@apilink Dataset} by calling {@apilink Dataset.pushData}.*/async pushData(...args: Parameters<Dataset['pushData']>): Promise<void> {- const dataset = await Dataset.open(undefined, { config: this.config });+ const dataset = await this.getDataset();return dataset.pushData(...args);}/**- * Retrieves the default crawler {@apilink Dataset} by calling {@apilink Dataset.open}.+ * Retrieves the default crawler {@apilink Dataset}.*/async getDataset(): Promise<Dataset> {return Dataset.open(undefined, { config: this.config });@@ -954,6 +959,47 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextreturn dataset.getData(...args);}+ /**+ * Retrieves all the data from the default crawler {@apilink Dataset} and exports them to the specified format.+ * Supported formats are currently 'json' and 'csv', and will be inferred from the `path` automatically.+ */+ async exportData<Data>(path: string, format?: 'json' | 'csv', options?: DatasetExportOptions): Promise<Data[]> {+ const supportedFormats = ['json', 'csv'];++ if (!format && path.match(/\.(json|csv)$/i)) {+ format = path.toLowerCase().match(/\.(json|csv)$/)![1] as 'json' | 'csv';+ }++ if (!format) {+ throw new Error(`Failed to infer format from the path: '${path}'. Supported formats: ${supportedFormats.join(', ')}`);+ }++ if (!supportedFormats.includes(format)) {+ throw new Error(`Unsupported format: '${format}'. Use one of ${supportedFormats.join(', ')}`);+ }++ const dataset = await this.getDataset();+ const items = await dataset.export(options);++ if (format === 'csv') {+ const value = stringify([+ Object.keys(items[0]),+ ...items.map((item) => Object.values(item)),+ ]);+ await ensureDir(dirname(path));+ await writeFile(path, value);+ this.log.info(`Export to ${path} finished!`);+ }++ if (format === 'json') {+ await ensureDir(dirname(path));+ await writeJSON(path, items, { spaces: 4 });+ this.log.info(`Export to ${path} finished!`);+ }++ return items;+ }+protected async _init(): Promise<void> {if (!this.events.isInitialized()) {await this.events.init();diff --git a/packages/core/src/storages/dataset.ts b/packages/core/src/storages/dataset.tsindex 333067af2..35fa2391f 100644--- a/packages/core/src/storages/dataset.ts+++ b/packages/core/src/storages/dataset.ts@@ -139,6 +139,8 @@ export interface DatasetDataOptions {skipEmpty?: boolean;}+export interface DatasetExportOptions extends Omit<DatasetDataOptions, 'offset' | 'limit'> {}+export interface DatasetIteratorOptions extends Omit<DatasetDataOptions, 'offset' | 'limit' | 'clean' | 'skipHidden' | 'skipEmpty'> {/** @internal */offset?: number;@@ -162,7 +164,7 @@ export interface DatasetIteratorOptions extends Omit<DatasetDataOptions, 'offsetformat?: string;}-export interface ExportOptions {+export interface DatasetExportToOptions extends DatasetExportOptions {fromDataset?: string;toKVS?: string;}@@ -294,19 +296,15 @@ export class Dataset<Data extends Dictionary = Dictionary> {}/**- * Save the entirety of the dataset's contents into one file within a key-value store.- *- * @param key The name of the value to save the data in.- * @param [options] An optional options object where you can provide the dataset and target KVS name.- * @param [contentType] Only JSON and CSV are supported currently, defaults to JSON.+ * Returns all the data from the dataset. This will iterate through the whole dataset+ * via the `listItems()` client method, which gives you only paginated results.*/- async exportTo(key: string, options?: ExportOptions, contentType?: string): Promise<void> {- const kvStore = await KeyValueStore.open(options?.toKVS ?? null, { config: this.config });+ async export(options: DatasetExportOptions = {}): Promise<Data[]> {const items: Data[] = [];const fetchNextChunk = async (offset = 0): Promise<void> => {const limit = 1000;- const value = await this.client.listItems({ offset, limit });+ const value = await this.client.listItems({ offset, limit, ...options });if (value.count === 0) {return;@@ -321,19 +319,37 @@ export class Dataset<Data extends Dictionary = Dictionary> {await fetchNextChunk();+ return items;+ }++ /**+ * Save the entirety of the dataset's contents into one file within a key-value store.+ *+ * @param key The name of the value to save the data in.+ * @param [options] An optional options object where you can provide the dataset and target KVS name.+ * @param [contentType] Only JSON and CSV are supported currently, defaults to JSON.+ */+ async exportTo(key: string, options?: DatasetExportToOptions, contentType?: string): Promise<Data[]> {+ const kvStore = await KeyValueStore.open(options?.toKVS ?? null, { config: this.config });+ const items = await this.export(options);+if (contentType === 'text/csv') {const value = stringify([Object.keys(items[0]),...items.map((item) => Object.values(item)),]);- return kvStore.setValue(key, value, { contentType });+ await kvStore.setValue(key, value, { contentType });+ return items;}if (contentType === 'application/json') {- return kvStore.setValue(key, items);+ await kvStore.setValue(key, items);+ return items;}throw new Error(`Unsupported content type: ${contentType}`);++ return items;}/**@@ -342,7 +358,7 @@ export class Dataset<Data extends Dictionary = Dictionary> {* @param key The name of the value to save the data in.* @param [options] An optional options object where you can provide the target KVS name.*/- async exportToJSON(key: string, options?: Omit<ExportOptions, 'fromDataset'>) {+ async exportToJSON(key: string, options?: Omit<DatasetExportToOptions, 'fromDataset'>) {await this.exportTo(key, options, 'application/json');}@@ -352,7 +368,7 @@ export class Dataset<Data extends Dictionary = Dictionary> {* @param key The name of the value to save the data in.* @param [options] An optional options object where you can provide the target KVS name.*/- async exportToCSV(key: string, options?: Omit<ExportOptions, 'fromDataset'>) {+ async exportToCSV(key: string, options?: Omit<DatasetExportToOptions, 'fromDataset'>) {await this.exportTo(key, options, 'text/csv');}@@ -362,7 +378,7 @@ export class Dataset<Data extends Dictionary = Dictionary> {* @param key The name of the value to save the data in.* @param [options] An optional options object where you can provide the dataset and target KVS name.*/- static async exportToJSON(key: string, options?: ExportOptions) {+ static async exportToJSON(key: string, options?: DatasetExportToOptions) {const dataset = await this.open(options?.fromDataset);await dataset.exportToJSON(key, options);}@@ -373,7 +389,7 @@ export class Dataset<Data extends Dictionary = Dictionary> {* @param key The name of the value to save the data in.* @param [options] An optional options object where you can provide the dataset and target KVS name.*/- static async exportToCSV(key: string, options?: ExportOptions) {+ static async exportToCSV(key: string, options?: DatasetExportToOptions) {const dataset = await this.open(options?.fromDataset);await dataset.exportToCSV(key, options);}diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.tsindex 6fffd0e9a..055b05b92 100644--- a/test/core/crawlers/basic_crawler.test.ts+++ b/test/core/crawlers/basic_crawler.test.ts@@ -1,6 +1,9 @@+import { mkdir, mkdtemp } from 'fs/promises';import type { Server } from 'http';import http from 'http';import type { AddressInfo } from 'net';+import { readFile, rm } from 'node:fs/promises';+import { join } from 'path';import log from '@apify/log';import type {@@ -1383,8 +1386,14 @@ describe('BasicCrawler', () => {});describe('Dataset helpers, crawler paralellism', () => {- const payload: Dictionary<any>[] = [{ foo: 'bar' }];- const getPayload: (id: string) => Dictionary<any>[] = (id) => [{ foo: id }];+ const payload: Dictionary[] = [{ foo: 'bar', baz: 123 }];+ const getPayload: (id: string) => Dictionary[] = (id) => [{ foo: id }];++ const tmpDir: string = `${__dirname}/tmp/foo/bar`;++ beforeAll(async () => {+ await rm(tmpDir, { recursive: true, force: true });+ });test('should expose default Dataset methods', async () => {const crawler = new BasicCrawler();@@ -1395,6 +1404,39 @@ describe('BasicCrawler', () => {.toEqual(payload);});+ test('export data', async () => {+ const row: Dictionary = { foo: 'bar', baz: 123 };+ const crawler = new BasicCrawler();++ await crawler.pushData(row);+ await crawler.pushData(row);+ await crawler.pushData(row);++ await crawler.exportData(`${tmpDir}/result.csv`);+ await crawler.exportData(`${tmpDir}/result.json`);++ const csv = await readFile(`${tmpDir}/result.csv`);+ expect(csv.toString()).toBe('foo,baz\nbar,123\nbar,123\nbar,123\n');+ const json = await readFile(`${tmpDir}/result.json`);+ expect(json.toString()).toBe('[\n'+ + ' {\n'+ + ' "foo": "bar",\n'+ + ' "baz": 123\n'+ + ' },\n'+ + ' {\n'+ + ' "foo": "bar",\n'+ + ' "baz": 123\n'+ + ' },\n'+ + ' {\n'+ + ' "foo": "bar",\n'+ + ' "baz": 123\n'+ + ' }\n'+ + ']\n');++ await rm(`${tmpDir}/result.csv`);+ await rm(`${tmpDir}/result.json`);+ });+test('should expose pushData helper', async () => {const crawler = new BasicCrawler({requestHandler: async ({ pushData }) => pushData(payload),