Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat: add `collectAllKeys` option for `BasicCrawler.exportData` (#3129)
Jindřich Bär2ddfc9c61082
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex b418aab94..cd0a22bf8 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -1244,7 +1244,9 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextif (items.length === 0) {value = '';} else {- const keys = Object.keys(items[0]);+ const keys = options?.collectAllKeys+ ? Array.from(new Set(items.flatMap(Object.keys)))+ : Object.keys(items[0]);value = stringify([keys,diff --git a/packages/core/src/storages/dataset.ts b/packages/core/src/storages/dataset.tsindex 676c59a20..dc27d09b3 100644--- a/packages/core/src/storages/dataset.ts+++ b/packages/core/src/storages/dataset.ts@@ -141,7 +141,13 @@ export interface DatasetDataOptions {skipEmpty?: boolean;}-export interface DatasetExportOptions extends Omit<DatasetDataOptions, 'offset' | 'limit'> {}+export interface DatasetExportOptions extends Omit<DatasetDataOptions, 'offset' | 'limit'> {+ /**+ * If true, includes all unique keys from all dataset items in the CSV export header.+ * If omitted or false, only keys from the first item are used.+ */+ collectAllKeys?: boolean;+}export interface DatasetIteratorOptionsextends Omit<DatasetDataOptions, 'offset' | 'limit' | 'clean' | 'skipHidden' | 'skipEmpty'> {@@ -170,11 +176,6 @@ export interface DatasetIteratorOptionsexport interface DatasetExportToOptions extends DatasetExportOptions {fromDataset?: string;toKVS?: string;- /**- * If true, includes all unique keys from all dataset items in the CSV export.- * If omitted or false, only keys from the first item are used.- */- collectAllKeys?: boolean;}/**diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.tsindex 4068cdc5f..bce9bbd9e 100644--- a/test/core/crawlers/basic_crawler.test.ts+++ b/test/core/crawlers/basic_crawler.test.ts@@ -1745,6 +1745,19 @@ describe('BasicCrawler', () => {expect((await crawler.getData()).items).toEqual(payload);});+ test('crawler.exportData works with `collectAllKeys`', async () => {+ const crawler = new BasicCrawler();+ await crawler.pushData([{ foo: 'bar', baz: 123 }]);+ await crawler.pushData([{ foo: 'baz', qux: 456 }]);++ await crawler.exportData(`${tmpDir}/result.csv`, 'csv', { collectAllKeys: true });++ const csv = await readFile(`${tmpDir}/result.csv`);+ expect(csv.toString()).toBe('foo,baz,qux\nbar,123,\nbaz,,456\n');++ await rm(`${tmpDir}/result.csv`);+ });+test("Crawlers with different Configurations don't share Datasets", async () => {const crawlerA = new BasicCrawler({}, new Configuration({ persistStorage: false }));const crawlerB = new BasicCrawler({}, new Configuration({ persistStorage: false }));