Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

feat: add `collectAllKeys` option for `BasicCrawler.exportData` (#3129)

Jindřich Bär2ddfc9c61082
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
index b418aab94..cd0a22bf8 100644
--- a/packages/basic-crawler/src/internals/basic-crawler.ts
+++ b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -1244,7 +1244,9 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
if (items.length === 0) {
value = '';
} else {
- const keys = Object.keys(items[0]);
+ const keys = options?.collectAllKeys
+ ? Array.from(new Set(items.flatMap(Object.keys)))
+ : Object.keys(items[0]);
value = stringify([
keys,
diff --git a/packages/core/src/storages/dataset.ts b/packages/core/src/storages/dataset.ts
index 676c59a20..dc27d09b3 100644
--- a/packages/core/src/storages/dataset.ts
+++ b/packages/core/src/storages/dataset.ts
@@ -141,7 +141,13 @@ export interface DatasetDataOptions {
skipEmpty?: boolean;
}
-export interface DatasetExportOptions extends Omit<DatasetDataOptions, 'offset' | 'limit'> {}
+export interface DatasetExportOptions extends Omit<DatasetDataOptions, 'offset' | 'limit'> {
+ /**
+ * If true, includes all unique keys from all dataset items in the CSV export header.
+ * If omitted or false, only keys from the first item are used.
+ */
+ collectAllKeys?: boolean;
+}
export interface DatasetIteratorOptions
extends Omit<DatasetDataOptions, 'offset' | 'limit' | 'clean' | 'skipHidden' | 'skipEmpty'> {
@@ -170,11 +176,6 @@ export interface DatasetIteratorOptions
export interface DatasetExportToOptions extends DatasetExportOptions {
fromDataset?: string;
toKVS?: string;
- /**
- * If true, includes all unique keys from all dataset items in the CSV export.
- * If omitted or false, only keys from the first item are used.
- */
- collectAllKeys?: boolean;
}
/**
diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.ts
index 4068cdc5f..bce9bbd9e 100644
--- a/test/core/crawlers/basic_crawler.test.ts
+++ b/test/core/crawlers/basic_crawler.test.ts
@@ -1745,6 +1745,19 @@ describe('BasicCrawler', () => {
expect((await crawler.getData()).items).toEqual(payload);
});
+ test('crawler.exportData works with `collectAllKeys`', async () => {
+ const crawler = new BasicCrawler();
+ await crawler.pushData([{ foo: 'bar', baz: 123 }]);
+ await crawler.pushData([{ foo: 'baz', qux: 456 }]);
+
+ await crawler.exportData(`${tmpDir}/result.csv`, 'csv', { collectAllKeys: true });
+
+ const csv = await readFile(`${tmpDir}/result.csv`);
+ expect(csv.toString()).toBe('foo,baz,qux\nbar,123,\nbaz,,456\n');
+
+ await rm(`${tmpDir}/result.csv`);
+ });
+
test("Crawlers with different Configurations don't share Datasets", async () => {
const crawlerA = new BasicCrawler({}, new Configuration({ persistStorage: false }));
const crawlerB = new BasicCrawler({}, new Configuration({ persistStorage: false }));