Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: don't fail `exportData` calls on empty datasets (#3115)
MirzaSamadAhmedBaig298f170ef032
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex 2e7225370..b418aab94 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -1240,7 +1240,20 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextconst items = await dataset.export(options);if (format === 'csv') {- const value = stringify([Object.keys(items[0]), ...items.map((item) => Object.values(item))]);+ let value: string;+ if (items.length === 0) {+ value = '';+ } else {+ const keys = Object.keys(items[0]);++ value = stringify([+ keys,+ ...items.map((item) => {+ return keys.map((k) => item[k]);+ }),+ ]);+ }+await ensureDir(dirname(path));await writeFile(path, value);this.log.info(`Export to ${path} finished!`);diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.tsindex 1fc2e3808..4068cdc5f 100644--- a/test/core/crawlers/basic_crawler.test.ts+++ b/test/core/crawlers/basic_crawler.test.ts@@ -1716,6 +1716,21 @@ describe('BasicCrawler', () => {await rm(`${tmpDir}/result.json`);});+ test('exports do not fail on empty dataset', async () => {+ const crawler = new BasicCrawler();++ await crawler.exportData(`${tmpDir}/result.csv`);+ await crawler.exportData(`${tmpDir}/result.json`);++ const csv = await readFile(`${tmpDir}/result.csv`);+ expect(csv.toString()).toBe('');+ const json = await readFile(`${tmpDir}/result.json`);+ expect(json.toString()).toBe('[]\n');++ await rm(`${tmpDir}/result.csv`);+ await rm(`${tmpDir}/result.json`);+ });+test('should expose pushData helper', async () => {const crawler = new BasicCrawler({requestHandler: async ({ pushData }) => pushData(payload),