Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Make object validation less strict in pushData (#917)
Ondra Urban7aee31d9050e
diff --git a/src/storages/dataset.js b/src/storages/dataset.jsindex e14541ca1..9b6c1b2e3 100644--- a/src/storages/dataset.js+++ b/src/storages/dataset.js@@ -21,9 +21,14 @@ const SAFETY_BUFFER_PERCENT = 0.01 / 100; // 0.01%*/export const checkAndSerialize = (item, limitBytes, index) => {const s = typeof index === 'number' ? ` at index ${index} ` : ' ';++ const isItemObject = item && typeof item === 'object' && !Array.isArray(item);+ if (!isItemObject) {+ throw new Error(`Data item${s}is not an object. You can push only objects into a dataset.`);+ }+let payload;try {- ow(item, ow.object.plain);payload = JSON.stringify(item);} catch (err) {throw new Error(`Data item${s}is not serializable to JSON.\nCause: ${err.message}`);diff --git a/test/storages/dataset.test.js b/test/storages/dataset.test.jsindex 76360125a..b2ad77538 100644--- a/test/storages/dataset.test.js+++ b/test/storages/dataset.test.js@@ -447,8 +447,7 @@ describe('dataset', () => {await expect(dataset.pushData(123)).rejects.toThrow('Expected argument to be of type `object` but received type `number`');await expect(dataset.pushData(true)).rejects.toThrow('Expected argument to be of type `object` but received type `boolean`');await expect(dataset.pushData(false)).rejects.toThrow('Expected argument to be of type `object` but received type `boolean`');- await expect(dataset.pushData(() => {})).rejects.toThrow('Data item is not serializable to JSON.\n'- + 'Cause: Expected object `item` to be a plain object');+ await expect(dataset.pushData(() => {})).rejects.toThrow('Data item is not an object. You can push only objects into a dataset.');const circularObj = {};circularObj.xxx = circularObj;