Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Make object validation less strict in pushData (#917)

Ondra Urban7aee31d9050e
diff --git a/src/storages/dataset.js b/src/storages/dataset.js
index e14541ca1..9b6c1b2e3 100644
--- a/src/storages/dataset.js
+++ b/src/storages/dataset.js
@@ -21,9 +21,14 @@ const SAFETY_BUFFER_PERCENT = 0.01 / 100; // 0.01%
*/
export const checkAndSerialize = (item, limitBytes, index) => {
const s = typeof index === 'number' ? ` at index ${index} ` : ' ';
+
+ const isItemObject = item && typeof item === 'object' && !Array.isArray(item);
+ if (!isItemObject) {
+ throw new Error(`Data item${s}is not an object. You can push only objects into a dataset.`);
+ }
+
let payload;
try {
- ow(item, ow.object.plain);
payload = JSON.stringify(item);
} catch (err) {
throw new Error(`Data item${s}is not serializable to JSON.\nCause: ${err.message}`);
diff --git a/test/storages/dataset.test.js b/test/storages/dataset.test.js
index 76360125a..b2ad77538 100644
--- a/test/storages/dataset.test.js
+++ b/test/storages/dataset.test.js
@@ -447,8 +447,7 @@ describe('dataset', () => {
await expect(dataset.pushData(123)).rejects.toThrow('Expected argument to be of type `object` but received type `number`');
await expect(dataset.pushData(true)).rejects.toThrow('Expected argument to be of type `object` but received type `boolean`');
await expect(dataset.pushData(false)).rejects.toThrow('Expected argument to be of type `object` but received type `boolean`');
- await expect(dataset.pushData(() => {})).rejects.toThrow('Data item is not serializable to JSON.\n'
- + 'Cause: Expected object `item` to be a plain object');
+ await expect(dataset.pushData(() => {})).rejects.toThrow('Data item is not an object. You can push only objects into a dataset.');
const circularObj = {};
circularObj.xxx = circularObj;