Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Add unit tests for new functions; Handle edge cases
Ondra Urban1298780ebf63
diff --git a/src/dataset.js b/src/dataset.jsindex 2308cff89..f3d4f0e1c 100644--- a/src/dataset.js+++ b/src/dataset.js@@ -40,6 +40,7 @@ export const checkAndSerialize = (item, limitBytes, index) => {const s = typeof index === 'number' ? ` at index ${index} ` : ' ';let payload;try {+ checkParamOrThrow(item, 'item', 'Object');payload = JSON.stringify(item);} catch (err) {throw new Error(`Data item${s}is not serializable to JSON.\nCause: ${err.message}`);@@ -58,6 +59,8 @@ export const checkAndSerialize = (item, limitBytes, index) => {* JSON array. Fits as many payloads as possible into a single JSON array and then moves* on to the next, preserving item order.*+ * The function assumes that none of the items is larger than limitBytes and does not validate.+ ** @param {Array} items* @param {Number} limitBytes* @returns {Array}@@ -65,15 +68,20 @@ export const checkAndSerialize = (item, limitBytes, index) => {*/export const chunkBySize = (items, limitBytes) => {if (!items.length) return [];+ if (items.length === 1) return items;let lastChunkBytes = 2; // Add 2 bytes for [] wrapper.- const chunks = [[]]; // Prepopulate with an empty array.-+ const chunks = [];// Split payloads into buckets of valid size.for (const payload of items) { // eslint-disable-lineconst bytes = Buffer.byteLength(payload);- if (lastChunkBytes + bytes < limitBytes) {+ if (bytes <= limitBytes && (bytes + 2) > limitBytes) {+ // handle cases where wrapping with [] would fail, but solo object is fine+ chunks.push(payload);+ lastChunkBytes = bytes;+ } else if (lastChunkBytes + bytes <= limitBytes) {+ if (!Array.isArray(_.last(chunks))) chunks.push([]); // ensure array_.last(chunks).push(payload);lastChunkBytes += bytes + 1; // add 1 byte for ',' separator} else {@@ -82,8 +90,8 @@ export const chunkBySize = (items, limitBytes) => {}}- // Stringify each chunk.- return chunks.map(chunk => `[${chunk.join(',')}]`);+ // Stringify array chunks.+ return chunks.map(chunk => (typeof chunk === 'string' ? chunk : `[${chunk.join(',')}]`));};/**diff --git a/test/dataset.js b/test/dataset.jsindex 287bc2b7b..3d6c94f8c 100644--- a/test/dataset.js+++ b/test/dataset.js@@ -7,7 +7,7 @@ import sinon from 'sinon';import { leftpad, delayPromise } from 'apify-shared/utilities';import { ENV_VARS, MAX_PAYLOAD_SIZE_BYTES } from '../build/constants';import { LOCAL_FILENAME_DIGITS, Dataset, DatasetLocal, LOCAL_EMULATION_SUBDIR,- LOCAL_GET_ITEMS_DEFAULT_LIMIT } from '../build/dataset';+ LOCAL_GET_ITEMS_DEFAULT_LIMIT, checkAndSerialize, chunkBySize } from '../build/dataset';import { apifyClient } from '../build/utils';import * as Apify from '../build/index';import { LOCAL_EMULATION_DIR, emptyLocalEmulationSubdir, expectNotLocalEmulation, expectDirEmpty, expectDirNonEmpty } from './_helper';@@ -361,11 +361,11 @@ describe('dataset', () => {const dataset = new Dataset('some-id');try {await dataset.pushData([- 1,- 'two',- 3,+ { foo: 0 },+ { foo: 1 },+ { foo: 2 },{ foo: full },- ['six'],+ { foo: 4 },]);throw new Error('Should fail!');} catch (err) {@@ -745,4 +745,55 @@ describe('dataset', () => {delete process.env[ENV_VARS.LOCAL_EMULATION_DIR];});});++ describe('utils', async () => {+ it('checkAndSerialize() works', () => {+ // basic+ const obj = { foo: 'bar' };+ const json = JSON.stringify(obj);+ expect(checkAndSerialize({}, 100)).to.be.eql('{}');+ expect(checkAndSerialize(obj, 100)).to.be.eql(json);+ // with index+ expect(checkAndSerialize(obj, 100, 1)).to.be.eql(json);+ // too large+ expect(() => checkAndSerialize(obj, 5)).to.throw(Error, 'Data item is too large');+ expect(() => checkAndSerialize(obj, 5, 7)).to.throw(Error, 'at index 7');+ // bad JSON+ const bad = {};+ bad.bad = bad;+ expect(() => checkAndSerialize(bad, 100)).to.throw(Error, 'not serializable');+ // bad data+ const str = 'hello';+ expect(() => checkAndSerialize(str, 100)).to.throw(Error, 'not serializable');+ expect(() => checkAndSerialize([], 100)).to.throw(Error, 'not serializable');+ expect(() => checkAndSerialize([str, str], 100)).to.throw(Error, 'not serializable');+ });+ it('chunkBySize', () => {+ const obj = { foo: 'bar' };+ const json = JSON.stringify(obj);+ const size = Buffer.byteLength(json);+ const triple = [json, json, json];+ const originalTriple = [obj, obj, obj];+ const chunk = `[${json}]`;+ const tripleChunk = `[${json},${json},${json}]`;+ const tripleSize = Buffer.byteLength(tripleChunk);+ // empty array+ expect(chunkBySize([], 10)).to.be.eql([]);+ // fits easily+ expect(chunkBySize([json], size + 10)).to.be.eql([json]);+ expect(chunkBySize(triple, tripleSize + 10)).to.be.eql([tripleChunk]);+ // parses back to original objects+ expect(originalTriple).to.be.eql(JSON.parse(tripleChunk));+ // fits exactly+ expect(chunkBySize([json], size)).to.be.eql([json]);+ expect(chunkBySize(triple, tripleSize)).to.be.eql([tripleChunk]);+ // chunks large items individually+ expect(chunkBySize(triple, size)).to.be.eql(triple);+ expect(chunkBySize(triple, size + 1)).to.be.eql(triple);+ expect(chunkBySize(triple, size + 2)).to.be.eql([chunk, chunk, chunk]);+ // chunks smaller items together+ expect(chunkBySize(triple, (2 * size) + 3)).to.be.eql([`[${json},${json}]`, chunk]);+ expect(chunkBySize([...triple, ...triple], (2 * size) + 3)).to.be.eql([`[${json},${json}]`, `[${json},${json}]`, `[${json},${json}]`]);+ });+ });});