Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Lint comments
Ondra Urban1b29678a04cd
diff --git a/src/dataset.js b/src/dataset.jsindex f3d4f0e1c..fb470ed5b 100644--- a/src/dataset.js+++ b/src/dataset.js@@ -77,16 +77,16 @@ export const chunkBySize = (items, limitBytes) => {const bytes = Buffer.byteLength(payload);if (bytes <= limitBytes && (bytes + 2) > limitBytes) {- // handle cases where wrapping with [] would fail, but solo object is fine+ // Handle cases where wrapping with [] would fail, but solo object is fine.chunks.push(payload);lastChunkBytes = bytes;} else if (lastChunkBytes + bytes <= limitBytes) {if (!Array.isArray(_.last(chunks))) chunks.push([]); // ensure array_.last(chunks).push(payload);- lastChunkBytes += bytes + 1; // add 1 byte for ',' separator+ lastChunkBytes += bytes + 1; // Add 1 byte for ',' separator.} else {chunks.push([payload]);- lastChunkBytes = bytes + 2; // add 2 bytes for [] wrapper+ lastChunkBytes = bytes + 2; // Add 2 bytes for [] wrapper.}}diff --git a/test/dataset.js b/test/dataset.jsindex 3d6c94f8c..9e082ffbd 100644--- a/test/dataset.js+++ b/test/dataset.js@@ -748,21 +748,21 @@ describe('dataset', () => {describe('utils', async () => {it('checkAndSerialize() works', () => {- // basic+ // Basicconst obj = { foo: 'bar' };const json = JSON.stringify(obj);expect(checkAndSerialize({}, 100)).to.be.eql('{}');expect(checkAndSerialize(obj, 100)).to.be.eql(json);- // with index+ // With indexexpect(checkAndSerialize(obj, 100, 1)).to.be.eql(json);- // too large+ // Too largeexpect(() => checkAndSerialize(obj, 5)).to.throw(Error, 'Data item is too large');expect(() => checkAndSerialize(obj, 5, 7)).to.throw(Error, 'at index 7');- // bad JSON+ // Bad JSONconst bad = {};bad.bad = bad;expect(() => checkAndSerialize(bad, 100)).to.throw(Error, 'not serializable');- // bad data+ // Bad dataconst str = 'hello';expect(() => checkAndSerialize(str, 100)).to.throw(Error, 'not serializable');expect(() => checkAndSerialize([], 100)).to.throw(Error, 'not serializable');@@ -777,21 +777,21 @@ describe('dataset', () => {const chunk = `[${json}]`;const tripleChunk = `[${json},${json},${json}]`;const tripleSize = Buffer.byteLength(tripleChunk);- // empty array+ // Empty arrayexpect(chunkBySize([], 10)).to.be.eql([]);- // fits easily+ // Fits easilyexpect(chunkBySize([json], size + 10)).to.be.eql([json]);expect(chunkBySize(triple, tripleSize + 10)).to.be.eql([tripleChunk]);- // parses back to original objects+ // Parses back to original objectsexpect(originalTriple).to.be.eql(JSON.parse(tripleChunk));- // fits exactly+ // Fits exactlyexpect(chunkBySize([json], size)).to.be.eql([json]);expect(chunkBySize(triple, tripleSize)).to.be.eql([tripleChunk]);- // chunks large items individually+ // Chunks large items individuallyexpect(chunkBySize(triple, size)).to.be.eql(triple);expect(chunkBySize(triple, size + 1)).to.be.eql(triple);expect(chunkBySize(triple, size + 2)).to.be.eql([chunk, chunk, chunk]);- // chunks smaller items together+ // Chunks smaller items togetherexpect(chunkBySize(triple, (2 * size) + 3)).to.be.eql([`[${json},${json}]`, chunk]);expect(chunkBySize([...triple, ...triple], (2 * size) + 3)).to.be.eql([`[${json},${json}]`, `[${json},${json}]`, `[${json},${json}]`]);});