Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Implement PR comments

Ondra Urban8c1fe72f6f2f
diff --git a/src/dataset.js b/src/dataset.js
index 41c63602a..a2b41319e 100644
--- a/src/dataset.js
+++ b/src/dataset.js
@@ -6,14 +6,13 @@ import Promise from 'bluebird';
import { leftpad } from 'apify-shared/utilities';
import LruCache from 'apify-shared/lru_cache';
import { checkParamOrThrow } from 'apify-client/build/utils';
-import { ENV_VARS, LOCAL_EMULATION_SUBDIRS } from './constants';
+import { ENV_VARS, LOCAL_EMULATION_SUBDIRS, MAX_PAYLOAD_SIZE_BYTES } from './constants';
import { apifyClient, ensureDirExists } from './utils';
export const LOCAL_EMULATION_SUBDIR = LOCAL_EMULATION_SUBDIRS.datasets;
export const LOCAL_FILENAME_DIGITS = 9;
export const LOCAL_GET_ITEMS_DEFAULT_LIMIT = 250000;
const MAX_OPENED_STORES = 1000;
-export const MAX_PAYLOAD_SIZE_BYTES = 9437184; // 9MB
const writeFilePromised = Promise.promisify(fs.writeFile);
const readFilePromised = Promise.promisify(fs.readFile);
@@ -62,19 +61,21 @@ export class Dataset {
*/
pushData(data) {
checkParamOrThrow(data, 'data', 'Array | Object');
+ const payloadLimitBytes = MAX_PAYLOAD_SIZE_BYTES - 32; // create some leeway
+
const dispatch = payload => datasets.putItems({ datasetId: this.datasetId, data: payload });
- const toJSONArray = array => `[${array.join(',')}]`;
+ const toJsonArray = array => `[${array.join(',')}]`;
const checkAndSerialize = (item, index) => {
const payload = JSON.stringify(item);
const bytes = Buffer.byteLength(payload);
- if (bytes > MAX_PAYLOAD_SIZE_BYTES) {
- const s = index ? ` at index ${index} ` : ' ';
- throw new Error(`Pushed data${s}too large! Actual: ${bytes} bytes. Limit: ${MAX_PAYLOAD_SIZE_BYTES}`);
+ if (bytes > payloadLimitBytes) {
+ const s = typeof index === 'number' ? ` at index ${index} ` : ' ';
+ throw new Error(`Data item${s}is too large (size: ${bytes} bytes, limit: ${MAX_PAYLOAD_SIZE_BYTES} bytes)`);
}
return { bytes, payload };
};
- // handle singular Objects
+ // Handle singular Objects
if (!Array.isArray(data)) {
try {
const { payload } = checkAndSerialize(data);
@@ -84,7 +85,7 @@ export class Dataset {
}
}
- // handle Arrays
+ // Handle Arrays
let items;
let totalBytes = 0;
try {
@@ -97,32 +98,34 @@ export class Dataset {
return Promise.reject(err);
}
- // send immediately if not over limit
- if (totalBytes < MAX_PAYLOAD_SIZE_BYTES) return dispatch(toJSONArray(items.map(i => i.payload)));
+ // Send immediately if not over limit
+ if (totalBytes < payloadLimitBytes) return dispatch(toJsonArray(items.map(i => i.payload)));
- // reducer object
- const r = { lastChunkBytes: 0, chunks: [] };
- // split payloads into buckets of valid size
+ let lastChunkBytes = 0;
+ const chunks = [];
+ // Split payloads into buckets of valid size
for (const { bytes, payload } of items) { // eslint-disable-line
- const last = r.chunks.length - 1;
- if (r.lastChunkBytes + bytes < MAX_PAYLOAD_SIZE_BYTES) {
- if (!Array.isArray(r.chunks[last])) r.chunks.push([payload]);
- else r.chunks[last].push(payload);
- r.lastChunkBytes += bytes;
+ // Ensure last item is always an array
+ if (!Array.isArray(_.last(chunks))) chunks.push([]);
+ if (lastChunkBytes + bytes < payloadLimitBytes) {
+ // If the item fits in the last chunk, push it there.
+ _.last(chunks).push(payload);
+ lastChunkBytes += bytes + 1; // add 1 byte for ',' separator
} else {
- const chunk = r.chunks.pop();
- r.chunks.push(toJSONArray(chunk));
- r.chunks.push([payload]);
- r.lastChunkBytes = bytes;
+ // Otherwise, the last chunk is full. Serialize it and create a new chunk.
+ const chunk = chunks.pop();
+ chunks.push(toJsonArray(chunk));
+ chunks.push([payload]);
+ lastChunkBytes = bytes + 2; // add 2 bytes for [] wrapper
}
}
- // stringify last chunk
- const lastChunk = r.chunks.pop();
- r.chunks.push(toJSONArray(lastChunk));
+ // Stringify last chunk
+ const lastChunk = chunks.pop();
+ chunks.push(toJsonArray(lastChunk));
- // invoke client in series to preserve order of data
- return Promise.mapSeries(r.chunks, chunk => dispatch(chunk));
+ // Invoke client in series to preserve order of data
+ return Promise.mapSeries(chunks, chunk => dispatch(chunk));
}
/**
diff --git a/test/dataset.js b/test/dataset.js
index bd0e90690..54dfb12b4 100644
--- a/test/dataset.js
+++ b/test/dataset.js
@@ -5,9 +5,9 @@ import fs from 'fs-extra';
import path from 'path';
import sinon from 'sinon';
import { leftpad, delayPromise } from 'apify-shared/utilities';
-import { ENV_VARS } from '../build/constants';
+import { ENV_VARS, MAX_PAYLOAD_SIZE_BYTES } from '../build/constants';
import { LOCAL_FILENAME_DIGITS, Dataset, DatasetLocal, LOCAL_EMULATION_SUBDIR,
- LOCAL_GET_ITEMS_DEFAULT_LIMIT, MAX_PAYLOAD_SIZE_BYTES } from '../build/dataset';
+ LOCAL_GET_ITEMS_DEFAULT_LIMIT } from '../build/dataset';
import { apifyClient } from '../build/utils';
import * as Apify from '../build/index';
import { LOCAL_EMULATION_DIR, emptyLocalEmulationSubdir, expectNotLocalEmulation, expectDirEmpty, expectDirNonEmpty } from './_helper';
@@ -344,7 +344,7 @@ describe('dataset', () => {
throw new Error('Should fail!');
} catch (err) {
expect(err).to.be.an('error');
- expect(err.message).to.include('Pushed data too large!');
+ expect(err.message).to.include('Data item is too large');
}
});
it('should throw on too large file in an array', async () => {
@@ -361,7 +361,7 @@ describe('dataset', () => {
throw new Error('Should fail!');
} catch (err) {
expect(err).to.be.an('error');
- expect(err.message).to.include('Pushed data at index 3 too large!');
+ expect(err.message).to.include('Data item at index 3 is too large');
}
});