Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

added limit for iterators and error handling for too large input.

Petr Patek294db103c014
diff --git a/src/dataset.js b/src/dataset.js
index a339bd59a..80e55eed8 100644
--- a/src/dataset.js
+++ b/src/dataset.js
@@ -9,6 +9,7 @@ import { checkParamOrThrow } from 'apify-client/build/utils';
import { ENV_VARS, LOCAL_STORAGE_SUBDIRS, MAX_PAYLOAD_SIZE_BYTES } from 'apify-shared/consts';
import { apifyClient, ensureDirExists, openRemoteStorage, openLocalStorage, ensureTokenOrLocalStorageEnvExists } from './utils';
+export const DATASET_ITERATORS_DEFAULT_LIMIT = 250000;
export const LOCAL_STORAGE_SUBDIR = LOCAL_STORAGE_SUBDIRS.datasets;
export const LOCAL_FILENAME_DIGITS = 9;
export const LOCAL_GET_ITEMS_DEFAULT_LIMIT = 250000;
@@ -276,7 +277,16 @@ export class Dataset {
const { datasetId } = this;
const params = Object.assign({ datasetId }, options);
- return datasets.getItems(params);
+ try {
+ return await datasets.getItems(params);
+ } catch (e) {
+ if (e.message.includes('Cannot create a string longer than')) {
+ throw new Error(
+ 'getData: The response is too large for parsing. You can fix this by lowering the "limit" option.',
+ );
+ }
+ throw e;
+ }
}
/**
@@ -337,6 +347,7 @@ export class Dataset {
async forEach(iteratee, options = {}, index = 0) {
if (!options.offset) options.offset = 0;
if (options.format && options.format !== 'json') throw new Error('Dataset.forEach/map/reduce() support only a "json" format.');
+ if (!options.limit) options.limit = DATASET_ITERATORS_DEFAULT_LIMIT;
const { items, total, limit, offset } = await this.getData(options);
diff --git a/test/dataset.test.js b/test/dataset.test.js
index 3b28bc3e6..9ed2e5e43 100644
--- a/test/dataset.test.js
+++ b/test/dataset.test.js
@@ -498,6 +498,16 @@ describe('dataset', () => {
mock.verify();
mock.restore();
+ const stub = sinon.stub(apifyClient.datasets, 'getItems')
+ .callsFake(() => { throw new Error('Cannot create a string longer than 0x3fffffe7 characters'); });
+ let e;
+ try {
+ await dataset.getData();
+ } catch (err) {
+ e = err;
+ }
+ expect(e.message).toEqual('getData: The response is too large for parsing. You can fix this by lowering the "limit" option.');
+ stub.reset();
});
test('getInfo() should work', async () => {