Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Remove limit and offset from dataset iterator docs [skip-ci]

Ondra Urbanf7fea143bc0f
diff --git a/docs/api/Dataset.md b/docs/api/Dataset.md
index 31d5f7294..33ed8c46f 100644
--- a/docs/api/Dataset.md
+++ b/docs/api/Dataset.md
@@ -285,11 +285,6 @@ await dataset.forEach(async (item, index) => {
<td colspan="3"><p>All <code>forEach()</code> parameters are passed
via an options object with the following keys:</p>
</td></tr><tr>
-<td><code>[options.offset]</code></td><td><code>Number</code></td><td><code>0</code></td>
-</tr>
-<tr>
-<td colspan="3"><p>Number of array elements that should be skipped at the start.</p>
-</td></tr><tr>
<td><code>[options.desc]</code></td><td><code>Boolean</code></td><td><code>false</code></td>
</tr>
<tr>
@@ -305,11 +300,6 @@ await dataset.forEach(async (item, index) => {
<tr>
<td colspan="3"><p>If provided then objects will be unwound based on provided field.</p>
</td></tr><tr>
-<td><code>[options.limit]</code></td><td><code>Number</code></td><td><code>250000</code></td>
-</tr>
-<tr>
-<td colspan="3"><p>How many items to load in one request.</p>
-</td></tr><tr>
<td><code>[index]</code></td><td><code>Number</code></td><td><code>0</code></td>
</tr>
<tr>
@@ -343,11 +333,6 @@ If `iteratee` returns a `Promise` then it's awaited before a next call.
<td colspan="3"><p>All <code>map()</code> parameters are passed
via an options object with the following keys:</p>
</td></tr><tr>
-<td><code>[options.offset]</code></td><td><code>Number</code></td><td><code>0</code></td>
-</tr>
-<tr>
-<td colspan="3"><p>Number of array elements that should be skipped at the start.</p>
-</td></tr><tr>
<td><code>[options.desc]</code></td><td><code>Boolean</code></td><td><code>false</code></td>
</tr>
<tr>
@@ -362,11 +347,6 @@ If `iteratee` returns a `Promise` then it's awaited before a next call.
</tr>
<tr>
<td colspan="3"><p>If provided then objects will be unwound based on provided field.</p>
-</td></tr><tr>
-<td><code>[options.limit]</code></td><td><code>Number</code></td><td><code>250000</code></td>
-</tr>
-<tr>
-<td colspan="3"><p>How many items to load in one request.</p>
</td></tr></tbody>
</table>
<a name="Dataset+reduce"></a>
@@ -406,11 +386,6 @@ If `iteratee()` returns a `Promise` then it's awaited before a next call.
<td colspan="3"><p>All <code>reduce()</code> parameters are passed
via an options object with the following keys:</p>
</td></tr><tr>
-<td><code>[options.offset]</code></td><td><code>Number</code></td><td><code>0</code></td>
-</tr>
-<tr>
-<td colspan="3"><p>Number of array elements that should be skipped at the start.</p>
-</td></tr><tr>
<td><code>[options.desc]</code></td><td><code>Boolean</code></td><td><code>false</code></td>
</tr>
<tr>
@@ -425,11 +400,6 @@ If `iteratee()` returns a `Promise` then it's awaited before a next call.
</tr>
<tr>
<td colspan="3"><p>If provided then objects will be unwound based on provided field.</p>
-</td></tr><tr>
-<td><code>[options.limit]</code></td><td><code>Number</code></td><td><code>250000</code></td>
-</tr>
-<tr>
-<td colspan="3"><p>How many items to load in one request.</p>
</td></tr></tbody>
</table>
<a name="Dataset+drop"></a>
diff --git a/src/dataset.js b/src/dataset.js
index 37cc5e79b..25d9d7ce8 100644
--- a/src/dataset.js
+++ b/src/dataset.js
@@ -9,7 +9,7 @@ import { checkParamOrThrow } from 'apify-client/build/utils';
import { ENV_VARS, LOCAL_STORAGE_SUBDIRS, MAX_PAYLOAD_SIZE_BYTES } from 'apify-shared/consts';
import { apifyClient, ensureDirExists, openRemoteStorage, openLocalStorage, ensureTokenOrLocalStorageEnvExists } from './utils';
-export const DATASET_ITERATORS_DEFAULT_LIMIT = 250000;
+export const DATASET_ITERATORS_DEFAULT_LIMIT = 10000;
export const LOCAL_STORAGE_SUBDIR = LOCAL_STORAGE_SUBDIRS.datasets;
export const LOCAL_FILENAME_DIGITS = 9;
export const LOCAL_GET_ITEMS_DEFAULT_LIMIT = 250000;
@@ -336,11 +336,9 @@ export class Dataset {
* @param {Function} iteratee A function that is called for every item in the dataset.
* @param {Object} [options] All `forEach()` parameters are passed
* via an options object with the following keys:
- * @param {Number} [options.offset=0] Number of array elements that should be skipped at the start.
* @param {Boolean} [options.desc=false] If `true` then the objects are sorted by `createdAt` in descending order.
* @param {Array} [options.fields] If provided then returned objects will only contain specified keys.
* @param {String} [options.unwind] If provided then objects will be unwound based on provided field.
- * @param {Number} [options.limit=250000] How many items to load in one request.
* @param {Number} [index=0] Specifies the initial index number passed to the `iteratee` function.
* @return {Promise}
*/
@@ -373,11 +371,9 @@ export class Dataset {
* @param {Function} iteratee
* @param {Object} options All `map()` parameters are passed
* via an options object with the following keys:
- * @param {Number} [options.offset=0] Number of array elements that should be skipped at the start.
* @param {Boolean} [options.desc=false] If `true` then the objects are sorted by createdAt in descending order.
* @param {Array} [options.fields] If provided then returned objects will only contain specified keys
* @param {String} [options.unwind] If provided then objects will be unwound based on provided field.
- * @param {Number} [options.limit=250000] How many items to load in one request.
* @return {Promise<Array>}
*/
map(iteratee, options) {
@@ -410,11 +406,9 @@ export class Dataset {
* @param {*} memo Initial state of the reduction.
* @param {Object} options All `reduce()` parameters are passed
* via an options object with the following keys:
- * @param {Number} [options.offset=0] Number of array elements that should be skipped at the start.
* @param {Boolean} [options.desc=false] If `true` then the objects are sorted by createdAt in descending order.
* @param {Array} [options.fields] If provided then returned objects will only contain specified keys
* @param {String} [options.unwind] If provided then objects will be unwound based on provided field.
- * @param {Number} [options.limit=250000] How many items to load in one request.
* @return {Promise<*>}
*/
reduce(iteratee, memo, options) {