Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

add item count limit to enqueueLinks (#607)

Tobeyc3bd484745f9
diff --git a/jest.config.js b/jest.config.js
index bcf0df8f5..e7cc34ed5 100644
--- a/jest.config.js
+++ b/jest.config.js
@@ -3,7 +3,7 @@ const path = require('path');
module.exports = {
verbose: true,
rootDir: path.join(__dirname, './'),
- testMatch: [path.join(__dirname, '**/?(*.)+(spec|test).[tj]s?(x)')],
+ testMatch: ['**/?(*.)+(spec|test).[tj]s?(x)'],
transform: {
'^.+\\.ts$': 'ts-jest',
'^.+\\.js$': 'babel-jest',
diff --git a/src/enqueue_links/enqueue_links.js b/src/enqueue_links/enqueue_links.js
index da1ed1c9c..253d73751 100644
--- a/src/enqueue_links/enqueue_links.js
+++ b/src/enqueue_links/enqueue_links.js
@@ -52,6 +52,8 @@ import { RequestTransform } from './shared';
* @param {Page} [options.page]
* Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
* Either `page` or `$` option must be provided.
+ * @param {Number} [options.limit]
+ * Limit the count of actually enqueued URLs to this number. Useful for testing across the entire crawling scope.
* @param {CheerioStatic} [options.$]
* [`Cheerio`](https://github.com/cheeriojs/cheerio) function with loaded HTML.
* Either `page` or `$` option must be provided.
@@ -101,6 +103,7 @@ export async function enqueueLinks(options = {}) {
const {
page,
$,
+ limit,
selector = 'a',
requestQueue,
baseUrl,
@@ -121,6 +124,7 @@ export async function enqueueLinks(options = {}) {
if (page && $) {
throw new Error('Only one of the parameters "options.page" or "options.$" must be provided!');
}
+ checkParamOrThrow(limit, 'limit', 'Maybe Number');
checkParamOrThrow(selector, 'selector', 'String');
checkParamPrototypeOrThrow(requestQueue, 'requestQueue', [RequestQueue, RequestQueueLocal], 'Apify.RequestQueue');
checkParamOrThrow(baseUrl, 'baseUrl', 'Maybe String');
@@ -137,7 +141,9 @@ export async function enqueueLinks(options = {}) {
if (transformRequestFunction) {
requestOptions = requestOptions.map(transformRequestFunction).filter(r => !!r);
}
- const requests = createRequests(requestOptions, pseudoUrlInstances);
+ let requests = createRequests(requestOptions, pseudoUrlInstances);
+ if (limit) requests = requests.slice(0, limit);
+
return addRequestsToQueueInBatches(requests, requestQueue);
}
diff --git a/test/enqueue_links/enqueue_links.test.js b/test/enqueue_links/enqueue_links.test.js
index 364cb7a3f..e5f9f4e93 100644
--- a/test/enqueue_links/enqueue_links.test.js
+++ b/test/enqueue_links/enqueue_links.test.js
@@ -57,6 +57,32 @@ describe('enqueueLinks()', () => {
browser = null;
});
+ test('works with item limit', async () => {
+ const enqueued = [];
+ const requestQueue = new RequestQueue('xxx');
+ requestQueue.addRequest = async (request) => {
+ enqueued.push(request);
+ };
+
+ await enqueueLinks({ page, limit: 3, selector: '.click', requestQueue });
+
+ expect(enqueued).toHaveLength(3);
+
+ expect(enqueued[0].url).toBe('https://example.com/a/b/first');
+ expect(enqueued[0].method).toBe('GET');
+ expect(enqueued[0].userData).toEqual({});
+
+ expect(enqueued[1].url).toBe('https://example.com/a/b/third');
+ expect(enqueued[1].method).toBe('GET');
+ expect(enqueued[1].userData).toEqual({});
+
+ expect(enqueued[2].url).toBe('https://another.com/a/fifth');
+ expect(enqueued[2].method).toBe('GET');
+ expect(enqueued[2].userData).toEqual({});
+
+ expect(enqueued[3]).toBe(undefined);
+ });
+
test('works with PseudoUrl instances', async () => {
const enqueued = [];
const requestQueue = new RequestQueue('xxx');