Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
add item count limit to enqueueLinks (#607)
Tobeyc3bd484745f9
diff --git a/jest.config.js b/jest.config.jsindex bcf0df8f5..e7cc34ed5 100644--- a/jest.config.js+++ b/jest.config.js@@ -3,7 +3,7 @@ const path = require('path');module.exports = {verbose: true,rootDir: path.join(__dirname, './'),- testMatch: [path.join(__dirname, '**/?(*.)+(spec|test).[tj]s?(x)')],+ testMatch: ['**/?(*.)+(spec|test).[tj]s?(x)'],transform: {'^.+\\.ts$': 'ts-jest','^.+\\.js$': 'babel-jest',diff --git a/src/enqueue_links/enqueue_links.js b/src/enqueue_links/enqueue_links.jsindex da1ed1c9c..253d73751 100644--- a/src/enqueue_links/enqueue_links.js+++ b/src/enqueue_links/enqueue_links.js@@ -52,6 +52,8 @@ import { RequestTransform } from './shared';* @param {Page} [options.page]* Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.* Either `page` or `$` option must be provided.+ * @param {Number} [options.limit]+ * Limit the count of actually enqueued URLs to this number. Useful for testing across the entire crawling scope.* @param {CheerioStatic} [options.$]* [`Cheerio`](https://github.com/cheeriojs/cheerio) function with loaded HTML.* Either `page` or `$` option must be provided.@@ -101,6 +103,7 @@ export async function enqueueLinks(options = {}) {const {page,$,+ limit,selector = 'a',requestQueue,baseUrl,@@ -121,6 +124,7 @@ export async function enqueueLinks(options = {}) {if (page && $) {throw new Error('Only one of the parameters "options.page" or "options.$" must be provided!');}+ checkParamOrThrow(limit, 'limit', 'Maybe Number');checkParamOrThrow(selector, 'selector', 'String');checkParamPrototypeOrThrow(requestQueue, 'requestQueue', [RequestQueue, RequestQueueLocal], 'Apify.RequestQueue');checkParamOrThrow(baseUrl, 'baseUrl', 'Maybe String');@@ -137,7 +141,9 @@ export async function enqueueLinks(options = {}) {if (transformRequestFunction) {requestOptions = requestOptions.map(transformRequestFunction).filter(r => !!r);}- const requests = createRequests(requestOptions, pseudoUrlInstances);+ let requests = createRequests(requestOptions, pseudoUrlInstances);+ if (limit) requests = requests.slice(0, limit);+return addRequestsToQueueInBatches(requests, requestQueue);}diff --git a/test/enqueue_links/enqueue_links.test.js b/test/enqueue_links/enqueue_links.test.jsindex 364cb7a3f..e5f9f4e93 100644--- a/test/enqueue_links/enqueue_links.test.js+++ b/test/enqueue_links/enqueue_links.test.js@@ -57,6 +57,32 @@ describe('enqueueLinks()', () => {browser = null;});+ test('works with item limit', async () => {+ const enqueued = [];+ const requestQueue = new RequestQueue('xxx');+ requestQueue.addRequest = async (request) => {+ enqueued.push(request);+ };++ await enqueueLinks({ page, limit: 3, selector: '.click', requestQueue });++ expect(enqueued).toHaveLength(3);++ expect(enqueued[0].url).toBe('https://example.com/a/b/first');+ expect(enqueued[0].method).toBe('GET');+ expect(enqueued[0].userData).toEqual({});++ expect(enqueued[1].url).toBe('https://example.com/a/b/third');+ expect(enqueued[1].method).toBe('GET');+ expect(enqueued[1].userData).toEqual({});++ expect(enqueued[2].url).toBe('https://another.com/a/fifth');+ expect(enqueued[2].method).toBe('GET');+ expect(enqueued[2].userData).toEqual({});++ expect(enqueued[3]).toBe(undefined);+ });+test('works with PseudoUrl instances', async () => {const enqueued = [];const requestQueue = new RequestQueue('xxx');