Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: Do not enqueue more links than what the crawler is capable of processing (#2990)

Jan Bucharea094c819232
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
index 1d84656b4..e6a79f6f5 100644
--- a/packages/basic-crawler/src/internals/basic-crawler.ts
+++ b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -516,7 +516,8 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
protected sameDomainDelayMillis: number;
protected domainAccessedTime: Map<string, number>;
protected maxSessionRotations: number;
- protected handledRequestsCount: number;
+ protected maxRequestsPerCrawl?: number;
+ protected handledRequestsCount = 0;
protected statusMessageLoggingInterval: number;
protected statusMessageCallback?: StatusMessageCallback;
protected sessionPoolOptions: SessionPoolOptions;
@@ -529,7 +530,8 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
protected respectRobotsTxtFile: boolean;
protected onSkippedRequest?: SkippedRequestCallback;
private _closeEvents?: boolean;
-
+ private shouldLogMaxProcessedRequestsExceeded = true;
+ private shouldLogMaxEnqueuedRequestsExceeded = true;
private experiments: CrawlerExperiments;
private readonly robotsTxtFileCache: LruCache<RobotsTxtFile>;
private _experimentWarnings: Partial<Record<keyof CrawlerExperiments, boolean>> = {};
@@ -642,6 +644,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
this.domainAccessedTime = new Map();
this.experiments = experiments;
this.robotsTxtFileCache = new LruCache({ maxLength: 1000 });
+ this.handleSkippedRequest = this.handleSkippedRequest.bind(this);
this._handlePropertyNameChange({
newName: 'requestHandler',
@@ -707,7 +710,6 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
this.maxRequestRetries = maxRequestRetries;
this.sameDomainDelayMillis = sameDomainDelaySecs * 1000;
this.maxSessionRotations = maxSessionRotations;
- this.handledRequestsCount = 0;
this.stats = new Statistics({
logMessage: `${log.getOptions().prefix} request statistics:`,
log,
@@ -741,8 +743,10 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
this.internalTimeoutMillis = Math.min(this.internalTimeoutMillis, maxSignedInteger);
- let shouldLogMaxPagesExceeded = true;
- const isMaxPagesExceeded = () => maxRequestsPerCrawl && maxRequestsPerCrawl <= this.handledRequestsCount;
+ this.maxRequestsPerCrawl = maxRequestsPerCrawl;
+
+ const isMaxPagesExceeded = () =>
+ this.maxRequestsPerCrawl && this.maxRequestsPerCrawl <= this.handledRequestsCount;
// eslint-disable-next-line prefer-const
let { isFinishedFunction, isTaskReadyFunction } = autoscaledPoolOptions;
@@ -759,12 +763,12 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
runTaskFunction: this._runTaskFunction.bind(this),
isTaskReadyFunction: async () => {
if (isMaxPagesExceeded()) {
- if (shouldLogMaxPagesExceeded) {
+ if (this.shouldLogMaxProcessedRequestsExceeded) {
log.info(
'Crawler reached the maxRequestsPerCrawl limit of ' +
- `${maxRequestsPerCrawl} requests and will shut down soon. Requests that are in progress will be allowed to finish.`,
+ `${this.maxRequestsPerCrawl} requests and will shut down soon. Requests that are in progress will be allowed to finish.`,
);
- shouldLogMaxPagesExceeded = false;
+ this.shouldLogMaxProcessedRequestsExceeded = false;
}
return false;
}
@@ -774,7 +778,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
isFinishedFunction: async () => {
if (isMaxPagesExceeded()) {
log.info(
- `Earlier, the crawler reached the maxRequestsPerCrawl limit of ${maxRequestsPerCrawl} requests ` +
+ `Earlier, the crawler reached the maxRequestsPerCrawl limit of ${this.maxRequestsPerCrawl} requests ` +
'and all requests that were in progress at that time have now finished. ' +
`In total, the crawler processed ${this.handledRequestsCount} requests and will shut down.`,
);
@@ -916,6 +920,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
if (this.requestQueue?.name === 'default' && purgeRequestQueue) {
await this.requestQueue.drop();
this.requestQueue = await this._getRequestQueue();
+ this.handledRequestsCount = 0; // This would've been reset by this._init() further down below, but at that point `handledRequestsCount` could prevent `addRequests` from adding the initial requests
}
this.stats.reset();
@@ -924,6 +929,8 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
}
this.running = true;
+ this.shouldLogMaxProcessedRequestsExceeded = true;
+ this.shouldLogMaxEnqueuedRequestsExceeded = true;
await purgeDefaultStorages({ onlyPurgeOnce: true });
@@ -1043,6 +1050,44 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
return kvs.getAutoSavedValue<State>(BasicCrawler.CRAWLEE_STATE_KEY, defaultValue);
}
+ protected get pendingRequestCountApproximation(): number {
+ let result = 0;
+
+ if (this.requestQueue) {
+ result += this.requestQueue.getPendingCount();
+ }
+ if (this.requestList) {
+ result += this.requestList.length() - this.requestList.handledCount();
+ }
+
+ return result;
+ }
+
+ protected calculateEnqueuedRequestLimit(explicitLimit?: number): number | undefined {
+ if (this.maxRequestsPerCrawl === undefined) {
+ return explicitLimit;
+ }
+
+ const limit = Math.max(
+ 0,
+ this.maxRequestsPerCrawl - this.handledRequestsCount - this.pendingRequestCountApproximation,
+ );
+
+ return Math.min(limit, explicitLimit ?? Infinity);
+ }
+
+ protected async handleSkippedRequest(options: Parameters<SkippedRequestCallback>[0]): Promise<void> {
+ if (options.reason === 'limit' && this.shouldLogMaxEnqueuedRequestsExceeded) {
+ this.log.info(
+ 'The number of requests enqueued by the crawler reached the maxRequestsPerCrawl limit of ' +
+ `${this.maxRequestsPerCrawl} requests and no further requests will be added.`,
+ );
+ this.shouldLogMaxEnqueuedRequestsExceeded = false;
+ }
+
+ await this.onSkippedRequest?.(options);
+ }
+
/**
* Adds requests to the queue in batches. By default, it will resolve after the initial batch is added, and continue
* adding the rest in background. You can configure the batch size via `batchSize` option and the sleep time in between
@@ -1060,36 +1105,50 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
): Promise<CrawlerAddRequestsResult> {
const requestQueue = await this.getRequestQueue();
- if (!this.respectRobotsTxtFile) {
- return requestQueue.addRequestsBatched(requests, options);
- }
+ const requestLimit = this.calculateEnqueuedRequestLimit();
const allowedRequests: (string | Source)[] = [];
- const skipped = new Set<string>();
+ const skippedBecauseOfRobots = new Set<string>();
+ const skippedBecauseOfLimit = new Set<string>();
for (const request of requests) {
const url = typeof request === 'string' ? request : request.url!;
+ if (requestLimit !== undefined && allowedRequests.length >= requestLimit) {
+ skippedBecauseOfLimit.add(url);
+ continue;
+ }
+
if (await this.isAllowedBasedOnRobotsTxtFile(url)) {
allowedRequests.push(request);
} else {
- skipped.add(url);
- await this.onSkippedRequest?.({ url, reason: 'robotsTxt' });
+ skippedBecauseOfRobots.add(url);
+ await this.handleSkippedRequest({ url, reason: 'robotsTxt' });
}
}
- if (skipped.size > 0) {
+ if (skippedBecauseOfRobots.size > 0) {
this.log.warning(`Some requests were skipped because they were disallowed based on the robots.txt file`, {
- skipped: [...skipped],
+ skipped: [...skippedBecauseOfRobots],
});
+ }
- if (this.onSkippedRequest) {
- await Promise.all(
- [...skipped].map((url) => {
- return this.onSkippedRequest!({ url, reason: 'robotsTxt' });
- }),
- );
- }
+ if (skippedBecauseOfRobots.size > 0 || skippedBecauseOfLimit.size > 0) {
+ await Promise.all(
+ [...skippedBecauseOfRobots]
+ .map((url) => {
+ return this.handleSkippedRequest({ url, reason: 'robotsTxt' });
+ })
+ .concat(
+ [...skippedBecauseOfLimit].map((url) => {
+ return this.handleSkippedRequest({ url, reason: 'limit' });
+ }),
+ ),
+ );
+ }
+
+ if (!allowedRequests.length) {
+ return { addedRequests: [], waitForAllRequestsToBeAdded: Promise.resolve([]) };
}
return requestQueue.addRequestsBatched(allowedRequests, options);
@@ -1385,7 +1444,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
request.state = RequestState.SKIPPED;
request.noRetry = true;
await source.markRequestHandled(request);
- await this.onSkippedRequest?.({
+ await this.handleSkippedRequest({
url: request.url,
reason: 'robotsTxt',
});
@@ -1413,7 +1472,8 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
// specify the RQ first to allow overriding it
requestQueue: await this.getRequestQueue(),
robotsTxtFile: await this.getRobotsTxtFileForUrl(request!.url),
- onSkippedRequest: this.onSkippedRequest,
+ onSkippedRequest: this.handleSkippedRequest,
+ limit: this.calculateEnqueuedRequestLimit(options.limit),
...options,
});
},
diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.ts
index a0eb8ca9f..5424a54b6 100644
--- a/packages/browser-crawler/src/internals/browser-crawler.ts
+++ b/packages/browser-crawler/src/internals/browser-crawler.ts
@@ -623,11 +623,11 @@ export abstract class BrowserCrawler<
crawlingContext.enqueueLinks = async (enqueueOptions) => {
return browserCrawlerEnqueueLinks({
- options: enqueueOptions,
+ options: { ...enqueueOptions, limit: this.calculateEnqueuedRequestLimit(enqueueOptions?.limit) },
page,
requestQueue: await this.getRequestQueue(),
robotsTxtFile: await this.getRobotsTxtFileForUrl(crawlingContext.request.url),
- onSkippedRequest: this.onSkippedRequest,
+ onSkippedRequest: this.handleSkippedRequest,
originalRequestUrl: crawlingContext.request.url,
finalRequestUrl: crawlingContext.request.loadedUrl,
});
diff --git a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts b/packages/cheerio-crawler/src/internals/cheerio-crawler.ts
index 700ef3e0c..fe0394a9d 100644
--- a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts
+++ b/packages/cheerio-crawler/src/internals/cheerio-crawler.ts
@@ -191,11 +191,11 @@ export class CheerioCrawler extends HttpCrawler<CheerioCrawlingContext> {
body,
enqueueLinks: async (enqueueOptions?: EnqueueLinksOptions) => {
return cheerioCrawlerEnqueueLinks({
- options: enqueueOptions,
+ options: { ...enqueueOptions, limit: this.calculateEnqueuedRequestLimit(enqueueOptions?.limit) },
$,
requestQueue: await this.getRequestQueue(),
robotsTxtFile: await this.getRobotsTxtFileForUrl(crawlingContext.request.url),
- onSkippedRequest: this.onSkippedRequest,
+ onSkippedRequest: this.handleSkippedRequest,
originalRequestUrl: crawlingContext.request.url,
finalRequestUrl: crawlingContext.request.loadedUrl,
});
diff --git a/packages/core/src/enqueue_links/enqueue_links.ts b/packages/core/src/enqueue_links/enqueue_links.ts
index 2e9d73e09..cadf55d26 100644
--- a/packages/core/src/enqueue_links/enqueue_links.ts
+++ b/packages/core/src/enqueue_links/enqueue_links.ts
@@ -18,7 +18,7 @@ import {
filterRequestsByPatterns,
} from './shared';
-export type SkippedRequestCallback = (args: { url: string; reason: 'robotsTxt' }) => Awaitable<void>;
+export type SkippedRequestCallback = (args: { url: string; reason: 'robotsTxt' | 'limit' }) => Awaitable<void>;
export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
/** Limit the amount of actually enqueued URLs to this number. Useful for testing across the entire crawling scope. */
@@ -427,7 +427,15 @@ export async function enqueueLinks(
}
let requests = createFilteredRequests();
- if (limit) requests = requests.slice(0, limit);
+ if (limit && limit < requests.length) {
+ if (onSkippedRequest) {
+ for (const request of requests.slice(limit)) {
+ await onSkippedRequest({ url: request.url, reason: 'limit' });
+ }
+ }
+
+ requests = requests.slice(0, limit);
+ }
const { addedRequests } = await requestQueue.addRequestsBatched(requests, {
forefront,
diff --git a/packages/core/src/storages/request_provider.ts b/packages/core/src/storages/request_provider.ts
index 0b89c8a66..278c47906 100644
--- a/packages/core/src/storages/request_provider.ts
+++ b/packages/core/src/storages/request_provider.ts
@@ -46,6 +46,7 @@ export abstract class RequestProvider implements IStorage {
assumedHandledCount = 0;
private initialCount = 0;
+ private initialHandledCount = 0; // We track this separately from `assumedHandledCount` which is used non-trivially by RequestQueueV1
protected queueHeadIds = new ListDictionary<string>();
protected requestCache: LruCache<RequestLruItem>;
@@ -93,6 +94,15 @@ export abstract class RequestProvider implements IStorage {
return this.assumedTotalCount + this.initialCount;
}
+ /**
+ * Returns an offline approximation of the total number of pending requests in the queue.
+ *
+ * Survives restarts and Actor migrations.
+ */
+ getPendingCount() {
+ return this.getTotalCount() - this.initialHandledCount - this.assumedHandledCount;
+ }
+
/**
* Adds a request to the queue.
*
@@ -766,8 +776,10 @@ export abstract class RequestProvider implements IStorage {
const queue = await manager.openStorage(queueIdOrName, options.storageClient);
queue.proxyConfiguration = options.proxyConfiguration;
- // eslint-disable-next-line dot-notation
- queue['initialCount'] = (await queue.client.get())?.totalRequestCount ?? 0;
+ const queueInfo = await queue.client.get();
+
+ queue.initialCount = queueInfo?.totalRequestCount ?? 0;
+ queue.initialHandledCount = queueInfo?.handledRequestCount ?? 0;
return queue;
}
diff --git a/packages/jsdom-crawler/src/internals/jsdom-crawler.ts b/packages/jsdom-crawler/src/internals/jsdom-crawler.ts
index 5e0dde069..db89cd2bd 100644
--- a/packages/jsdom-crawler/src/internals/jsdom-crawler.ts
+++ b/packages/jsdom-crawler/src/internals/jsdom-crawler.ts
@@ -302,11 +302,11 @@ export class JSDOMCrawler extends HttpCrawler<JSDOMCrawlingContext> {
},
enqueueLinks: async (enqueueOptions?: EnqueueLinksOptions) => {
return domCrawlerEnqueueLinks({
- options: enqueueOptions,
+ options: { ...enqueueOptions, limit: this.calculateEnqueuedRequestLimit(enqueueOptions?.limit) },
window,
requestQueue: await this.getRequestQueue(),
robotsTxtFile: await this.getRobotsTxtFileForUrl(crawlingContext.request.url),
- onSkippedRequest: this.onSkippedRequest,
+ onSkippedRequest: this.handleSkippedRequest,
originalRequestUrl: crawlingContext.request.url,
finalRequestUrl: crawlingContext.request.loadedUrl,
});
diff --git a/packages/linkedom-crawler/src/internals/linkedom-crawler.ts b/packages/linkedom-crawler/src/internals/linkedom-crawler.ts
index 39153c263..c1e59d7c2 100644
--- a/packages/linkedom-crawler/src/internals/linkedom-crawler.ts
+++ b/packages/linkedom-crawler/src/internals/linkedom-crawler.ts
@@ -185,11 +185,11 @@ export class LinkeDOMCrawler extends HttpCrawler<LinkeDOMCrawlingContext> {
},
enqueueLinks: async (enqueueOptions?: LinkeDOMCrawlerEnqueueLinksOptions) => {
return linkedomCrawlerEnqueueLinks({
- options: enqueueOptions,
+ options: { ...enqueueOptions, limit: this.calculateEnqueuedRequestLimit(enqueueOptions?.limit) },
window: document.defaultView,
requestQueue: await this.getRequestQueue(),
robotsTxtFile: await this.getRobotsTxtFileForUrl(crawlingContext.request.url),
- onSkippedRequest: this.onSkippedRequest,
+ onSkippedRequest: this.handleSkippedRequest,
originalRequestUrl: crawlingContext.request.url,
finalRequestUrl: crawlingContext.request.loadedUrl,
});
diff --git a/test/core/crawlers/adaptive_playwright_crawler.test.ts b/test/core/crawlers/adaptive_playwright_crawler.test.ts
index 93b700469..669828f19 100644
--- a/test/core/crawlers/adaptive_playwright_crawler.test.ts
+++ b/test/core/crawlers/adaptive_playwright_crawler.test.ts
@@ -215,10 +215,15 @@ describe('AdaptivePlaywrightCrawler', () => {
renderingType,
});
const url = new URL(`http://${HOSTNAME}:${port}${path}`);
+ const enqueuedUrls = new Set<string>();
const requestHandler: AdaptivePlaywrightCrawlerOptions['requestHandler'] = vi.fn(
- async ({ enqueueLinks }) => {
- await enqueueLinks();
+ async ({ enqueueLinks, request }) => {
+ if (request.label === 'enqueued-url') {
+ enqueuedUrls.add(request.url);
+ } else {
+ await enqueueLinks({ label: 'enqueued-url' });
+ }
},
);
@@ -226,13 +231,13 @@ describe('AdaptivePlaywrightCrawler', () => {
{
requestHandler,
renderingTypePredictor,
+ maxRequestsPerCrawl: 10,
},
[url.toString()],
);
await crawler.run();
- const enqueuedUrls = (await localStorageEmulator.getRequestQueueItems()).map((item) => item.url);
expect(new Set(enqueuedUrls)).toEqual(
new Set([
`http://${HOSTNAME}:${port}/static?q=1`,
diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.ts
index 5777dc248..87d8de272 100644
--- a/test/core/crawlers/basic_crawler.test.ts
+++ b/test/core/crawlers/basic_crawler.test.ts
@@ -1,8 +1,7 @@
-import { mkdir, mkdtemp, readFile, rm } from 'node:fs/promises';
+import { readFile, rm } from 'node:fs/promises';
import type { Server } from 'node:http';
import http from 'node:http';
import type { AddressInfo } from 'node:net';
-import { join } from 'node:path';
import type { CrawlingContext, ErrorHandler, RequestHandler } from '@crawlee/basic';
import {
@@ -22,6 +21,7 @@ import type { Dictionary } from '@crawlee/utils';
import { sleep } from '@crawlee/utils';
import express from 'express';
import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator';
+import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'vitest';
import log from '@apify/log';
@@ -131,6 +131,45 @@ describe('BasicCrawler', () => {
expect(processed).toHaveLength(sourcesCopy.length * 3);
});
+ test('should process 4 requests total when calling run() twice with maxRequestsPerCrawl: 2', async () => {
+ const processed: { url: string }[] = [];
+
+ const requestHandler: RequestHandler = async ({ request }) => {
+ await sleep(10);
+ processed.push({ url: request.url });
+ };
+
+ const crawler = new BasicCrawler({
+ maxRequestsPerCrawl: 2,
+ minConcurrency: 1,
+ maxConcurrency: 1,
+ requestHandler,
+ });
+
+ // First run should process 2 requests
+ await crawler.run([...Array(5).keys()].map((index) => `https://example.com/first/${index}`));
+ expect(processed).toHaveLength(2);
+
+ // Make sure no extra requests were enqueued
+ expect(await localStorageEmulator.getRequestQueueItems()).toEqual([]);
+
+ // Second run should process 2 more requests
+ await crawler.run([...Array(5).keys()].map((index) => `https://example.com/second/${index}`));
+ expect(processed).toHaveLength(4);
+
+ // Make sure no extra requests were enqueued
+ expect(await localStorageEmulator.getRequestQueueItems()).toEqual([]);
+
+ const processedUrls = processed.map((p) => p.url);
+
+ expect(processedUrls).toEqual([
+ 'https://example.com/first/0',
+ 'https://example.com/first/1',
+ 'https://example.com/second/0',
+ 'https://example.com/second/1',
+ ]);
+ });
+
test('should correctly combine shorthand and full length options', async () => {
const shorthandOptions = {
minConcurrency: 123,
@@ -1398,6 +1437,139 @@ describe('BasicCrawler', () => {
});
});
+ describe('Request enqueuing limits with maxRequestsPerCrawl', () => {
+ test('should not enqueue more requests than maxRequestsPerCrawl allows', async () => {
+ const requestQueue = await RequestQueue.open();
+ const addRequestsBatchedSpy = vitest.spyOn(requestQueue, 'addRequestsBatched');
+
+ const crawler = new BasicCrawler({
+ requestQueue,
+ maxRequestsPerCrawl: 5,
+ requestHandler: async () => {},
+ });
+
+ crawler['handledRequestsCount'] = 2; // eslint-disable-line dot-notation
+
+ // Try to add 6 requests - should only add 3 due to limit
+ const requestsToAdd = [
+ 'http://example.com/1',
+ 'http://example.com/2',
+ 'http://example.com/3',
+ 'http://example.com/4',
+ 'http://example.com/5',
+ 'http://example.com/6',
+ ];
+
+ await crawler.addRequests(requestsToAdd);
+
+ // Should only have added the first 3 requests (since 2 were already processed, limit allows 3 more)
+ expect(addRequestsBatchedSpy).toHaveBeenCalledWith(
+ ['http://example.com/1', 'http://example.com/2', 'http://example.com/3'],
+ {},
+ );
+ });
+
+ test('should not enqueue more requests than maxRequestsPerCrawl allows across multiple addRequests calls', async () => {
+ const requestQueue = await RequestQueue.open();
+ const addRequestsBatchedSpy = vitest.spyOn(requestQueue, 'addRequestsBatched');
+
+ const crawler = new BasicCrawler({
+ requestQueue,
+ maxRequestsPerCrawl: 5,
+ requestHandler: async () => {},
+ });
+
+ crawler['handledRequestsCount'] = 1; // eslint-disable-line dot-notation
+
+ // First call - should add 2 requests (2 more slots to go)
+ await crawler.addRequests(['http://example.com/1', 'http://example.com/2']);
+
+ // Second call - should add only 2 more requests
+ await crawler.addRequests([
+ 'http://example.com/3',
+ 'http://example.com/4',
+ 'http://example.com/5', // This should be ignored
+ 'http://example.com/6', // This should be ignored
+ ]);
+
+ // Third call - should add no requests (limit already reached)
+ await crawler.addRequests(['http://example.com/7', 'http://example.com/8']);
+
+ expect(addRequestsBatchedSpy).toHaveBeenCalledTimes(2); // Third call shouldn't trigger addRequestsBatched
+ expect(addRequestsBatchedSpy.mock.calls).toEqual([
+ [['http://example.com/1', 'http://example.com/2'], {}],
+ [['http://example.com/3', 'http://example.com/4'], {}],
+ ]);
+ });
+
+ test('should respect robots.txt when limiting requests', async () => {
+ const requestQueue = await RequestQueue.open();
+ const addRequestsBatchedSpy = vitest.spyOn(requestQueue, 'addRequestsBatched');
+
+ const crawler = new BasicCrawler({
+ requestQueue,
+ maxRequestsPerCrawl: 2,
+ respectRobotsTxtFile: true,
+ requestHandler: async () => {},
+ });
+
+ crawler['handledRequestsCount'] = 0; // eslint-disable-line dot-notation
+
+ // Mock robots.txt checking to disallow some URLs
+ vitest.spyOn(crawler as any, 'isAllowedBasedOnRobotsTxtFile').mockImplementation(async (url) => {
+ return url !== 'http://example.com/2';
+ });
+
+ await crawler.addRequests([
+ 'http://example.com/1', // Allowed by robots.txt
+ 'http://example.com/2', // Blocked by robots.txt
+ 'http://example.com/3', // Allowed by robots.txt && within limit
+ 'http://example.com/4', // Would exceed limit
+ ]);
+
+ // Should only have added the first request (allowed by robots.txt and within limit)
+ expect(addRequestsBatchedSpy).toHaveBeenCalledWith(['http://example.com/1', 'http://example.com/3'], {});
+ });
+
+ test('enqueueLinks should respect maxRequestsPerCrawl', async () => {
+ const requestQueue = await RequestQueue.open();
+ const addRequestsBatchedSpy = vitest.spyOn(requestQueue, 'addRequestsBatched');
+
+ // Will try to add 6 requests - should only add 3 due to limit
+ const requestsToAdd = [
+ 'http://example.com/1',
+ 'http://example.com/2',
+ 'http://example.com/3',
+ 'http://example.com/4',
+ 'http://example.com/5',
+ 'http://example.com/6',
+ ];
+
+ const crawler = new BasicCrawler({
+ requestQueue,
+ maxRequestsPerCrawl: 5,
+ requestHandler: async (context) => {
+ if (context.request.label) {
+ return;
+ }
+
+ crawler['handledRequestsCount'] = 2; // eslint-disable-line dot-notation
+
+ await context.enqueueLinks({ urls: requestsToAdd, label: 'not-undefined' });
+ },
+ });
+
+ await crawler.run(['http://example.com']);
+
+ // Should only have added the first 2 requests (since 2 were already processed and 1 is in progress, limit allows 2 more)
+ expect(addRequestsBatchedSpy).toHaveBeenCalledTimes(2);
+ expect(addRequestsBatchedSpy.mock.calls).toMatchObject([
+ [['http://example.com'], {}], // added by crawler.run()
+ [[{ url: 'http://example.com/1' }, { url: 'http://example.com/2' }], {}],
+ ]);
+ });
+ });
+
describe('Dataset helpers, crawler parallelism', () => {
const payload: Dictionary[] = [{ foo: 'bar', baz: 123 }];
const getPayload: (id: string) => Dictionary[] = (id) => [{ foo: id }];