Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

feat: add `maxCrawlDepth` crawler option (#3045)

Jindřich Bär0090df93a12d
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
index 2b0c5c431..854c8a5e9 100644
--- a/packages/basic-crawler/src/internals/basic-crawler.ts
+++ b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -266,6 +266,13 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCraw
*/
maxRequestsPerCrawl?: number;
+ /**
+ * Maximum depth of the crawl. If not set, the crawl will continue until all requests are processed.
+ * Setting this to `0` will only process the initial requests, skipping all links enqueued by `crawlingContext.enqueueLinks` and `crawlingContext.addRequests`.
+ * Passing `1` will process the initial requests and all links enqueued by `crawlingContext.enqueueLinks` and `crawlingContext.addRequests` in the handler for initial requests.
+ */
+ maxCrawlDepth?: number;
+
/**
* Custom options passed to the underlying {@apilink AutoscaledPool} constructor.
* > *NOTE:* The {@apilink AutoscaledPoolOptions.runTaskFunction|`runTaskFunction`}
@@ -516,6 +523,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
protected requestHandlerTimeoutMillis!: number;
protected internalTimeoutMillis: number;
protected maxRequestRetries: number;
+ protected maxCrawlDepth?: number;
protected sameDomainDelayMillis: number;
protected domainAccessedTime: Map<string, number>;
protected maxSessionRotations: number;
@@ -559,6 +567,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
sameDomainDelaySecs: ow.optional.number,
maxSessionRotations: ow.optional.number,
maxRequestsPerCrawl: ow.optional.number,
+ maxCrawlDepth: ow.optional.number,
autoscaledPoolOptions: ow.optional.object,
sessionPoolOptions: ow.optional.object,
useSessionPool: ow.optional.boolean,
@@ -600,6 +609,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
sameDomainDelaySecs = 0,
maxSessionRotations = 10,
maxRequestsPerCrawl,
+ maxCrawlDepth,
autoscaledPoolOptions = {},
keepAlive,
sessionPoolOptions = {},
@@ -711,6 +721,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
}
this.maxRequestRetries = maxRequestRetries;
+ this.maxCrawlDepth = maxCrawlDepth;
this.sameDomainDelayMillis = sameDomainDelaySecs * 1000;
this.maxSessionRotations = maxSessionRotations;
this.stats = new Statistics({
@@ -1112,8 +1123,10 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
const skippedBecauseOfRobots = new Set<string>();
const skippedBecauseOfLimit = new Set<string>();
+ const skippedBecauseOfMaxCrawlDepth = new Set<string>();
const isAllowedBasedOnRobotsTxtFile = this.isAllowedBasedOnRobotsTxtFile.bind(this);
+ const maxCrawlDepth = this.maxCrawlDepth;
async function* filteredRequests() {
let yieldedRequestCount = 0;
@@ -1126,6 +1139,11 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
continue;
}
+ if (maxCrawlDepth !== undefined && (request as any).crawlDepth > maxCrawlDepth) {
+ skippedBecauseOfMaxCrawlDepth.add(url);
+ continue;
+ }
+
if (await isAllowedBasedOnRobotsTxtFile(url)) {
yield request;
yieldedRequestCount += 1;
@@ -1143,7 +1161,11 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
});
}
- if (skippedBecauseOfRobots.size > 0 || skippedBecauseOfLimit.size > 0) {
+ if (
+ skippedBecauseOfRobots.size > 0 ||
+ skippedBecauseOfLimit.size > 0 ||
+ skippedBecauseOfMaxCrawlDepth.size > 0
+ ) {
await Promise.all(
[...skippedBecauseOfRobots]
.map((url) => {
@@ -1153,6 +1175,9 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
[...skippedBecauseOfLimit].map((url) => {
return this.handleSkippedRequest({ url, reason: 'limit' });
}),
+ [...skippedBecauseOfMaxCrawlDepth].map((url) => {
+ return this.handleSkippedRequest({ url, reason: 'depth' });
+ }),
),
);
}
@@ -1480,10 +1505,35 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
robotsTxtFile: await this.getRobotsTxtFileForUrl(request!.url),
onSkippedRequest: this.handleSkippedRequest,
limit: this.calculateEnqueuedRequestLimit(options.limit),
+ transformRequestFunction: (newRequest) => {
+ newRequest.crawlDepth = (request?.crawlDepth ?? 0) + 1;
+
+ if (this.maxCrawlDepth !== undefined && newRequest.crawlDepth > this.maxCrawlDepth) {
+ newRequest.skippedReason = 'depth';
+ return false;
+ }
+
+ return options.transformRequestFunction?.(newRequest) ?? newRequest;
+ },
...options,
});
},
- addRequests: this.addRequests.bind(this),
+ addRequests: async (requests: RequestsLike, options: CrawlerAddRequestsOptions = {}) => {
+ const newRequestDepth = (request?.crawlDepth ?? 0) + 1;
+
+ async function* injectDepth() {
+ for await (const rq of requests) {
+ if (typeof rq === 'string') {
+ yield { url: rq, crawlDepth: newRequestDepth };
+ } else {
+ rq.crawlDepth ??= newRequestDepth;
+ yield rq;
+ }
+ }
+ }
+
+ return this.addRequests(injectDepth(), options);
+ },
pushData: this.pushData.bind(this),
useState: this.useState.bind(this),
sendRequest: createSendRequest(this.httpClient, request!, session, () => crawlingContext.proxyInfo?.url),
diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.ts
index 68618f5e3..41a8adcb5 100644
--- a/packages/browser-crawler/src/internals/browser-crawler.ts
+++ b/packages/browser-crawler/src/internals/browser-crawler.ts
@@ -1,6 +1,7 @@
import type {
Awaitable,
BasicCrawlerOptions,
+ BasicCrawlingContext,
CrawlingContext,
Dictionary,
EnqueueLinksOptions,
@@ -623,6 +624,7 @@ export abstract class BrowserCrawler<
crawlingContext.proxyInfo = browserControllerInstance.launchContext.proxyInfo as ProxyInfo;
}
+ const contextEnqueueLinks = crawlingContext.enqueueLinks;
crawlingContext.enqueueLinks = async (enqueueOptions) => {
return browserCrawlerEnqueueLinks({
options: { ...enqueueOptions, limit: this.calculateEnqueuedRequestLimit(enqueueOptions?.limit) },
@@ -632,6 +634,7 @@ export abstract class BrowserCrawler<
onSkippedRequest: this.handleSkippedRequest,
originalRequestUrl: crawlingContext.request.url,
finalRequestUrl: crawlingContext.request.loadedUrl,
+ enqueueLinks: contextEnqueueLinks,
});
};
}
@@ -802,35 +805,54 @@ interface EnqueueLinksInternalOptions {
}
/** @internal */
-export async function browserCrawlerEnqueueLinks({
- options,
- page,
- requestQueue,
- robotsTxtFile,
- onSkippedRequest,
- originalRequestUrl,
- finalRequestUrl,
-}: EnqueueLinksInternalOptions) {
+interface BoundEnqueueLinksInternalOptions {
+ enqueueLinks: BasicCrawlingContext['enqueueLinks'];
+ options?: ReadonlyDeep<Omit<EnqueueLinksOptions, 'requestQueue'>> & Pick<EnqueueLinksOptions, 'requestQueue'>;
+ originalRequestUrl: string;
+ finalRequestUrl?: string;
+ page: CommonPage;
+}
+
+/** @internal */
+function containsEnqueueLinks(
+ options: EnqueueLinksInternalOptions | BoundEnqueueLinksInternalOptions,
+): options is BoundEnqueueLinksInternalOptions {
+ return !!(options as BoundEnqueueLinksInternalOptions).enqueueLinks;
+}
+
+/** @internal */
+export async function browserCrawlerEnqueueLinks(
+ options: EnqueueLinksInternalOptions | BoundEnqueueLinksInternalOptions,
+) {
+ const { options: enqueueLinksOptions, finalRequestUrl, originalRequestUrl, page } = options;
+
const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
- enqueueStrategy: options?.strategy,
+ enqueueStrategy: enqueueLinksOptions?.strategy,
finalRequestUrl,
originalRequestUrl,
- userProvidedBaseUrl: options?.baseUrl,
+ userProvidedBaseUrl: enqueueLinksOptions?.baseUrl,
});
const urls = await extractUrlsFromPage(
page as any,
- options?.selector ?? 'a',
- options?.baseUrl ?? finalRequestUrl ?? originalRequestUrl,
+ enqueueLinksOptions?.selector ?? 'a',
+ enqueueLinksOptions?.baseUrl ?? finalRequestUrl ?? originalRequestUrl,
);
+ if (containsEnqueueLinks(options)) {
+ return options.enqueueLinks({
+ urls,
+ baseUrl,
+ ...enqueueLinksOptions,
+ });
+ }
return enqueueLinks({
- requestQueue,
- robotsTxtFile,
- onSkippedRequest,
+ requestQueue: options.requestQueue,
+ robotsTxtFile: options.robotsTxtFile,
+ onSkippedRequest: options.onSkippedRequest,
urls,
baseUrl,
- ...(options as EnqueueLinksOptions),
+ ...(enqueueLinksOptions as EnqueueLinksOptions),
});
}
diff --git a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts b/packages/cheerio-crawler/src/internals/cheerio-crawler.ts
index fe0394a9d..4ed785497 100644
--- a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts
+++ b/packages/cheerio-crawler/src/internals/cheerio-crawler.ts
@@ -2,6 +2,7 @@ import type { IncomingMessage } from 'node:http';
import { text as readStreamToString } from 'node:stream/consumers';
import type {
+ BasicCrawlingContext,
Configuration,
EnqueueLinksOptions,
ErrorHandler,
@@ -185,6 +186,8 @@ export class CheerioCrawler extends HttpCrawler<CheerioCrawlingContext> {
_useHtmlParser2: true,
} as CheerioOptions);
+ const originalEnqueueLinks = crawlingContext.enqueueLinks;
+
return {
dom,
$,
@@ -198,6 +201,7 @@ export class CheerioCrawler extends HttpCrawler<CheerioCrawlingContext> {
onSkippedRequest: this.handleSkippedRequest,
originalRequestUrl: crawlingContext.request.url,
finalRequestUrl: crawlingContext.request.loadedUrl,
+ enqueueLinks: originalEnqueueLinks,
});
},
};
@@ -247,40 +251,57 @@ interface EnqueueLinksInternalOptions {
finalRequestUrl?: string;
}
+interface BoundEnqueueLinksInternalOptions {
+ enqueueLinks: BasicCrawlingContext['enqueueLinks'];
+ options?: EnqueueLinksOptions;
+ $: cheerio.CheerioAPI | null;
+ originalRequestUrl: string;
+ finalRequestUrl?: string;
+}
+
+/** @internal */
+function containsEnqueueLinks(
+ options: EnqueueLinksInternalOptions | BoundEnqueueLinksInternalOptions,
+): options is BoundEnqueueLinksInternalOptions {
+ return !!(options as BoundEnqueueLinksInternalOptions).enqueueLinks;
+}
+
/** @internal */
-export async function cheerioCrawlerEnqueueLinks({
- options,
- $,
- requestQueue,
- robotsTxtFile,
- onSkippedRequest,
- originalRequestUrl,
- finalRequestUrl,
-}: EnqueueLinksInternalOptions) {
+export async function cheerioCrawlerEnqueueLinks(
+ options: EnqueueLinksInternalOptions | BoundEnqueueLinksInternalOptions,
+) {
+ const { options: enqueueLinksOptions, $, originalRequestUrl, finalRequestUrl } = options;
if (!$) {
throw new Error('Cannot enqueue links because the DOM is not available.');
}
const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
- enqueueStrategy: options?.strategy,
+ enqueueStrategy: enqueueLinksOptions?.strategy,
finalRequestUrl,
originalRequestUrl,
- userProvidedBaseUrl: options?.baseUrl,
+ userProvidedBaseUrl: enqueueLinksOptions?.baseUrl,
});
const urls = extractUrlsFromCheerio(
$,
- options?.selector ?? 'a',
- options?.baseUrl ?? finalRequestUrl ?? originalRequestUrl,
+ enqueueLinksOptions?.selector ?? 'a',
+ enqueueLinksOptions?.baseUrl ?? finalRequestUrl ?? originalRequestUrl,
);
+ if (containsEnqueueLinks(options)) {
+ return options.enqueueLinks({
+ urls,
+ baseUrl,
+ ...enqueueLinksOptions,
+ });
+ }
return enqueueLinks({
- requestQueue,
- robotsTxtFile,
- onSkippedRequest,
+ requestQueue: options.requestQueue,
+ robotsTxtFile: options.robotsTxtFile,
+ onSkippedRequest: options.onSkippedRequest,
urls,
baseUrl,
- ...options,
+ ...enqueueLinksOptions,
});
}
diff --git a/packages/core/src/enqueue_links/enqueue_links.ts b/packages/core/src/enqueue_links/enqueue_links.ts
index 6ccbf643d..6a87e1173 100644
--- a/packages/core/src/enqueue_links/enqueue_links.ts
+++ b/packages/core/src/enqueue_links/enqueue_links.ts
@@ -401,11 +401,14 @@ export async function enqueueLinks(
}
}
- async function reportSkippedRequests(skippedRequests: { url: string }[], reason: SkippedRequestReason) {
+ async function reportSkippedRequests(
+ skippedRequests: { url: string; skippedReason?: SkippedRequestReason }[],
+ reason: SkippedRequestReason,
+ ) {
if (onSkippedRequest && skippedRequests.length > 0) {
await Promise.all(
skippedRequests.map((request) => {
- return onSkippedRequest({ url: request.url, reason });
+ return onSkippedRequest({ url: request.url, reason: request.skippedReason ?? reason });
}),
);
}
diff --git a/packages/core/src/enqueue_links/shared.ts b/packages/core/src/enqueue_links/shared.ts
index ffc3b643b..e7344b1ad 100644
--- a/packages/core/src/enqueue_links/shared.ts
+++ b/packages/core/src/enqueue_links/shared.ts
@@ -47,7 +47,7 @@ export type RegExpObject = { regexp: RegExp } & Pick<
export type RegExpInput = RegExp | RegExpObject;
-export type SkippedRequestReason = 'robotsTxt' | 'limit' | 'filters' | 'redirect';
+export type SkippedRequestReason = 'robotsTxt' | 'limit' | 'filters' | 'redirect' | 'depth';
export type SkippedRequestCallback = (args: {
url: string;
diff --git a/packages/core/src/request.ts b/packages/core/src/request.ts
index db96c2c49..bafc6a1d2 100644
--- a/packages/core/src/request.ts
+++ b/packages/core/src/request.ts
@@ -9,6 +9,7 @@ import ow from 'ow';
import { normalizeUrl } from '@apify/utilities';
import type { EnqueueLinksOptions } from './enqueue_links/enqueue_links';
+import type { SkippedRequestReason } from './enqueue_links/shared';
import { log as defaultLog } from './log';
import type { AllowedHttpMethods } from './typedefs';
import { keys } from './typedefs';
@@ -34,6 +35,7 @@ const requestOptionalPredicates = {
keepUrlFragment: ow.optional.boolean,
useExtendedUniqueKey: ow.optional.boolean,
skipNavigation: ow.optional.boolean,
+ crawlDepth: ow.optional.number.greaterThanOrEqual(0),
state: ow.optional.number.greaterThanOrEqual(0).lessThanOrEqual(6),
};
@@ -175,6 +177,7 @@ export class Request<UserData extends Dictionary = Dictionary> {
useExtendedUniqueKey = false,
skipNavigation,
enqueueStrategy,
+ crawlDepth,
} = options as RequestOptions & {
loadedUrl?: string;
retryCount?: number;
@@ -248,6 +251,7 @@ export class Request<UserData extends Dictionary = Dictionary> {
if (skipNavigation != null) this.skipNavigation = skipNavigation;
if (maxRetries != null) this.maxRetries = maxRetries;
+ if (crawlDepth != null) this.userData.__crawlee.crawlDepth ??= crawlDepth;
// If it's already set, don't override it (for instance when fetching from storage)
if (enqueueStrategy) {
@@ -269,6 +273,23 @@ export class Request<UserData extends Dictionary = Dictionary> {
}
}
+ /**
+ * Depth of the request in the current crawl tree.
+ * Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
+ */
+ get crawlDepth(): number {
+ return this.userData.__crawlee?.crawlDepth ?? 0;
+ }
+
+ /**
+ * Depth of the request in the current crawl tree.
+ * Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
+ */
+ set crawlDepth(value: number) {
+ (this.userData as Dictionary).__crawlee ??= {};
+ this.userData.__crawlee.crawlDepth = value;
+ }
+
/** Indicates the number of times the crawling of the request has rotated the session due to a session or a proxy error. */
get sessionRotationCount(): number {
return this.userData.__crawlee?.sessionRotationCount ?? 0;
@@ -503,6 +524,20 @@ export interface RequestOptions<UserData extends Dictionary = Dictionary> {
*/
skipNavigation?: boolean;
+ /**
+ * Depth of the request in the current crawl tree.
+ * Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
+ * @default 0
+ */
+ crawlDepth?: number;
+
+ /**
+ * Reason for skipping this request.
+ * This is used to provide more information about why the request was skipped.
+ * @internal
+ */
+ skippedReason?: SkippedRequestReason;
+
/**
* Maximum number of retries for this request. Allows to override the global `maxRequestRetries` option of `BasicCrawler`.
*/
diff --git a/packages/jsdom-crawler/src/internals/jsdom-crawler.ts b/packages/jsdom-crawler/src/internals/jsdom-crawler.ts
index db89cd2bd..0178684af 100644
--- a/packages/jsdom-crawler/src/internals/jsdom-crawler.ts
+++ b/packages/jsdom-crawler/src/internals/jsdom-crawler.ts
@@ -1,6 +1,7 @@
import type { IncomingMessage } from 'node:http';
import type {
+ BasicCrawlingContext,
Configuration,
EnqueueLinksOptions,
ErrorHandler,
@@ -352,40 +353,57 @@ interface EnqueueLinksInternalOptions {
finalRequestUrl?: string;
}
+interface BoundEnqueueLinksInternalOptions {
+ enqueueLinks: BasicCrawlingContext['enqueueLinks'];
+ options?: EnqueueLinksOptions;
+ window: DOMWindow | null;
+ originalRequestUrl: string;
+ finalRequestUrl?: string;
+}
+
/** @internal */
-export async function domCrawlerEnqueueLinks({
- options,
- window,
- requestQueue,
- robotsTxtFile,
- onSkippedRequest,
- originalRequestUrl,
- finalRequestUrl,
-}: EnqueueLinksInternalOptions) {
+function containsEnqueueLinks(
+ options: EnqueueLinksInternalOptions | BoundEnqueueLinksInternalOptions,
+): options is BoundEnqueueLinksInternalOptions {
+ return !!(options as BoundEnqueueLinksInternalOptions).enqueueLinks;
+}
+
+/** @internal */
+export async function domCrawlerEnqueueLinks(options: EnqueueLinksInternalOptions | BoundEnqueueLinksInternalOptions) {
+ const { options: enqueueLinksOptions, window, originalRequestUrl, finalRequestUrl } = options;
+
if (!window) {
throw new Error('Cannot enqueue links because the JSDOM is not available.');
}
const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
- enqueueStrategy: options?.strategy,
+ enqueueStrategy: enqueueLinksOptions?.strategy,
finalRequestUrl,
originalRequestUrl,
- userProvidedBaseUrl: options?.baseUrl,
+ userProvidedBaseUrl: enqueueLinksOptions?.baseUrl,
});
const urls = extractUrlsFromWindow(
window,
- options?.selector ?? 'a',
- options?.baseUrl ?? finalRequestUrl ?? originalRequestUrl,
+ enqueueLinksOptions?.selector ?? 'a',
+ enqueueLinksOptions?.baseUrl ?? finalRequestUrl ?? originalRequestUrl,
);
+ if (containsEnqueueLinks(options)) {
+ return options.enqueueLinks({
+ urls,
+ baseUrl,
+ ...enqueueLinksOptions,
+ });
+ }
+
return enqueueLinks({
- requestQueue,
- robotsTxtFile,
- onSkippedRequest,
+ requestQueue: options.requestQueue,
+ robotsTxtFile: options.robotsTxtFile,
+ onSkippedRequest: options.onSkippedRequest,
urls,
baseUrl,
- ...options,
+ ...enqueueLinksOptions,
});
}
diff --git a/packages/linkedom-crawler/src/internals/linkedom-crawler.ts b/packages/linkedom-crawler/src/internals/linkedom-crawler.ts
index c1e59d7c2..bb06f6a2f 100644
--- a/packages/linkedom-crawler/src/internals/linkedom-crawler.ts
+++ b/packages/linkedom-crawler/src/internals/linkedom-crawler.ts
@@ -1,6 +1,7 @@
import type { IncomingMessage } from 'node:http';
import type {
+ BasicCrawlingContext,
EnqueueLinksOptions,
ErrorHandler,
GetUserDataFromRequest,
@@ -226,7 +227,7 @@ export class LinkeDOMCrawler extends HttpCrawler<LinkeDOMCrawlingContext> {
}
interface EnqueueLinksInternalOptions {
- options?: LinkeDOMCrawlerEnqueueLinksOptions;
+ options?: EnqueueLinksOptions;
window: Window | null;
requestQueue: RequestProvider;
robotsTxtFile?: RobotsTxtFile;
@@ -235,40 +236,59 @@ interface EnqueueLinksInternalOptions {
finalRequestUrl?: string;
}
+interface BoundEnqueueLinksInternalOptions {
+ enqueueLinks: BasicCrawlingContext['enqueueLinks'];
+ options?: EnqueueLinksOptions;
+ window: Window | null;
+ originalRequestUrl: string;
+ finalRequestUrl?: string;
+}
+
/** @internal */
-export async function linkedomCrawlerEnqueueLinks({
- options,
- window,
- requestQueue,
- robotsTxtFile,
- onSkippedRequest,
- originalRequestUrl,
- finalRequestUrl,
-}: EnqueueLinksInternalOptions) {
+function containsEnqueueLinks(
+ options: EnqueueLinksInternalOptions | BoundEnqueueLinksInternalOptions,
+): options is BoundEnqueueLinksInternalOptions {
+ return !!(options as BoundEnqueueLinksInternalOptions).enqueueLinks;
+}
+
+/** @internal */
+export async function linkedomCrawlerEnqueueLinks(
+ options: EnqueueLinksInternalOptions | BoundEnqueueLinksInternalOptions,
+) {
+ const { options: enqueueLinksOptions, window, originalRequestUrl, finalRequestUrl } = options;
+
if (!window) {
throw new Error('Cannot enqueue links because the DOM is not available.');
}
const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
- enqueueStrategy: options?.strategy,
+ enqueueStrategy: enqueueLinksOptions?.strategy,
finalRequestUrl,
originalRequestUrl,
- userProvidedBaseUrl: options?.baseUrl,
+ userProvidedBaseUrl: enqueueLinksOptions?.baseUrl,
});
const urls = extractUrlsFromWindow(
window,
- options?.selector ?? 'a',
- options?.baseUrl ?? finalRequestUrl ?? originalRequestUrl,
+ enqueueLinksOptions?.selector ?? 'a',
+ enqueueLinksOptions?.baseUrl ?? finalRequestUrl ?? originalRequestUrl,
);
+ if (containsEnqueueLinks(options)) {
+ return options.enqueueLinks({
+ urls,
+ baseUrl,
+ ...enqueueLinksOptions,
+ });
+ }
+
return enqueueLinks({
- requestQueue,
- robotsTxtFile,
- onSkippedRequest,
+ requestQueue: options.requestQueue,
+ robotsTxtFile: options.robotsTxtFile,
+ onSkippedRequest: options.onSkippedRequest,
urls,
baseUrl,
- ...options,
+ ...enqueueLinksOptions,
});
}
diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.ts
index 02c6df89a..363c0f13a 100644
--- a/test/core/crawlers/basic_crawler.test.ts
+++ b/test/core/crawlers/basic_crawler.test.ts
@@ -170,6 +170,60 @@ describe('BasicCrawler', () => {
]);
});
+ test('addRequests should respect maxCrawlDepth', async () => {
+ const processedUrls: string[] = [];
+
+ const requestHandler: RequestHandler = async ({ request, addRequests }) => {
+ processedUrls.push(request.url);
+ const url = new URL(request.url);
+ url.pathname = `${url.pathname}deep/`;
+
+ await addRequests([url.toString()]);
+ };
+
+ const crawler = new BasicCrawler({
+ maxCrawlDepth: 2,
+ maxRequestsPerCrawl: 10, // safeguard against infinite loops
+ requestHandler,
+ });
+
+ await crawler.run(['https://example.com/']);
+
+ expect(processedUrls).toEqual([
+ 'https://example.com/',
+ 'https://example.com/deep/',
+ 'https://example.com/deep/deep/',
+ ]);
+ });
+
+ test('enqueueLinks should respect maxCrawlDepth', async () => {
+ const processedUrls: string[] = [];
+
+ const requestHandler: RequestHandler = async ({ request, enqueueLinks }) => {
+ processedUrls.push(request.url);
+ const url = new URL(request.url);
+ url.pathname = `${url.pathname}deep/`;
+
+ await enqueueLinks({
+ urls: [url.toString()],
+ });
+ };
+
+ const crawler = new BasicCrawler({
+ maxCrawlDepth: 2,
+ maxRequestsPerCrawl: 10, // safeguard against infinite loops
+ requestHandler,
+ });
+
+ await crawler.run(['https://example.com/']);
+
+ expect(processedUrls).toEqual([
+ 'https://example.com/',
+ 'https://example.com/deep/',
+ 'https://example.com/deep/deep/',
+ ]);
+ });
+
test('should correctly combine shorthand and full length options', async () => {
const shorthandOptions = {
minConcurrency: 123,
diff --git a/test/core/crawlers/browser_crawler.test.ts b/test/core/crawlers/browser_crawler.test.ts
index acca42c4c..738bab4a7 100644
--- a/test/core/crawlers/browser_crawler.test.ts
+++ b/test/core/crawlers/browser_crawler.test.ts
@@ -1065,4 +1065,25 @@ describe('BrowserCrawler', () => {
expect(succeeded).toHaveLength(1);
expect(succeeded[0]).toEqual('Redirecting outside');
});
+
+ test('enqueueLinks should respect maxCrawlDepth', async () => {
+ const succeeded: string[] = [];
+
+ const crawler = new BrowserCrawlerTest({
+ browserPoolOptions: {
+ browserPlugins: [puppeteerPlugin],
+ },
+ maxCrawlDepth: 1,
+ maxRequestsPerCrawl: 10, // avoiding accidental runaway
+ requestHandler: async ({ page, enqueueLinks }) => {
+ succeeded.push(await page.title());
+ await enqueueLinks({ strategy: EnqueueStrategy.All });
+ },
+ });
+
+ await crawler.run([`${serverAddress}/special/html-type`]);
+
+ expect(succeeded).toHaveLength(2);
+ expect(succeeded).toEqual(['Example Domain', 'Example Domains']);
+ });
});
diff --git a/test/core/crawlers/cheerio_crawler.test.ts b/test/core/crawlers/cheerio_crawler.test.ts
index 64b830f63..9e663d13a 100644
--- a/test/core/crawlers/cheerio_crawler.test.ts
+++ b/test/core/crawlers/cheerio_crawler.test.ts
@@ -1374,4 +1374,22 @@ describe('CheerioCrawler', () => {
expect(succeeded).toHaveLength(1);
expect(succeeded[0]).toEqual('Redirecting outside');
});
+
+ test('enqueueLinks should respect maxCrawlDepth', async () => {
+ const succeeded: string[] = [];
+
+ const crawler = new CheerioCrawler({
+ maxCrawlDepth: 1,
+ maxRequestsPerCrawl: 10, // to avoid accidental runaway
+ requestHandler: async ({ $, enqueueLinks }) => {
+ succeeded.push($('title').text());
+ await enqueueLinks({ strategy: EnqueueStrategy.All });
+ },
+ });
+
+ await crawler.run([`${serverAddress}/special/html-type`]);
+
+ expect(succeeded).toHaveLength(2);
+ expect(succeeded).toEqual(['Example Domain', 'Example Domains']);
+ });
});
diff --git a/test/core/storages/request_queue.test.ts b/test/core/storages/request_queue.test.ts
index f35a7ccbb..881c4a91b 100644
--- a/test/core/storages/request_queue.test.ts
+++ b/test/core/storages/request_queue.test.ts
@@ -708,7 +708,7 @@ describe('RequestQueue remote', () => {
const r1 = new Request({
url,
method,
- userData: { __crawlee: { skipNavigation: true, maxRetries: 10, foo: 123, bar: true } },
+ userData: { __crawlee: { skipNavigation: true, maxRetries: 10, foo: 123, bar: true, crawlDepth: 10 } },
});
const r2 = new Request({
url,
@@ -723,8 +723,15 @@ describe('RequestQueue remote', () => {
expect(desc1!.enumerable).toBe(false);
expect(r1.skipNavigation).toBe(true);
expect(r1.maxRetries).toBe(10);
+ expect(r1.crawlDepth).toBe(10);
r1.maxRetries = 5;
- expect(r1.userData.__crawlee).toMatchObject({ skipNavigation: true, maxRetries: 5, foo: 123, bar: true });
+ expect(r1.userData.__crawlee).toMatchObject({
+ skipNavigation: true,
+ maxRetries: 5,
+ foo: 123,
+ bar: true,
+ crawlDepth: 10,
+ });
const desc2 = Object.getOwnPropertyDescriptor(r2.userData, '__crawlee');
expect(desc2!.enumerable).toBe(false);
expect(r2.maxRetries).toBeUndefined();