Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: respect `maxCrawlDepth` with a custom enqueueLinks `transformRequestFunction` (#3159)

Lukáš Průšae2ecb745da61
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
index bf1b75c5b..f5b28a95b 100644
--- a/packages/basic-crawler/src/internals/basic-crawler.ts
+++ b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -1570,6 +1570,10 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
robotsTxtFile: await this.getRobotsTxtFileForUrl(request!.url),
onSkippedRequest: this.handleSkippedRequest,
limit: this.calculateEnqueuedRequestLimit(options.limit),
+ ...options,
+
+ // Allow all previous option defaults to be overridden by the user, but execute this as a wrapper
+ // around the original function, to inject `crawlDepth` and check the `maxCrawlDepth` limit.
transformRequestFunction: (newRequest) => {
newRequest.crawlDepth = (request?.crawlDepth ?? 0) + 1;
@@ -1580,7 +1584,6 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
return options.transformRequestFunction?.(newRequest) ?? newRequest;
},
- ...options,
});
},
addRequests: async (requests: RequestsLike, options: CrawlerAddRequestsOptions = {}) => {
diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.ts
index 216c57c5d..24c1fe9ba 100644
--- a/test/core/crawlers/basic_crawler.test.ts
+++ b/test/core/crawlers/basic_crawler.test.ts
@@ -3,7 +3,13 @@ import type { Server } from 'node:http';
import http from 'node:http';
import type { AddressInfo } from 'node:net';
-import type { CrawlingContext, ErrorHandler, RequestHandler } from '@crawlee/basic';
+import type {
+ CrawlingContext,
+ EnqueueLinksOptions,
+ ErrorHandler,
+ RequestHandler,
+ RequestOptions,
+} from '@crawlee/basic';
import {
BasicCrawler,
Configuration,
@@ -196,32 +202,47 @@ describe('BasicCrawler', () => {
]);
});
- test('enqueueLinks should respect maxCrawlDepth', async () => {
- const processedUrls: string[] = [];
+ describe('enqueueLinks should respect maxCrawlDepth', async () => {
+ const expectedUrls = ['https://example.com/', 'https://example.com/deep/', 'https://example.com/deep/deep/'];
- const requestHandler: RequestHandler = async ({ request, enqueueLinks }) => {
- processedUrls.push(request.url);
- const url = new URL(request.url);
- url.pathname = `${url.pathname}deep/`;
+ const runCrawlWithDepthLimit = async (enqueueLinksOptions?: EnqueueLinksOptions): Promise<string[]> => {
+ const processedUrls: string[] = [];
+
+ const requestHandler: RequestHandler = async ({ request, enqueueLinks }) => {
+ processedUrls.push(request.url);
+ const url = new URL(request.url);
+ url.pathname = `${url.pathname}deep/`;
- await enqueueLinks({
- urls: [url.toString()],
+ await enqueueLinks({
+ urls: [url.toString()],
+ ...enqueueLinksOptions,
+ });
+ };
+
+ const crawler = new BasicCrawler({
+ maxCrawlDepth: 2,
+ maxRequestsPerCrawl: 10, // safeguard against infinite loops
+ requestHandler,
});
+
+ await crawler.run(['https://example.com/']);
+
+ return processedUrls;
};
- const crawler = new BasicCrawler({
- maxCrawlDepth: 2,
- maxRequestsPerCrawl: 10, // safeguard against infinite loops
- requestHandler,
+ test("without user's transformRequestFunction", async () => {
+ const processedUrls = await runCrawlWithDepthLimit();
+ expect(processedUrls).toEqual(expectedUrls);
});
- await crawler.run(['https://example.com/']);
+ test('while executing user provided transformRequestFunction', async () => {
+ const transformRequestFunction = vi.fn((request: RequestOptions) => request);
- expect(processedUrls).toEqual([
- 'https://example.com/',
- 'https://example.com/deep/',
- 'https://example.com/deep/deep/',
- ]);
+ const processedUrls = await runCrawlWithDepthLimit({ transformRequestFunction });
+ expect(processedUrls).toEqual(expectedUrls);
+
+ expect(transformRequestFunction).toHaveBeenCalled();
+ });
});
test('should correctly combine shorthand and full length options', async () => {