Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: use shared enqueue links wrapper in `AdaptivePlaywrightCrawler` (#3188)
Lukáš Průša9569d1919333
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex f3919651f..203ddc183 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -17,12 +17,14 @@ import type {ProxyInfo,Request,RequestsLike,+ RequestTransform,RestrictedCrawlingContext,RouterHandler,RouterRoutes,Session,SessionPoolOptions,SkippedRequestCallback,+ Source,StatisticsOptions,StatisticState,} from '@crawlee/core';@@ -1564,49 +1566,19 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextrequest,session,enqueueLinks: async (options: SetRequired<EnqueueLinksOptions, 'urls'>) => {- await this.getRequestQueue();-- return enqueueLinks({- // specify the RQ first to allow overriding it- requestQueue: this.requestManager!,- robotsTxtFile: await this.getRobotsTxtFileForUrl(request!.url),- onSkippedRequest: this.handleSkippedRequest,- limit: this.calculateEnqueuedRequestLimit(options.limit),- ...options,-- // Allow all previous option defaults to be overridden by the user, but execute this as a wrapper- // around the original function, to inject `crawlDepth` and check the `maxCrawlDepth` limit.- transformRequestFunction: (newRequest) => {- newRequest.crawlDepth = (request?.crawlDepth ?? 0) + 1;-- if (this.maxCrawlDepth !== undefined && newRequest.crawlDepth > this.maxCrawlDepth) {- newRequest.skippedReason = 'depth';- return false;- }-- return options.transformRequestFunction?.(newRequest) ?? newRequest;- },- });+ const requestQueue = await this.getRequestQueue();++ return this.enqueueLinksWithCrawlDepth(options, request, requestQueue);},addRequests: async (requests: RequestsLike, options: CrawlerAddRequestsOptions = {}) => {- const newRequestDepth = (request?.crawlDepth ?? 0) + 1;-- async function* injectDepth() {- for await (const rq of requests) {- if (typeof rq === 'string') {- yield { url: rq, crawlDepth: newRequestDepth };- } else {- rq.crawlDepth ??= newRequestDepth;- yield rq;- }- }- }+ const newCrawlDepth = request.crawlDepth + 1;+ const requestsGenerator = this.addCrawlDepthRequestGenerator(requests, newCrawlDepth);- return this.addRequests(injectDepth(), options);+ return this.addRequests(requestsGenerator, options);},pushData: this.pushData.bind(this),useState: this.useState.bind(this),- sendRequest: createSendRequest(this.httpClient, request!, session, () => crawlingContext.proxyInfo?.url),+ sendRequest: createSendRequest(this.httpClient, request, session, () => crawlingContext.proxyInfo?.url),getKeyValueStore: async (idOrName?: string) => KeyValueStore.open(idOrName, { config: this.config }),};@@ -1686,6 +1658,61 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext}}+ /**+ * Wrapper around the crawling context's `enqueueLinks` method:+ * - Injects `crawlDepth` to each request being added based on the crawling context request.+ * - Provides defaults for the `enqueueLinks` options based on the crawler configuration.+ * - These options can be overridden by the user.+ * @internal+ */+ protected async enqueueLinksWithCrawlDepth(+ options: SetRequired<EnqueueLinksOptions, 'urls'>,+ request: Request<Dictionary>,+ requestQueue: RequestProvider,+ ): Promise<BatchAddRequestsResult> {+ const transformRequestFunctionWrapper: RequestTransform = (newRequest) => {+ newRequest.crawlDepth = request.crawlDepth + 1;++ if (this.maxCrawlDepth !== undefined && newRequest.crawlDepth > this.maxCrawlDepth) {+ newRequest.skippedReason = 'depth';+ return false;+ }++ // After injecting the crawlDepth, we call the user-provided transform function, if there is one.+ return options.transformRequestFunction?.(newRequest) ?? newRequest;+ };++ return enqueueLinks({+ requestQueue,+ robotsTxtFile: await this.getRobotsTxtFileForUrl(request!.url),+ onSkippedRequest: this.handleSkippedRequest,+ limit: this.calculateEnqueuedRequestLimit(options.limit),++ // Allow user options to override defaults set above ⤴+ ...options,++ transformRequestFunction: transformRequestFunctionWrapper,+ });+ }++ /**+ * Generator function that yields requests injected with the given crawl depth.+ * @internal+ */+ protected async *addCrawlDepthRequestGenerator(+ requests: RequestsLike,+ newRequestDepth: number,+ ): AsyncGenerator<Source, void, undefined> {+ for await (const request of requests) {+ if (typeof request === 'string') {+ yield { url: request, crawlDepth: newRequestDepth };+ } else {+ request.crawlDepth ??= newRequestDepth;+ yield request;+ }+ }+ }+/*** Run async callback with given timeout and retry. Returns the result of the callback.* @ignorediff --git a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.tsindex 7dfd634bf..3658dcb94 100644--- a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts+++ b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts@@ -4,6 +4,7 @@ import type {BaseHttpResponseData,EnqueueLinksOptions,GetUserDataFromRequest,+ RequestQueue,RestrictedCrawlingContext,RouterRoutes,StatisticPersistedState,@@ -12,7 +13,6 @@ import type {} from '@crawlee/core';import {Configuration,- enqueueLinks,RequestHandlerResult,RequestState,resolveBaseUrlForEnqueueLinksFiltering,@@ -677,26 +677,23 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler {userProvidedBaseUrl: options?.baseUrl,});- return await enqueueLinks({- limit: this.calculateEnqueuedRequestLimit(options.limit),- onSkippedRequest: this.handleSkippedRequest,- ...options,- baseUrl,- requestQueue: {- addRequestsBatched: async (requests) => {- await result.addRequests(requests);- return {- addedRequests: requests.map(({ uniqueKey, id }) => ({- uniqueKey,- requestId: id ?? '',- wasAlreadyPresent: false,- wasAlreadyHandled: false,- })),- waitForAllRequestsToBeAdded: Promise.resolve([]),- };- },- },- });+ const addRequestsBatched: RequestQueue['addRequestsBatched'] = async (requests: Request<Dictionary>[]) => {+ await result.addRequests(requests);++ return {+ addedRequests: requests.map(({ uniqueKey, id }) => ({+ uniqueKey,+ requestId: id ?? '',+ wasAlreadyPresent: false,+ wasAlreadyHandled: false,+ })),+ waitForAllRequestsToBeAdded: Promise.resolve([]),+ };+ };+ // We need to use a mock request queue implementation, in order to add the requests into our result object+ const mockRequestQueue = { addRequestsBatched } as RequestQueue;++ return await this.enqueueLinksWithCrawlDepth({ ...options, baseUrl }, request, mockRequestQueue);}private createLogProxy(log: Log, logs: LogProxyCall[]) {diff --git a/test/core/crawlers/adaptive_playwright_crawler.test.ts b/test/core/crawlers/adaptive_playwright_crawler.test.tsindex 650d14337..589fd88ab 100644--- a/test/core/crawlers/adaptive_playwright_crawler.test.ts+++ b/test/core/crawlers/adaptive_playwright_crawler.test.ts@@ -2,7 +2,12 @@ import type { Server } from 'node:http';import type { AddressInfo } from 'node:net';import { Configuration, type Dictionary, EventType, KeyValueStore } from '@crawlee/core';-import type { AdaptivePlaywrightCrawlerOptions, Request } from '@crawlee/playwright';+import type {+ AdaptivePlaywrightCrawlerContext,+ AdaptivePlaywrightCrawlerOptions,+ LoadedContext,+ Request,+} from '@crawlee/playwright';import { AdaptivePlaywrightCrawler, RenderingTypePredictor, RequestList } from '@crawlee/playwright';import { sleep } from 'crawlee';import express from 'express';@@ -229,6 +234,35 @@ describe('AdaptivePlaywrightCrawler', () => {expect(resultChecker).toHaveBeenCalledTimes(1);});+ test.each([['static'], ['clientOnly']] as const)(+ 'crawlingContext.addRequests() should add requests correctly (%s)',+ async (renderingType) => {+ const renderingTypePredictor = makeRiggedRenderingTypePredictor({+ detectionProbabilityRecommendation: 0,+ renderingType,+ });+ const url = new URL(`http://${HOSTNAME}:${port}`).toString();++ let requestContext: LoadedContext<AdaptivePlaywrightCrawlerContext> | undefined;+ const requestHandler: AdaptivePlaywrightCrawlerOptions['requestHandler'] = async (context) => {+ const isStartUrl = context.request.url === url;++ if (isStartUrl) await context.addRequests([`${url}/1`]);+ else requestContext = context;+ };++ const crawler = await makeOneshotCrawler(+ { requestHandler, renderingTypePredictor, maxRequestsPerCrawl: 10 },+ [],+ );++ await crawler.run([{ url, crawlDepth: 2 }]);++ assert(requestContext);+ expect(requestContext.request).toMatchObject({ url: `${url}/1`, crawlDepth: 3 });+ },+ );+describe('should enqueue links correctly', () => {test.each([['/static', 'static'],@@ -239,12 +273,12 @@ describe('AdaptivePlaywrightCrawler', () => {renderingType,});const url = new URL(`http://${HOSTNAME}:${port}${path}`);- const enqueuedUrls = new Set<string>();+ const enqueuedRequests: Request[] = [];const requestHandler: AdaptivePlaywrightCrawlerOptions['requestHandler'] = vi.fn(async ({ enqueueLinks, request }) => {if (request.label === 'enqueued-url') {- enqueuedUrls.add(request.url);+ enqueuedRequests.push(request);} else {await enqueueLinks({ label: 'enqueued-url' });}@@ -262,6 +296,7 @@ describe('AdaptivePlaywrightCrawler', () => {await crawler.run();+ const enqueuedUrls = Array.from(enqueuedRequests).map((request) => request.url);expect(new Set(enqueuedUrls)).toEqual(new Set([`http://${HOSTNAME}:${port}/static?q=1`,@@ -271,6 +306,12 @@ describe('AdaptivePlaywrightCrawler', () => {`http://${HOSTNAME}:${port}/static?q=5`,]),);++ expect(enqueuedRequests[0]).toMatchObject({+ url: enqueuedUrls[0],+ label: 'enqueued-url',+ crawlDepth: 1,+ });});});diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.tsindex 9c63293d3..4d3a75213 100644--- a/test/core/crawlers/basic_crawler.test.ts+++ b/test/core/crawlers/basic_crawler.test.ts@@ -9,6 +9,7 @@ import type {ErrorHandler,RequestHandler,RequestOptions,+ Source,} from '@crawlee/basic';import {BasicCrawler,@@ -27,6 +28,8 @@ import type { Dictionary } from '@crawlee/utils';import { sleep } from '@crawlee/utils';import express from 'express';import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator';+import type { SetRequired } from 'type-fest';+import type { Mock } from 'vitest';import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'vitest';import log from '@apify/log';@@ -202,49 +205,129 @@ describe('BasicCrawler', () => {]);});- describe('enqueueLinks should respect maxCrawlDepth', async () => {- const expectedUrls = ['https://example.com/', 'https://example.com/deep/', 'https://example.com/deep/deep/'];+ test('enqueueLinks should respect maxCrawlDepth', async () => {+ const processedUrls: string[] = [];- const runCrawlWithDepthLimit = async (enqueueLinksOptions?: EnqueueLinksOptions): Promise<string[]> => {- const processedUrls: string[] = [];+ const requestHandler: RequestHandler = async ({ request, enqueueLinks }) => {+ processedUrls.push(request.url);+ const url = new URL(request.url);+ url.pathname = `${url.pathname}deep/`;- const requestHandler: RequestHandler = async ({ request, enqueueLinks }) => {- processedUrls.push(request.url);- const url = new URL(request.url);- url.pathname = `${url.pathname}deep/`;+ await enqueueLinks({+ urls: [url.toString()],+ });+ };- await enqueueLinks({- urls: [url.toString()],- ...enqueueLinksOptions,- });+ const crawler = new BasicCrawler({+ maxCrawlDepth: 2,+ maxRequestsPerCrawl: 10, // safeguard against infinite loops+ requestHandler,+ });++ await crawler.run(['https://example.com/']);++ expect(processedUrls).toEqual([+ 'https://example.com/',+ 'https://example.com/deep/',+ 'https://example.com/deep/deep/',+ ]);+ });++ describe('enqueueLinksWithCrawlDepth()', () => {+ let onSkippedRequestMock: Mock;+ let addRequestsBatchedMock: Mock;+ let options: SetRequired<EnqueueLinksOptions, 'urls'>;+ let request: Request;+ let requestQueue: RequestQueue;++ type EnqueueLinksWrapperOptions = Parameters<BasicCrawler['enqueueLinksWithCrawlDepth']>;+ class TestCrawler extends BasicCrawler {+ public exposedEnqueueLinksWithCrawlDepth(...enqueueLinksOptions: EnqueueLinksWrapperOptions) {+ return this.enqueueLinksWithCrawlDepth(...enqueueLinksOptions);+ }+ }++ const crawler = new TestCrawler({ maxCrawlDepth: 3 });++ beforeEach(() => {+ addRequestsBatchedMock = vi.fn().mockImplementation(async () => ({}));+ onSkippedRequestMock = vi.fn();++ options = {+ urls: ['https://example.com/1/', 'https://example.com/2/'],+ onSkippedRequest: onSkippedRequestMock,};+ request = new Request({ url: 'https://example.com/', crawlDepth: 2 });+ requestQueue = {+ addRequestsBatched: addRequestsBatchedMock as RequestQueue['addRequestsBatched'],+ } as RequestQueue;+ });- const crawler = new BasicCrawler({- maxCrawlDepth: 2,- maxRequestsPerCrawl: 10, // safeguard against infinite loops- requestHandler,- });+ it('should generate requests with maxCrawlDepth', async () => {+ await crawler.exposedEnqueueLinksWithCrawlDepth(options, request, requestQueue);- await crawler.run(['https://example.com/']);+ const requests = addRequestsBatchedMock.mock.calls[0][0];+ expect(requests).toHaveLength(2);+ expect(requests[0]).toMatchObject({ url: 'https://example.com/1/', crawlDepth: 3 });+ expect(requests[1]).toMatchObject({ url: 'https://example.com/2/', crawlDepth: 3 });- return processedUrls;- };+ expect(onSkippedRequestMock).not.toBeCalled();+ });++ it('should skip requests with crawlDepth exceeding maxCrawlDepth', async () => {+ const requestWithMaxDepth = new Request({ url: 'https://example.com/', crawlDepth: 3 });+ await crawler.exposedEnqueueLinksWithCrawlDepth(options, requestWithMaxDepth, requestQueue);++ const requests = addRequestsBatchedMock.mock.calls[0][0];+ expect(requests).toHaveLength(0);- test("without user's transformRequestFunction", async () => {- const processedUrls = await runCrawlWithDepthLimit();- expect(processedUrls).toEqual(expectedUrls);+ const skippedRequests = onSkippedRequestMock.mock.calls.map((call) => call[0]);+ expect(skippedRequests).toHaveLength(2);+ expect(skippedRequests[0]).toStrictEqual({ url: 'https://example.com/1/', reason: 'depth' });+ expect(skippedRequests[1]).toStrictEqual({ url: 'https://example.com/2/', reason: 'depth' });});- test('while executing user provided transformRequestFunction', async () => {- const transformRequestFunction = vi.fn((request: RequestOptions) => request);+ it('should respect user provided transformRequestFunction', async () => {+ const transformRequestFunction = vi.fn((req: RequestOptions) => req);+ const optionsWithTransform = { ...options, transformRequestFunction };- const processedUrls = await runCrawlWithDepthLimit({ transformRequestFunction });- expect(processedUrls).toEqual(expectedUrls);+ await crawler.exposedEnqueueLinksWithCrawlDepth(optionsWithTransform, request, requestQueue);expect(transformRequestFunction).toHaveBeenCalled();++ const requests = addRequestsBatchedMock.mock.calls[0][0];+ expect(requests).toHaveLength(2);+ expect(requests[0]).toMatchObject({ url: 'https://example.com/1/', crawlDepth: 3 });});});+ it('addCrawlDepthRequestGenerator() should generate requests with maxCrawlDepth', async () => {+ type AddCrawlDepthWrapperOptions = Parameters<BasicCrawler['addCrawlDepthRequestGenerator']>;+ class TestCrawler extends BasicCrawler {+ public exposedAddCrawlDepthRequestGenerator(...enqueueLinksOptions: AddCrawlDepthWrapperOptions) {+ return this.addCrawlDepthRequestGenerator(...enqueueLinksOptions);+ }+ }++ const crawler = new TestCrawler();++ const requests = ['https://example.com/1/', { url: 'https://example.com/2/' }];+ const newCrawlDepth = 4;+ const addRequestsGenerator = crawler.exposedAddCrawlDepthRequestGenerator(requests, newCrawlDepth);++ const generatedRequests: Source[] = [];+ for await (const generatedRequest of addRequestsGenerator) {+ generatedRequests.push(generatedRequest);+ }++ expect(generatedRequests).toHaveLength(2);+ expect(generatedRequests[0].url).toBe('https://example.com/1/');+ expect(generatedRequests[0].crawlDepth).toBe(4);++ expect(generatedRequests[1].url).toBe('https://example.com/2/');+ expect(generatedRequests[1].crawlDepth).toBe(4);+ });+test('should correctly combine shorthand and full length options', async () => {const shorthandOptions = {minConcurrency: 123,