Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

refactor: Minor adjustments to AdaptivePlaywrightCrawler (#2912)

Jan Buchara4fa8381ab44
diff --git a/packages/core/src/crawlers/crawler_commons.ts b/packages/core/src/crawlers/crawler_commons.ts
index e92e70973..fdc7777ac 100644
--- a/packages/core/src/crawlers/crawler_commons.ts
+++ b/packages/core/src/crawlers/crawler_commons.ts
@@ -100,7 +100,7 @@ export interface RestrictedCrawlingContext<UserData extends Dictionary = Diction
*/
getKeyValueStore: (
idOrName?: string,
- ) => Promise<Pick<KeyValueStore, 'id' | 'name' | 'getValue' | 'getAutoSavedValue' | 'setValue'>>;
+ ) => Promise<Pick<KeyValueStore, 'id' | 'name' | 'getValue' | 'getAutoSavedValue' | 'setValue' | 'getPublicUrl'>>;
/**
* A preconfigured logger for the request handler.
@@ -303,6 +303,7 @@ export class RequestHandlerResult {
setValue: async (key, value, options) => {
this.setKeyValueStoreChangedValue(idOrName, key, value, options);
},
+ getPublicUrl: store.getPublicUrl.bind(store),
};
};
diff --git a/packages/playwright-crawler/src/index.ts b/packages/playwright-crawler/src/index.ts
index 8b2efafca..06c049034 100644
--- a/packages/playwright-crawler/src/index.ts
+++ b/packages/playwright-crawler/src/index.ts
@@ -2,6 +2,7 @@ export * from '@crawlee/browser';
export * from './internals/playwright-crawler';
export * from './internals/playwright-launcher';
export * from './internals/adaptive-playwright-crawler';
+export { RenderingTypePredictor } from './internals/utils/rendering-type-prediction';
export * as playwrightUtils from './internals/utils/playwright-utils';
export * as playwrightClickElements from './internals/enqueue-links/click-elements';
diff --git a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts
index 95284f23b..83d38bb86 100644
--- a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts
+++ b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts
@@ -1,11 +1,7 @@
-import {
- extractUrlsFromPage,
- type LoadedContext,
- type LoadedRequest,
- type Request,
- type RouterHandler,
-} from '@crawlee/browser';
+import type { BrowserHook, LoadedContext, LoadedRequest, Request, RouterHandler } from '@crawlee/browser';
+import { extractUrlsFromPage } from '@crawlee/browser';
import type {
+ BaseHttpResponseData,
GetUserDataFromRequest,
RestrictedCrawlingContext,
RouterRoutes,
@@ -18,11 +14,12 @@ import type { Awaitable, Dictionary } from '@crawlee/types';
import { type CheerioRoot, extractUrlsFromCheerio } from '@crawlee/utils';
import { type Cheerio, type Element, load } from 'cheerio';
import isEqual from 'lodash.isequal';
+import type { Page } from 'playwright';
import type { Log } from '@apify/log';
import { addTimeoutToPromise } from '@apify/timeout';
-import type { PlaywrightCrawlerOptions, PlaywrightCrawlingContext } from './playwright-crawler';
+import type { PlaywrightCrawlerOptions, PlaywrightCrawlingContext, PlaywrightGotoOptions } from './playwright-crawler';
import { PlaywrightCrawler } from './playwright-crawler';
import { type RenderingType, RenderingTypePredictor } from './utils/rendering-type-prediction';
@@ -88,7 +85,18 @@ class AdaptivePlaywrightCrawlerStatistics extends Statistics {
}
}
-export interface AdaptivePlaywrightCrawlerContext extends RestrictedCrawlingContext {
+export interface AdaptivePlaywrightCrawlerContext<UserData extends Dictionary = Dictionary>
+ extends RestrictedCrawlingContext<UserData> {
+ /**
+ * The HTTP response, either from the HTTP client or from the initial request from playwright's navigation.
+ */
+ response: BaseHttpResponseData;
+
+ /**
+ * Playwright Page object. If accessed in HTTP-only rendering, this will throw an error and make the AdaptivePlaywrightCrawlerContext retry the request in a browser.
+ */
+ page: Page;
+
/**
* Wait for an element matching the selector to appear and return a Cheerio object of matched elements.
* Timeout defaults to 5s.
@@ -125,8 +133,17 @@ export interface AdaptivePlaywrightCrawlerContext extends RestrictedCrawlingCont
parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
}
+interface AdaptiveHook
+ extends BrowserHook<
+ Pick<AdaptivePlaywrightCrawlerContext, 'id' | 'request' | 'session' | 'proxyInfo' | 'log'> & { page?: Page },
+ PlaywrightGotoOptions
+ > {}
+
export interface AdaptivePlaywrightCrawlerOptions
- extends Omit<PlaywrightCrawlerOptions, 'requestHandler' | 'handlePageFunction'> {
+ extends Omit<
+ PlaywrightCrawlerOptions,
+ 'requestHandler' | 'handlePageFunction' | 'preNavigationHooks' | 'postNavigationHooks'
+ > {
/**
* Function that is called to process each request.
*
@@ -140,6 +157,20 @@ export interface AdaptivePlaywrightCrawlerOptions
*/
requestHandler?: (crawlingContext: LoadedContext<AdaptivePlaywrightCrawlerContext>) => Awaitable<void>;
+ /**
+ * Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies.
+ * The function accepts a subset of the crawling context. If you attempt to access the `page` property during HTTP-only crawling,
+ * an exception will be thrown. If it's not caught, the request will be transparently retried in a browser.
+ */
+ preNavigationHooks?: AdaptiveHook[];
+
+ /**
+ * Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
+ * The function accepts a subset of the crawling context. If you attempt to access the `page` property during HTTP-only crawling,
+ * an exception will be thrown. If it's not caught, the request will be transparently retried in a browser.
+ */
+ postNavigationHooks?: AdaptiveHook[];
+
/**
* Specifies the frequency of rendering type detection checks - 0.1 means roughly 10% of requests.
* Defaults to 0.1 (so 10%).
@@ -165,6 +196,12 @@ export interface AdaptivePlaywrightCrawlerOptions
* A custom rendering type predictor
*/
renderingTypePredictor?: Pick<RenderingTypePredictor, 'predict' | 'storeResult'>;
+
+ /**
+ * Prevent direct access to storage in request handlers (only allow using context helpers).
+ * Defaults to `true`
+ */
+ preventDirectStorageAccess?: boolean;
}
const proxyLogMethods = [
@@ -214,6 +251,7 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler {
private renderingTypePredictor: NonNullable<AdaptivePlaywrightCrawlerOptions['renderingTypePredictor']>;
private resultChecker: NonNullable<AdaptivePlaywrightCrawlerOptions['resultChecker']>;
private resultComparator: NonNullable<AdaptivePlaywrightCrawlerOptions['resultComparator']>;
+ private preventDirectStorageAccess: boolean;
declare readonly stats: AdaptivePlaywrightCrawlerStatistics;
/**
@@ -235,6 +273,7 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler {
resultChecker,
resultComparator,
statisticsOptions,
+ preventDirectStorageAccess = true,
...rest
} = options;
@@ -265,12 +304,12 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler {
config,
...statisticsOptions,
});
+
+ this.preventDirectStorageAccess = preventDirectStorageAccess;
}
protected override async _runRequestHandler(crawlingContext: PlaywrightCrawlingContext): Promise<void> {
- const url = new URL(crawlingContext.request.loadedUrl ?? crawlingContext.request.url);
-
- const renderingTypePrediction = this.renderingTypePredictor.predict(url, crawlingContext.request.label);
+ const renderingTypePrediction = this.renderingTypePredictor.predict(crawlingContext.request);
const shouldDetectRenderingType = Math.random() < renderingTypePrediction.detectionProbabilityRecommendation;
if (!shouldDetectRenderingType) {
@@ -338,7 +377,7 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler {
})();
crawlingContext.log.debug(`Detected rendering type ${detectionResult} for ${crawlingContext.request.url}`);
- this.renderingTypePredictor.storeResult(url, crawlingContext.request.label, detectionResult);
+ this.renderingTypePredictor.storeResult(crawlingContext.request, detectionResult);
}
}
@@ -384,9 +423,11 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler {
return async (playwrightContext: PlaywrightCrawlingContext) =>
withCheckedStorageAccess(
() => {
- throw new Error(
- 'Directly accessing storage in a request handler is not allowed in AdaptivePlaywrightCrawler',
- );
+ if (this.preventDirectStorageAccess) {
+ throw new Error(
+ 'Directly accessing storage in a request handler is not allowed in AdaptivePlaywrightCrawler',
+ );
+ }
},
() =>
this.adaptiveRequestHandler({
@@ -394,7 +435,16 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler {
session: crawlingContext.session,
proxyInfo: crawlingContext.proxyInfo,
request: crawlingContext.request as LoadedRequest<Request>,
+ response: {
+ url: crawlingContext.response!.url(),
+ statusCode: crawlingContext.response!.status(),
+ headers: crawlingContext.response!.headers(),
+ trailers: {},
+ complete: true,
+ redirectUrls: [],
+ },
log: crawlingContext.log,
+ page: crawlingContext.page,
querySelector: async (selector, timeoutMs = 5_000) => {
const locator = playwrightContext.page.locator(selector).first();
await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
@@ -456,27 +506,51 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler {
const result = new RequestHandlerResult(this.config, AdaptivePlaywrightCrawler.CRAWLEE_STATE_KEY);
const logs: LogProxyCall[] = [];
- const response = await crawlingContext.sendRequest({});
- const loadedUrl = response.url;
- crawlingContext.request.loadedUrl = loadedUrl;
- const $ = load(response.body);
+ const pageGotoOptions = { timeout: this.navigationTimeoutMillis }; // Irrelevant, but required by BrowserCrawler
try {
await withCheckedStorageAccess(
() => {
- throw new Error(
- 'Directly accessing storage in a request handler is not allowed in AdaptivePlaywrightCrawler',
- );
+ if (this.preventDirectStorageAccess) {
+ throw new Error(
+ 'Directly accessing storage in a request handler is not allowed in AdaptivePlaywrightCrawler',
+ );
+ }
},
async () =>
addTimeoutToPromise(
- async () =>
- this.adaptiveRequestHandler({
+ async () => {
+ const hookContext: Parameters<AdaptiveHook>[0] = {
id: crawlingContext.id,
session: crawlingContext.session,
proxyInfo: crawlingContext.proxyInfo,
- request: crawlingContext.request as LoadedRequest<Request>,
+ request: crawlingContext.request,
log: this.createLogProxy(crawlingContext.log, logs),
+ };
+
+ await this._executeHooks(
+ this.preNavigationHooks,
+ {
+ ...hookContext,
+ get page(): Page {
+ throw new Error('Page object was used in HTTP-only pre-navigation hook');
+ },
+ } as PlaywrightCrawlingContext, // This is safe because `executeHooks` just passes the context to the hooks which accept the partial context
+ pageGotoOptions,
+ );
+
+ const response = await crawlingContext.sendRequest({});
+ const loadedUrl = response.url;
+ crawlingContext.request.loadedUrl = loadedUrl;
+ const $ = load(response.body);
+
+ await this.adaptiveRequestHandler({
+ ...hookContext,
+ request: crawlingContext.request as LoadedRequest<Request>,
+ response,
+ get page(): Page {
+ throw new Error('Page object was used in HTTP-only request handler');
+ },
async querySelector(selector, _timeoutMs?: number) {
return $(selector) as Cheerio<Element>;
},
@@ -514,7 +588,10 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler {
return this.allowStorageAccess(result.useState)(defaultValue);
},
getKeyValueStore: this.allowStorageAccess(result.getKeyValueStore),
- }),
+ });
+
+ await this._executeHooks(this.postNavigationHooks, crawlingContext, pageGotoOptions);
+ },
this.requestHandlerTimeoutInnerMillis,
'Request handler timed out',
),
diff --git a/packages/playwright-crawler/src/internals/utils/rendering-type-prediction.ts b/packages/playwright-crawler/src/internals/utils/rendering-type-prediction.ts
index ee837341e..68515e1e8 100644
--- a/packages/playwright-crawler/src/internals/utils/rendering-type-prediction.ts
+++ b/packages/playwright-crawler/src/internals/utils/rendering-type-prediction.ts
@@ -1,3 +1,4 @@
+import type { Request } from '@crawlee/core';
import LogisticRegression from 'ml-logistic-regression';
import { Matrix } from 'ml-matrix';
import stringComparison from 'string-comparison';
@@ -52,15 +53,17 @@ export class RenderingTypePredictor {
/**
* Predict the rendering type for a given URL and request label.
*/
- public predict(
- url: URL,
- label: string | undefined,
- ): { renderingType: RenderingType; detectionProbabilityRecommendation: number } {
+ public predict({ url, loadedUrl, label }: Request): {
+ renderingType: RenderingType;
+ detectionProbabilityRecommendation: number;
+ } {
if (this.logreg.classifiers.length === 0) {
return { renderingType: 'clientOnly', detectionProbabilityRecommendation: 1 };
}
- const urlFeature = new Matrix([this.calculateFeatureVector(urlComponents(url), label)]);
+ const predictionUrl = new URL(loadedUrl ?? url);
+
+ const urlFeature = new Matrix([this.calculateFeatureVector(urlComponents(predictionUrl), label)]);
const [prediction] = this.logreg.predict(urlFeature);
const scores = [
this.logreg.classifiers[0].testScores(urlFeature),
@@ -79,7 +82,9 @@ export class RenderingTypePredictor {
/**
* Store the rendering type for a given URL and request label. This updates the underlying prediction model, which may be costly.
*/
- public storeResult(url: URL, label: string | undefined, renderingType: RenderingType) {
+ public storeResult({ url, loadedUrl, label }: Request, renderingType: RenderingType) {
+ const resultUrl = new URL(loadedUrl ?? url);
+
if (!this.renderingTypeDetectionResults.has(renderingType)) {
this.renderingTypeDetectionResults.set(renderingType, new Map());
}
@@ -88,7 +93,7 @@ export class RenderingTypePredictor {
this.renderingTypeDetectionResults.get(renderingType)!.set(label, []);
}
- this.renderingTypeDetectionResults.get(renderingType)!.get(label)!.push(urlComponents(url));
+ this.renderingTypeDetectionResults.get(renderingType)!.get(label)!.push(urlComponents(resultUrl));
this.retrain();
}
diff --git a/test/core/crawlers/adaptive_playwright_crawler.test.ts b/test/core/crawlers/adaptive_playwright_crawler.test.ts
index 531e5a9b5..3aef3d8ec 100644
--- a/test/core/crawlers/adaptive_playwright_crawler.test.ts
+++ b/test/core/crawlers/adaptive_playwright_crawler.test.ts
@@ -2,7 +2,7 @@ import type { Server } from 'node:http';
import type { AddressInfo } from 'node:net';
import { type Dictionary, KeyValueStore } from '@crawlee/core';
-import type { AdaptivePlaywrightCrawlerOptions } from '@crawlee/playwright';
+import type { AdaptivePlaywrightCrawlerOptions, Request } from '@crawlee/playwright';
import { AdaptivePlaywrightCrawler, RequestList } from '@crawlee/playwright';
import express from 'express';
import { startExpressAppPromise } from 'test/shared/_helper';
@@ -96,8 +96,8 @@ describe('AdaptivePlaywrightCrawler', () => {
detectionProbabilityRecommendation: number;
renderingType: 'clientOnly' | 'static';
}) => ({
- predict: vi.fn((_url: URL) => prediction),
- storeResult: vi.fn((_url: URL, _label: string | unknown, _renderingType: string) => {}),
+ predict: vi.fn((_request: Request) => prediction),
+ storeResult: vi.fn((_request: Request, _renderingType: string) => {}),
});
describe('should detect page rendering type', () => {
@@ -131,8 +131,12 @@ describe('AdaptivePlaywrightCrawler', () => {
await crawler.run();
// Check the detection result
- expect(renderingTypePredictor.predict).toHaveBeenCalledWith(url, undefined);
- expect(renderingTypePredictor.storeResult).toHaveBeenCalledWith(url, undefined, expectedType);
+ expect(renderingTypePredictor.predict).toHaveBeenCalledOnce();
+ expect(renderingTypePredictor.predict.mock.lastCall?.[0]).toMatchObject({ url, label: undefined });
+
+ expect(renderingTypePredictor.storeResult).toHaveBeenCalledOnce();
+ expect(renderingTypePredictor.storeResult.mock.lastCall?.[0]).toMatchObject({ url, label: undefined });
+ expect(renderingTypePredictor.storeResult.mock.lastCall?.[1]).toEqual(expectedType);
// Check if the request handler was called twice
expect(requestHandler).toHaveBeenCalledTimes(2);
@@ -159,7 +163,9 @@ describe('AdaptivePlaywrightCrawler', () => {
await crawler.run();
- expect(renderingTypePredictor.predict).toHaveBeenCalledWith(url, undefined);
+ expect(renderingTypePredictor.predict).toHaveBeenCalledOnce();
+ expect(renderingTypePredictor.predict.mock.lastCall?.[0]).toMatchObject({ url, label: undefined });
+
expect(renderingTypePredictor.storeResult).not.toHaveBeenCalled();
});