Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

feat: Finalize AdaptivePlaywrightCrawler (#3907)

Jan Buchar2b07f0296a0a
diff --git a/docs/public-api/crawlee-playwright.api.md b/docs/public-api/crawlee-playwright.api.md
index f0cdc3770..5f1a3c4fa 100644
--- a/docs/public-api/crawlee-playwright.api.md
+++ b/docs/public-api/crawlee-playwright.api.md
@@ -73,6 +73,7 @@ export class AdaptivePlaywrightCrawler<ExtendedContext extends AdaptivePlaywrigh
readonly response: Response;
readonly page: Page;
readonly querySelector: AdaptivePlaywrightCrawlerContext["querySelector"];
+ readonly querySelectorAll: AdaptivePlaywrightCrawlerContext["querySelectorAll"];
readonly waitForSelector: AdaptivePlaywrightCrawlerContext["waitForSelector"];
readonly parseWithCheerio: AdaptivePlaywrightCrawlerContext["parseWithCheerio"];
}>;
@@ -94,6 +95,7 @@ export interface AdaptivePlaywrightCrawlerContext<UserData extends Dictionary_2
page: Page;
parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
querySelector(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
+ querySelectorAll(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
// (undocumented)
request: LoadedRequest<Request_3<UserData>>;
response: Response;
@@ -186,6 +188,9 @@ interface EnqueueLinksByClickingElementsOptions {
waitForPageIdleSecs?: number;
}
+// @public
+export function fullResultComparator(resultA: RequestHandlerResult, resultB: RequestHandlerResult): boolean;
+
// @public
function gotoExtended(page: Page, request: Request_3, gotoOptions?: PlaywrightDirectNavigationOptions): Promise<Response_2 | null>;
diff --git a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts
index fcbbce1e9..5e2617df7 100644
--- a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts
+++ b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts
@@ -125,11 +125,17 @@ export interface AdaptivePlaywrightCrawlerContext<
page: Page;
/**
- * Wait for an element matching the selector to appear and return a Cheerio object of matched elements.
+ * Wait for an element matching the selector to appear and return a Cheerio object of the first matched element.
* Timeout defaults to 5s.
*/
querySelector(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
+ /**
+ * Wait for an element matching the selector to appear and return a Cheerio object of all matched elements.
+ * Timeout defaults to 5s.
+ */
+ querySelectorAll(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
+
/**
* Wait for an element matching the selector to appear.
* Timeout defaults to 5s.
@@ -233,6 +239,8 @@ export interface AdaptivePlaywrightCrawlerOptions<
* If it returns 'inconclusive', the detection result won't be used.
* If no result comparator is specified, but there is a `resultChecker`, any site where the `resultChecker` returns true is considered static.
* If neither `resultComparator` nor `resultChecker` are specified, a deep comparison of returned dataset items is used as a default.
+ *
+ * For a stricter, ready-made comparator that also takes enqueued requests and key-value store changes into account, see {@apilink fullResultComparator}.
*/
resultComparator?: (
resultA: RequestHandlerResult,
@@ -432,6 +440,9 @@ export class AdaptivePlaywrightCrawler<
get querySelector(): AdaptivePlaywrightCrawlerContext['querySelector'] {
throw new Error(errorMessage('querySelector'));
},
+ get querySelectorAll(): AdaptivePlaywrightCrawlerContext['querySelectorAll'] {
+ throw new Error(errorMessage('querySelectorAll'));
+ },
get waitForSelector(): AdaptivePlaywrightCrawlerContext['waitForSelector'] {
throw new Error(errorMessage('waitForSelector'));
},
@@ -454,6 +465,9 @@ export class AdaptivePlaywrightCrawler<
throw new Error('Page object was used in HTTP-only request handler');
},
async querySelector(selector: string) {
+ return cheerioContext.$(selector).first();
+ },
+ async querySelectorAll(selector: string) {
return cheerioContext.$(selector);
},
enqueueLinks: async (options: EnqueueLinksOptions = {}) => {
@@ -493,6 +507,13 @@ export class AdaptivePlaywrightCrawler<
await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
const $ = await playwrightContext.parseWithCheerio();
+ return $(selector).first() as Cheerio<any>;
+ },
+ async querySelectorAll(selector: string, timeoutMs = 5000) {
+ const locator = playwrightContext.page.locator(selector).first();
+ await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
+ const $ = await playwrightContext.parseWithCheerio();
+
return $(selector) as Cheerio<any>;
},
enqueueLinks: async (options: EnqueueLinksOptions = {}, timeoutMs = 5000) => {
@@ -823,3 +844,35 @@ export function createAdaptivePlaywrightRouter<
export function createAdaptivePlaywrightRouter(routesOrSchemas?: any): any {
return Router.create(routesOrSchemas);
}
+
+/**
+ * An opt-in {@apilink AdaptivePlaywrightCrawlerOptions.resultComparator|`resultComparator`} that considers two
+ * request handler results equal only if *all* of their observable effects match - the pushed dataset items, the
+ * enqueued requests, and the key-value store changes. This is stricter than the default comparator, which only
+ * compares dataset items.
+ *
+ * **Beware:** enqueued URLs are compared exactly. The same page rendered in a browser and via plain HTTP often
+ * yields links that differ only in tracking query parameters, for example:
+ * - `https://sdk.apify.com/docs/guides/getting-started`
+ * - `https://sdk.apify.com/docs/guides/getting-started?__hsfp=1136113150&__hssc=7591405.1.173549427712`
+ *
+ * Such links are treated as *different*, which will make the crawler favor browser rendering for those pages.
+ *
+ * **Example usage:**
+ * ```ts
+ * const crawler = new AdaptivePlaywrightCrawler({
+ * resultComparator: fullResultComparator,
+ * async requestHandler({ pushData, enqueueLinks }) {
+ * // ...
+ * },
+ * });
+ * ```
+ */
+export function fullResultComparator(resultA: RequestHandlerResult, resultB: RequestHandlerResult): boolean {
+ return (
+ isDeepStrictEqual(resultA.datasetItems, resultB.datasetItems) &&
+ isDeepStrictEqual(resultA.enqueuedUrls, resultB.enqueuedUrls) &&
+ isDeepStrictEqual(resultA.enqueuedUrlLists, resultB.enqueuedUrlLists) &&
+ isDeepStrictEqual(resultA.keyValueStoreChanges, resultB.keyValueStoreChanges)
+ );
+}
diff --git a/packages/playwright-crawler/src/internals/utils/rendering-type-prediction.ts b/packages/playwright-crawler/src/internals/utils/rendering-type-prediction.ts
index db8639f32..d750d33c3 100644
--- a/packages/playwright-crawler/src/internals/utils/rendering-type-prediction.ts
+++ b/packages/playwright-crawler/src/internals/utils/rendering-type-prediction.ts
@@ -19,11 +19,20 @@ const calculateUrlSimilarity = (a: URLComponents, b: URLComponents): number | un
return 0;
}
- for (let i = 1; i < Math.max(a.length, b.length); i++) {
+ const maxLength = Math.max(a.length, b.length);
+
+ // Only the hostname is present (no path components to compare) - the hosts already match.
+ if (maxLength <= 1) {
+ return 1;
+ }
+
+ for (let i = 1; i < maxLength; i++) {
values.push(stringComparison.jaroWinkler.similarity(a[i] ?? '', b[i] ?? '') > 0.8 ? 1 : 0);
}
- return sum(values) / Math.max(a.length, b.length);
+ // The first component (index 0, the hostname) is excluded from the comparison above,
+ // so it must also be excluded from the denominator of the weighted average.
+ return sum(values) / (maxLength - 1);
};
const sum = (values: number[]) => values.reduce((acc, value) => acc + value);
diff --git a/test/core/crawlers/adaptive_playwright_crawler.test.ts b/test/core/crawlers/adaptive_playwright_crawler.test.ts
index 28b378bed..1ed884a90 100644
--- a/test/core/crawlers/adaptive_playwright_crawler.test.ts
+++ b/test/core/crawlers/adaptive_playwright_crawler.test.ts
@@ -22,6 +22,7 @@ import {
AdaptivePlaywrightCrawler,
BasicCrawler,
createAdaptivePlaywrightRouter,
+ fullResultComparator,
RenderingTypePredictor,
RequestList,
RequestValidationError,
@@ -212,6 +213,89 @@ describe('AdaptivePlaywrightCrawler', () => {
});
});
+ describe('querySelector and querySelectorAll', () => {
+ test.each([
+ ['/static', 'static'],
+ ['/dynamic', 'clientOnly'],
+ ] as const)('return first vs. all matched elements (%s)', async (path, renderingType) => {
+ const renderingTypePredictor = makeRiggedRenderingTypePredictor({
+ detectionProbabilityRecommendation: 0,
+ renderingType,
+ });
+ const url = new URL(`http://${HOSTNAME}:${port}${path}`);
+
+ const requestHandler: AdaptivePlaywrightCrawlerOptions['requestHandler'] = vi.fn(
+ async ({ pushData, querySelector, querySelectorAll }) => {
+ const first = await querySelector('a');
+ const all = await querySelectorAll('a');
+ await pushData({
+ firstCount: first.length,
+ firstText: first.text(),
+ allCount: all.length,
+ });
+ },
+ );
+
+ const crawler = await makeOneshotCrawler(
+ {
+ requestHandler,
+ renderingTypePredictor,
+ },
+ [url.toString()],
+ );
+
+ await crawler.run();
+
+ // `querySelector` returns only the first match, `querySelectorAll` returns the whole collection.
+ expect((await Dataset.getData()).items).toEqual([{ firstCount: 1, firstText: 'Link 1', allCount: 5 }]);
+ });
+ });
+
+ describe('fullResultComparator', () => {
+ // The `/dynamic` page renders its links only after JS runs, so the static (plain HTTP) run enqueues
+ // no links while the browser run enqueues five. The pushed dataset item is constant in both runs.
+ const makeCrawler = async (options: Partial<AdaptivePlaywrightCrawlerOptions>) => {
+ const renderingTypePredictor = makeRiggedRenderingTypePredictor({
+ detectionProbabilityRecommendation: 1, // always run detection
+ renderingType: 'clientOnly',
+ });
+ const url = new URL(`http://${HOSTNAME}:${port}/dynamic`);
+
+ const requestHandler: AdaptivePlaywrightCrawlerOptions['requestHandler'] = vi.fn(
+ async ({ pushData, enqueueLinks }) => {
+ await pushData({ heading: 'Heading' }); // identical in both runs
+ await enqueueLinks(); // differs between static (0 links) and browser (5 links)
+ },
+ );
+
+ const crawler = await makeOneshotCrawler({ requestHandler, renderingTypePredictor, ...options }, [
+ url.toString(),
+ ]);
+
+ return { crawler, renderingTypePredictor };
+ };
+
+ test('default comparator ignores enqueued links and detects the page as static', async () => {
+ const { crawler, renderingTypePredictor } = await makeCrawler({});
+
+ await crawler.run();
+
+ expect(renderingTypePredictor.storeResult).toHaveBeenCalledOnce();
+ expect(renderingTypePredictor.storeResult.mock.lastCall?.[1]).toEqual('static');
+ });
+
+ test('fullResultComparator takes enqueued links into account and detects the page as clientOnly', async () => {
+ const { crawler, renderingTypePredictor } = await makeCrawler({
+ resultComparator: fullResultComparator,
+ });
+
+ await crawler.run();
+
+ expect(renderingTypePredictor.storeResult).toHaveBeenCalledOnce();
+ expect(renderingTypePredictor.storeResult.mock.lastCall?.[1]).toEqual('clientOnly');
+ });
+ });
+
test.each([['static'], ['clientOnly']] as const)(
'should replay request handler logs (%s)',
async (renderingType) => {