Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

feat: adaptive crawler result checks get request context, rejections feed the predictor (#4272)

Jan Buchar02899241d570
diff --git a/docs/public-api/crawlee-playwright.api.md b/docs/public-api/crawlee-playwright.api.md
index 648bcd8c3..a6ac56bae 100644
--- a/docs/public-api/crawlee-playwright.api.md
+++ b/docs/public-api/crawlee-playwright.api.md
@@ -101,8 +101,8 @@ export interface AdaptivePlaywrightCrawlerOptions<ContextExtension = Dictionary<
preNavigationHooks?: AdaptiveHook<ContextExtension>[];
renderingTypeDetectionRatio?: number;
renderingTypePredictor?: IRenderingTypePredictor;
- resultChecker?: (result: StorageTransactionView) => boolean;
- resultComparator?: (resultA: StorageTransactionView, resultB: StorageTransactionView) => boolean | 'equal' | 'different' | 'inconclusive';
+ resultChecker?: (result: StorageTransactionView, context: AdaptiveResultContext) => boolean;
+ resultComparator?: (resultA: StorageTransactionView, resultB: StorageTransactionView, context: AdaptiveResultContext) => boolean | 'equal' | 'different' | 'inconclusive';
shouldPropagateError?: (error: Error, context: PlaywrightCrawlingContext) => Awaitable<boolean>;
}
@@ -124,6 +124,11 @@ type AdaptivePostNavigationHook<ContextExtension = Dictionary<never>> = BrowserH
request: LoadedRequest<CrawlingRequest>;
}, ContextExtension>;
+// @public
+export interface AdaptiveResultContext {
+ request: CrawlingRequest;
+}
+
// Not exported by the entry point; reachable only as a referenced type.
// @public (undocumented)
const adaptiveStatisticStateSchema: z.ZodObject<{
diff --git a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts
index 2327cbce5..336828cdb 100644
--- a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts
+++ b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts
@@ -164,6 +164,12 @@ type AdaptivePostNavigationHook<ContextExtension = Dictionary<never>> = BrowserH
ContextExtension
>;
+/** Passed to {@apilink AdaptivePlaywrightCrawlerOptions.resultChecker} and `resultComparator` alongside the results. */
+export interface AdaptiveResultContext {
+ /** The request whose handler produced the results. */
+ request: CrawlingRequest;
+}
+
export interface AdaptivePlaywrightCrawlerOptions<
ContextExtension = Dictionary<never>,
ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext & ContextExtension,
@@ -217,7 +223,7 @@ export interface AdaptivePlaywrightCrawlerOptions<
* If it returns false, the request is retried in a browser.
* If no callback is specified, every result is considered valid.
*/
- resultChecker?: (result: StorageTransactionView) => boolean;
+ resultChecker?: (result: StorageTransactionView, context: AdaptiveResultContext) => boolean;
/**
* An optional callback that decides whether an error thrown during the plain HTTP request handler
@@ -245,6 +251,7 @@ export interface AdaptivePlaywrightCrawlerOptions<
resultComparator?: (
resultA: StorageTransactionView,
resultB: StorageTransactionView,
+ context: AdaptiveResultContext,
) => boolean | 'equal' | 'different' | 'inconclusive';
/**
@@ -453,7 +460,8 @@ export class AdaptivePlaywrightCrawler<
if (resultComparator !== undefined) {
this.#resultComparator = resultComparator;
} else if (resultChecker !== undefined) {
- this.#resultComparator = (resultA, resultB) => this.#resultChecker(resultA) && this.#resultChecker(resultB);
+ this.#resultComparator = (resultA, resultB, context) =>
+ this.#resultChecker(resultA, context) && this.#resultChecker(resultB, context);
} else {
this.#resultComparator = (resultA, resultB) => {
return (
@@ -680,6 +688,10 @@ export class AdaptivePlaywrightCrawler<
const transactions: StorageTransaction[] = [];
try {
+ // A result the `resultChecker` rejected is as good as a detection saying the page needs a browser - but
+ // only once the browser run has succeeded. A thrown error says nothing about rendering.
+ let httpResultRejected = false;
+
if (renderingTypePrediction.renderingType === 'static' && !shouldDetectRenderingType) {
crawlingContext.log.debug(`Running HTTP-only request handler for ${crawlingContext.request.url}`);
this.statistics.state.httpOnlyRequestHandlerRuns++;
@@ -691,7 +703,7 @@ export class AdaptivePlaywrightCrawler<
transactions,
);
- if (plainHTTPRun.ok && this.#resultChecker(plainHTTPRun.result)) {
+ if (plainHTTPRun.ok && this.#resultChecker(plainHTTPRun.result, { request: crawlingContext.request })) {
crawlingContext.log.debug(`HTTP-only request handler succeeded for ${crawlingContext.request.url}`);
plainHTTPRun.logs?.forEach(([log, method, ...args]) => log[method](...(args as [any, any])));
await plainHTTPRun.result.commit();
@@ -722,6 +734,7 @@ export class AdaptivePlaywrightCrawler<
`HTTP-only request handler returned a suspicious result for ${crawlingContext.request.url}`,
);
this.statistics.state.renderingTypeMispredictions++;
+ httpResultRejected = true;
}
}
@@ -765,6 +778,11 @@ export class AdaptivePlaywrightCrawler<
browserRun.logs?.forEach(([log, method, ...args]) => log[method](...(args as [any, any])));
await browserRun.result.commit();
+ // Past the shutdown fence, `teardown()` has already drained - and possibly torn down the predictor.
+ if (httpResultRejected && !this.#shutDown) {
+ this.#storeRenderingType(crawlingContext.request, 'clientOnly');
+ }
+
if (shouldDetectRenderingType) {
const detectionPromise = (async () => {
crawlingContext.log.debug(`Detecting rendering type for ${crawlingContext.request.url}`);
@@ -782,7 +800,9 @@ export class AdaptivePlaywrightCrawler<
return 'clientOnly';
}
- const comparisonResult = this.#resultComparator(plainHTTPRun.result, browserRun.result);
+ const comparisonResult = this.#resultComparator(plainHTTPRun.result, browserRun.result, {
+ request: crawlingContext.request,
+ });
if (comparisonResult === true || comparisonResult === 'equal') {
return 'static';
}
@@ -799,24 +819,7 @@ export class AdaptivePlaywrightCrawler<
);
if (detectionResult !== undefined) {
- // Deliberately not awaited: a predictor that persists asynchronously gets to keep
- // batching its writes, and the drain below catches whatever is still pending.
- const stored = this.#renderingTypePredictor.value.storeResult(
- crawlingContext.request,
- detectionResult,
- );
-
- if (stored !== undefined) {
- // Nothing downstream awaits this, so a failed write would otherwise be silent.
- void this.#trackDetection(
- Promise.resolve(stored).catch((error) =>
- this.log.exception(
- error as Error,
- `Failed to store the rendering type detection result for ${crawlingContext.request.url}`,
- ),
- ),
- );
- }
+ this.#storeRenderingType(crawlingContext.request, detectionResult);
}
})();
@@ -831,6 +834,24 @@ export class AdaptivePlaywrightCrawler<
}
}
+ #storeRenderingType(request: CrawlingContext['request'], renderingType: RenderingType): void {
+ // Deliberately not awaited: a predictor that persists asynchronously gets to keep batching its writes,
+ // and `drainRenderingDetections()` catches whatever is still pending.
+ const stored = this.#renderingTypePredictor.value.storeResult(request, renderingType);
+
+ if (stored !== undefined) {
+ // Nothing downstream awaits this, so a failed write would otherwise be silent.
+ void this.#trackDetection(
+ Promise.resolve(stored).catch((error) =>
+ this.log.exception(
+ error as Error,
+ `Failed to store the rendering type detection result for ${request.url}`,
+ ),
+ ),
+ );
+ }
+ }
+
async #enqueueLinks(
urls: readonly string[],
options: EnqueueLinksOptions,
diff --git a/packages/playwright-crawler/src/internals/utils/rendering-type-prediction.ts b/packages/playwright-crawler/src/internals/utils/rendering-type-prediction.ts
index 29030539c..644b98187 100644
--- a/packages/playwright-crawler/src/internals/utils/rendering-type-prediction.ts
+++ b/packages/playwright-crawler/src/internals/utils/rendering-type-prediction.ts
@@ -67,7 +67,8 @@ export interface IRenderingTypePredictor {
}>;
/**
- * Report a detected rendering type, so that future predictions can take it into account.
+ * Report a rendering type established by a detection, or `clientOnly` when the `resultChecker` rejected an
+ * HTTP-only result, so that future predictions can take it into account.
*
* The crawler does not await this per detection - it collects the returned promises and drains them in
* `teardown()`, so an implementation that persists results asynchronously can keep batching its writes
diff --git a/test/core/crawlers/adaptive-playwright-crawler.test.ts b/test/core/crawlers/adaptive-playwright-crawler.test.ts
index fd4e24616..f03fc1e7e 100644
--- a/test/core/crawlers/adaptive-playwright-crawler.test.ts
+++ b/test/core/crawlers/adaptive-playwright-crawler.test.ts
@@ -463,12 +463,41 @@ describe('AdaptivePlaywrightCrawler', () => {
expect(requestHandler).toHaveBeenCalledTimes(2);
expect(resultChecker).toHaveBeenCalledTimes(1);
+ expect(resultChecker).toHaveBeenCalledWith(expect.anything(), {
+ request: expect.objectContaining({ url: url.toString() }),
+ });
expect(crawler.statistics.state).toMatchObject({
httpOnlyRequestHandlerRuns: 1,
renderingTypeMispredictions: 1,
browserRequestHandlerRuns: 1,
});
+ expect(renderingTypePredictor.storeResult).toHaveBeenCalledOnce();
+ expect(renderingTypePredictor.storeResult).toHaveBeenCalledWith(
+ expect.objectContaining({ url: url.toString() }),
+ 'clientOnly',
+ );
+ });
+
+ test('should not store a rendering type when the HTTP-only handler throws', async () => {
+ const renderingTypePredictor = makeRiggedRenderingTypePredictor({
+ detectionProbabilityRecommendation: 0,
+ renderingType: 'static',
+ });
+ const url = new URL(`http://${HOSTNAME}:${port}/static`);
+
+ // Only the HTTP attempt fails, so the browser run succeeds - an error says nothing about rendering.
+ const requestHandler: AdaptivePlaywrightCrawlerOptions['requestHandler'] = vi
+ .fn(async () => {})
+ .mockRejectedValueOnce(new Error('transient failure'));
+
+ const crawler = await makeOneshotCrawler({ requestHandler, renderingTypePredictor }, [url.toString()]);
+
+ await crawler.run();
+
+ expect(requestHandler).toHaveBeenCalledTimes(2);
+ expect(crawler.statistics.state.browserRequestHandlerRuns).toEqual(1);
+ expect(renderingTypePredictor.storeResult).not.toHaveBeenCalled();
});
test('should track its own fields alongside those of an injected statistics instance', async () => {