Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: Prevent browser-based crawling mode in AdaptivePlaywrightCrawler from swallowing logs (#3803)
Jan Buchar50293fe57a73
diff --git a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.tsindex d31fe77db..cc8a5291b 100644--- a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts+++ b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts@@ -685,6 +685,7 @@ export class AdaptivePlaywrightCrawler<throw browserRun.error;}+ browserRun.logs?.forEach(([log, method, ...args]) => log[method](...(args as [any, any])));await this.commitResult(crawlingContext, browserRun.result);if (shouldDetectRenderingType) {diff --git a/test/core/crawlers/adaptive_playwright_crawler.test.ts b/test/core/crawlers/adaptive_playwright_crawler.test.tsindex ff2e6d95a..9905425fd 100644--- a/test/core/crawlers/adaptive_playwright_crawler.test.ts+++ b/test/core/crawlers/adaptive_playwright_crawler.test.ts@@ -1,7 +1,15 @@import type { Server } from 'node:http';import type { AddressInfo } from 'node:net';-import { type Dictionary, EventType, KeyValueStore, serviceLocator } from '@crawlee/core';+import {+ BaseCrawleeLogger,+ type CrawleeLogger,+ type CrawleeLoggerOptions,+ type Dictionary,+ EventType,+ KeyValueStore,+ serviceLocator,+} from '@crawlee/core';import type {AdaptivePlaywrightCrawlerContext,AdaptivePlaywrightCrawlerOptions,@@ -22,6 +30,22 @@ import { z } from 'zod';import { startExpressAppPromise } from 'test/shared/_helper.js';import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator.js';+// A minimal logger that records every message into a shared array. Child loggers share the same+// array, so messages emitted by the crawler's prefixed child logger are captured as well.+class RecordingLogger extends BaseCrawleeLogger {+ constructor(private readonly messages: string[]) {+ super();+ }++ logWithLevel(_level: number, message: string): void {+ this.messages.push(message);+ }++ protected createChild(_options: Partial<CrawleeLoggerOptions>): CrawleeLogger {+ return new RecordingLogger(this.messages);+ }+}+describe('AdaptivePlaywrightCrawler', () => {// Set up an express server that will serve test pagesconst HOSTNAME = '127.0.0.1';@@ -183,6 +207,36 @@ describe('AdaptivePlaywrightCrawler', () => {});});+ test.each([['static'], ['clientOnly']] as const)(+ 'should replay request handler logs (%s)',+ async (renderingType) => {+ const renderingTypePredictor = makeRiggedRenderingTypePredictor({+ detectionProbabilityRecommendation: 0,+ renderingType,+ });+ const url = new URL(`http://${HOSTNAME}:${port}/static`);++ const messages: string[] = [];+ const requestHandler: AdaptivePlaywrightCrawlerOptions['requestHandler'] = vi.fn(async ({ log }) => {+ log.info('handler log message');+ });++ const crawler = await makeOneshotCrawler(+ {+ requestHandler,+ renderingTypePredictor,+ logger: new RecordingLogger(messages),+ },+ [url.toString()],+ );++ await crawler.run();++ expect(requestHandler).toHaveBeenCalled();+ expect(messages).toContain('handler log message');+ },+ );+test('should not store detection results on non-detection runs', async () => {const renderingTypePredictor = makeRiggedRenderingTypePredictor({detectionProbabilityRecommendation: 0,