Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

2.0 KB
Raw Blame
import { CheerioCrawler, BaseCrawleeLogger, LogLevel } from 'crawlee';
import type { CrawleeLogger, CrawleeLoggerOptions } from 'crawlee';
import winston from 'winston';

// Map Crawlee log levels to Winston levels
const CRAWLEE_TO_WINSTON: Record<number, string> = {
    [LogLevel.ERROR]: 'error',
    [LogLevel.SOFT_FAIL]: 'warn',
    [LogLevel.WARNING]: 'warn',
    [LogLevel.INFO]: 'info',
    [LogLevel.DEBUG]: 'debug',
    [LogLevel.PERF]: 'debug',
};

class WinstonAdapter extends BaseCrawleeLogger {
    #logger: winston.Logger;

    constructor(logger: winston.Logger, options?: Partial<CrawleeLoggerOptions>) {
        super(options);
        this.#logger = logger;
    }

    logWithLevel(level: number, message: string, data?: Record<string, unknown>): void {
        const winstonLevel = CRAWLEE_TO_WINSTON[level] ?? 'info';
        this.#logger.log(winstonLevel, message, data);
    }

    protected createChild(options: Partial<CrawleeLoggerOptions>): CrawleeLogger {
        return new WinstonAdapter(this.#logger.child({ prefix: options.prefix }), { ...this.getOptions(), ...options });
    }
}

// Create a Winston logger with your preferred configuration
const winstonLogger = winston.createLogger({
    level: 'debug',
    format: winston.format.combine(
        winston.format.colorize(),
        winston.format.timestamp(),
        winston.format.printf(({ level, message, timestamp, prefix }) => {
            const tag = prefix ? `[${prefix}] ` : '';
            return `${timestamp} ${level}: ${tag}${message}`;
        }),
    ),
    transports: [new winston.transports.Console()],
});

// Pass the adapter to the crawler via the `logger` option
const crawler = new CheerioCrawler({
    logger: new WinstonAdapter(winstonLogger),
    async requestHandler({ request, $, log }) {
        log.info(`Processing ${request.url}`);
        const title = $('title').text();
        log.debug('Page title extracted', { title });
    },
});

await crawler.run(['https://crawlee.dev']);