Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

4.4 KB
Raw Blame
import { fork } from 'node:child_process';

import { FileSystemStorageBackend } from '@crawlee/fs-storage';
import { Configuration, Dataset, PlaywrightCrawler, log } from 'crawlee';

import { router } from './routes.mjs';
import { getOrInitQueue } from './shared.mjs';

// For this example, we will spawn 2 separate processes that will scrape the store in parallel.

if (!process.env.IN_WORKER_THREAD) {
    // This is the main process. We will use this to spawn the worker threads.
    log.info('Setting up worker threads.');

    const currentFile = new URL(import.meta.url).pathname;

    // Store a promise per worker, so we wait for all to finish before exiting the main process
    const promises = [];

    // You can decide how many workers you want to spawn, but keep in mind you can only spawn so many before you overload your machine
    for (let i = 0; i < 2; i++) {
        const proc = fork(currentFile, {
            env: {
                // Share the current process's env across to the newly created process
                ...process.env,
                // ...but also tell the process that it's a worker process
                IN_WORKER_THREAD: 'true',
                // ...as well as which worker it is
                WORKER_INDEX: String(i),
            },
        });

        proc.on('online', () => {
            log.info(`Process ${i} is online.`);

            // Log out what the crawlers are doing
            // Note: we want to use console.log instead of log.info because we already get formatted output from the crawlers
            proc.stdout.on('data', (data) => {
                // eslint-disable-next-line no-console
                console.log(data.toString());
            });

            proc.stderr.on('data', (data) => {
                // eslint-disable-next-line no-console
                console.error(data.toString());
            });
        });

        proc.on('message', async (data) => {
            log.debug(`Process ${i} sent data.`, data);
            await Dataset.pushData(data);
        });

        promises.push(
            new Promise((resolve) => {
                proc.once('exit', (code, signal) => {
                    log.info(`Process ${i} exited with code ${code} and signal ${signal}`);
                    resolve();
                });
            }),
        );
    }

    await Promise.all(promises);

    log.info('Crawling complete!');
} else {
    // This is the worker process. We will use this to scrape the store.

    // Let's build a logger that will prefix the log messages with the worker index
    const workerLogger = log.child({ prefix: `[Worker ${process.env.WORKER_INDEX}]` });

    // This is better set with CRAWLEE_LOG_LEVEL env var
    // or a configuration option. This is just for show 😈
    workerLogger.setLevel(log.LEVELS.DEBUG);

    // Get the request queue
    const requestQueue = await getOrInitQueue(false);

    // Disable the automatic purge on start, so we don't lose the queue we prepared
    const config = new Configuration({
        purgeOnStart: false,
    });

    // Store the worker's own internal state (its default dataset, key-value store, etc.) in a separate
    // directory so the workers don't collide with each other. This directory is private to a single
    // worker, so we set `requestQueueAccess: 'single'` — the concurrency-safe locking only matters for
    // the shared `shop-urls` queue, which gets its own storage backend in `requestQueue.mjs`.
    const storageBackend = new FileSystemStorageBackend({
        localDataDirectory: `./storage/worker-${process.env.WORKER_INDEX}`,
        requestQueueAccess: 'single',
    });

    workerLogger.debug('Setting up crawler.');
    const crawler = new PlaywrightCrawler(
        {
            log: workerLogger,
            // Instead of the long requestHandler with
            // if clauses we provide a router instance.
            requestHandler: router,
            // Provide the request queue we've pre-filled in previous steps
            // highlight-start
            requestQueue,
            // highlight-end
            // Let's also limit the crawler's concurrency, we don't want to overload a single process 🐌
            maxConcurrency: 5,
            // Use the worker-specific, concurrency-safe storage backend we created above
            // highlight-start
            storageBackend,
            // highlight-end
        },
        config,
    );

    await crawler.run();
}