Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

8.2 KB
Raw Blame
import type { Dictionary, ProxyInfo } from '@crawlee/types';
import { z } from 'zod';

import { parseArgument, schemas } from './validators.js';

const proxyConfigurationOptionsSchema = z.strictObject({
    proxyUrls: z
        .array(z.union([z.url(), z.null()]))
        .nonempty()
        .optional(),
    newUrlFunction: schemas.anyFunction.optional(),
});

export interface ProxyConfigurationFunction {
    (): string | null | Promise<string | null>;
}

export interface ProxyConfigurationOptions {
    /**
     * An array of custom proxy URLs to be rotated.
     * Custom proxies are not compatible with Apify Proxy and an attempt to use both
     * configuration options will cause an error to be thrown on initialize.
     */
    proxyUrls?: (string | null)[];

    /**
     * Custom function that allows you to generate the new proxy URL dynamically.
     * Can return either stringified proxy URL or `null` if the proxy should not be used. Can be asynchronous.
     *
     * This function is used to generate the URL when {@apilink ProxyConfiguration.newUrl} or {@apilink ProxyConfiguration.newProxyInfo} is called.
     */
    newUrlFunction?: ProxyConfigurationFunction;

    /**
     * When truthy, the constructor throws unless one of `proxyUrls` / `newUrlFunction` was given. Falsy by
     * default, so a bare `ProxyConfiguration` can be constructed. Set by the Apify SDK, which builds the options
     * object itself; declared here only so that it stays type-checkable.
     * @internal
     */
    validateRequired?: boolean;
}

/**
 * Minimal contract that any object passed to a crawler as its `proxyConfiguration`
 * option must satisfy.
 *
 * Implement this interface to plug a custom proxy-provisioning strategy into any Crawlee
 * crawler — for example a remote proxy service or a thin wrapper around the built-in
 * `ProxyConfiguration` with different rotation rules. *
 *
 * @category Scaling
 */
export interface IProxyConfiguration {
    /**
     * Creates a new {@apilink ProxyInfo} object describing the proxy to use.
     * Returns `undefined` when no proxy should be used.
     *
     * @param proxyInfo A previously created `ProxyInfo`, e.g. one restored with a persisted session. Implementations
     *  should return an equivalent `ProxyInfo` that is usable in the current environment, or the argument itself when
     *  there is nothing to refresh.
     */
    newProxyInfo(proxyInfo?: ProxyInfo): Promise<ProxyInfo | undefined>;
}

/**
 * Configures connection to a proxy server with the provided options. Proxy servers are used to prevent target websites from blocking
 * your crawlers based on IP address rate limits or blacklists. Setting proxy configuration in your crawlers automatically configures
 * them to use the selected proxies for all connections. You can get information about the currently used proxy by inspecting
 * the {@apilink ProxyInfo} property in your crawler's page function. There, you can inspect the proxy's URL and other attributes.
 *
 * If you want to use your own proxies, use the {@apilink ProxyConfigurationOptions.proxyUrls} option. Your list of proxy URLs will
 * be rotated by the configuration if this option is provided.
 *
 * **Example usage:**
 *
 * ```javascript
 *
 * const proxyConfiguration = new ProxyConfiguration({
 *   proxyUrls: ['...', '...'],
 * });
 *
 * const crawler = new CheerioCrawler({
 *   // ...
 *   proxyConfiguration,
 *   requestHandler({ proxyInfo }) {
 *      const usedProxyUrl = proxyInfo.url; // Getting the proxy URL
 *   }
 * })
 *
 * ```
 * @category Scaling
 */
export class ProxyConfiguration implements IProxyConfiguration {
    readonly isManInTheMiddle = false;
    #nextCustomUrlIndex = 0;
    #proxyUrls?: (string | null)[];
    #newUrlFunction?: ProxyConfigurationFunction;

    /**
     * Creates a {@apilink ProxyConfiguration} instance based on the provided options. Proxy servers are used to prevent target websites from
     * blocking your crawlers based on IP address rate limits or blacklists. Setting proxy configuration in your crawlers automatically configures
     * them to use the selected proxies for all connections.
     *
     * ```javascript
     * const proxyConfiguration = new ProxyConfiguration({
     *     proxyUrls: ['http://user:[email protected]', 'http://user:[email protected]'],
     * });
     *
     * const crawler = new CheerioCrawler({
     *   // ...
     *   proxyConfiguration,
     *   requestHandler({ proxyInfo }) {
     *       const usedProxyUrl = proxyInfo.url; // Getting the proxy URL
     *   }
     * })
     *
     * ```
     */
    constructor(options: ProxyConfigurationOptions = {}) {
        // `validateRequired` is destructured off before the strict-object parse on purpose: the Apify SDK passes it
        // through a computed key (`['validateRequired' as string]: false`), and leaving it in `rest` would make
        // `Actor.createProxyConfiguration()` fail the `z.strictObject` check with a `ZodError`.
        const { validateRequired, ...rest } = options as Dictionary;

        if ('tieredProxyUrls' in rest) {
            throw new Error(
                'The `tieredProxyUrls` option has been removed in Crawlee v4. ' +
                    'See the v4 upgrading guide for the recommended migration to named sessions.',
            );
        }

        const { proxyUrls, newUrlFunction } = parseArgument(
            rest as ProxyConfigurationOptions,
            proxyConfigurationOptionsSchema,
        );

        if (proxyUrls && newUrlFunction) this.#throwCannotCombineCustomMethods();
        if (!proxyUrls && !newUrlFunction && validateRequired) this.#throwNoOptionsProvided();

        this.#proxyUrls = proxyUrls;
        this.#newUrlFunction = newUrlFunction;
    }

    /**
     * This function creates a new {@apilink ProxyInfo} info object.
     * It is used by CheerioCrawler and PuppeteerCrawler to generate proxy URLs and also to allow the user to inspect
     * the currently used proxy via the requestHandler parameter `proxyInfo`.
     * Use it if you want to work with a rich representation of a proxy URL.
     * If you need the URL string only, use {@apilink ProxyConfiguration.newUrl}.
     *
     * @param proxyInfo A previously created `ProxyInfo`, returned unchanged.
     * @return Represents information about used proxy and its configuration.
     */
    async newProxyInfo(proxyInfo?: ProxyInfo): Promise<ProxyInfo | undefined> {
        if (proxyInfo) return proxyInfo;

        const url = await this.newUrl();
        if (!url) return undefined;

        const { username, password, port, hostname } = new URL(url);

        return {
            url,
            username: decodeURIComponent(username),
            password: decodeURIComponent(password),
            hostname,
            port: port!,
        };
    }

    /**
     * Returns a new proxy URL based on provided configuration options.
     *
     * @return A string with a proxy URL, including authentication credentials and port number.
     *  For example, `http://bob:[email protected]:8000`
     */
    async newUrl(): Promise<string | undefined> {
        if (this.#newUrlFunction) {
            return (await this.#callNewUrlFunction()) ?? undefined;
        }

        return this.#handleProxyUrlsList() ?? undefined;
    }

    #handleProxyUrlsList(): string | null {
        return this.#proxyUrls![this.#nextCustomUrlIndex++ % this.#proxyUrls!.length];
    }

    /**
     * Calls the custom newUrlFunction and checks format of its return value
     */
    async #callNewUrlFunction() {
        const proxyUrl = await this.#newUrlFunction!();
        try {
            if (proxyUrl) {
                new URL(proxyUrl); // eslint-disable-line no-new
            }
            return proxyUrl;
        } catch (err) {
            throw new Error(
                `The provided newUrlFunction did not return a valid URL.\nCause: ${(err as Error).message}`,
            );
        }
    }

    #throwCannotCombineCustomMethods(): never {
        throw new Error(
            'Cannot combine custom proxies "options.proxyUrls" with custom generating function "options.newUrlFunction".',
        );
    }

    #throwNoOptionsProvided(): never {
        throw new Error('One of "options.proxyUrls" or "options.newUrlFunction" needs to be provided.');
    }
}