Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
refactor: Introduce IProxyConfiguration (#3892)
Jan Buchar219be89479df
diff --git a/docs/public-api/crawlee-basic.api.md b/docs/public-api/crawlee-basic.api.mdindex f8040f324..82ac79d97 100644--- a/docs/public-api/crawlee-basic.api.md+++ b/docs/public-api/crawlee-basic.api.md@@ -30,6 +30,7 @@ import type { EnqueueLinksOptions } from '@crawlee/core';import type { EventManager } from '@crawlee/core';import type { FinalStatistics } from '@crawlee/core';import type { GetUserDataFromRequest } from '@crawlee/core';+import { IProxyConfiguration } from '@crawlee/core';import { IRequestLoader } from '@crawlee/core';import { IRequestManager } from '@crawlee/core';import type { ISession } from '@crawlee/types';@@ -37,7 +38,6 @@ import type { ISessionPool } from '@crawlee/types';import { NumberPredicate } from 'ow';import { ObjectPredicate } from 'ow';import { Predicate } from 'ow';-import { ProxyConfiguration } from '@crawlee/core';import type { ProxyInfo } from '@crawlee/types';import type { ReadonlyDeep } from 'type-fest';import { Request as Request_2 } from '@crawlee/core';@@ -177,7 +177,7 @@ export class BasicCrawler<Context extends CrawlingContext = CrawlingContext, Con};// (undocumented)protected _pauseOnMigration(): Promise<void>;- proxyConfiguration?: ProxyConfiguration;+ proxyConfiguration?: IProxyConfiguration;pushData(data: Parameters<Dataset['pushData']>[0], datasetIdentifier?: string | StorageIdentifier): Promise<void>;protected _requestFunctionErrorHandler(error: Error, crawlingContext: CrawlingContext, request: Request_2, source: IRequestManager): Promise<void>;// (undocumented)@@ -242,7 +242,7 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingCmaxRequestsPerMinute?: number;minConcurrency?: number;onSkippedRequest?: SkippedRequestCallback;- proxyConfiguration?: ProxyConfiguration;+ proxyConfiguration?: IProxyConfiguration;requestHandler?: RequestHandler<ExtendedContext>;requestHandlerTimeoutSecs?: number;// @deprecateddiff --git a/docs/public-api/crawlee-core.api.md b/docs/public-api/crawlee-core.api.mdindex 852662871..c42eb4e02 100644--- a/docs/public-api/crawlee-core.api.md+++ b/docs/public-api/crawlee-core.api.md@@ -746,6 +746,11 @@ export type GlobObject = {glob: string;} & Pick<RequestOptions, 'method' | 'payload' | 'label' | 'userData' | 'headers'>;+// @public+export interface IProxyConfiguration {+ newProxyInfo(options?: NewUrlOptions): Promise<ProxyInfo | undefined>;+}+// @publicexport interface IRequestLoader {[Symbol.asyncIterator](): AsyncGenerator<Request_2>;@@ -994,7 +999,7 @@ export interface PersistenceOptions {}// @public-export class ProxyConfiguration {+export class ProxyConfiguration implements IProxyConfiguration {constructor(options?: ProxyConfigurationOptions);protected _callNewUrlFunction(options?: {request?: Request_2;@@ -1215,7 +1220,7 @@ export interface RequestListOptions {keepDuplicateUrls?: boolean;persistRequestsKey?: string;persistStateKey?: string;- proxyConfiguration?: ProxyConfiguration;+ proxyConfiguration?: IProxyConfiguration;sources?: RequestListSource[];sourcesFunction?: RequestListSourcesFunction;state?: RequestListState;@@ -1318,7 +1323,7 @@ export class RequestQueue implements IStorage, IRequestManager {name?: string;static open(identifier?: string | StorageIdentifier | null, options?: StorageOpenOptions): Promise<RequestQueue>;// (undocumented)- protected proxyConfiguration?: ProxyConfiguration;+ protected proxyConfiguration?: IProxyConfiguration;purge(): Promise<void>;// (undocumented)protected queuePausedForMigration: boolean;@@ -1340,7 +1345,7 @@ export interface RequestQueueOptions {// (undocumented)backend: RequestQueueBackend;metadata: RequestQueueInfo;- proxyConfiguration?: ProxyConfiguration;+ proxyConfiguration?: IProxyConfiguration;}// @public@@ -1929,7 +1934,7 @@ export class StorageInstanceManager {export interface StorageOpenOptions {config?: Configuration;httpClient?: BaseHttpClient;- proxyConfiguration?: ProxyConfiguration;+ proxyConfiguration?: IProxyConfiguration;storageBackend?: StorageBackend;}diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex d7fa481b0..1ce8f4453 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -14,9 +14,9 @@ import type {EventStatusMessageData,FinalStatistics,GetUserDataFromRequest,+ IProxyConfiguration,IRequestLoader,IRequestManager,- ProxyConfiguration,Request,RequestsLike,RequestTransform,@@ -419,7 +419,7 @@ export interface BasicCrawlerOptions<* If set, the crawler will be configured for all connections to use* the Proxy URLs provided and rotated according to the configuration.*/- proxyConfiguration?: ProxyConfiguration;+ proxyConfiguration?: IProxyConfiguration;/*** Custom configuration to use for this crawler.@@ -628,10 +628,10 @@ export class BasicCrawler<autoscaledPool?: AutoscaledPool;/**- * A reference to the underlying {@apilink ProxyConfiguration} class that manages the crawler's proxies.+ * A reference to the underlying {@apilink IProxyConfiguration} instance that manages the crawler's proxies.* Only available if used by the crawler.*/- proxyConfiguration?: ProxyConfiguration;+ proxyConfiguration?: IProxyConfiguration;/*** Default {@apilink Router} instance that will be used if we don't specify any {@apilink BasicCrawlerOptions.requestHandler|`requestHandler`}.diff --git a/packages/core/src/proxy_configuration.ts b/packages/core/src/proxy_configuration.tsindex 2c6f65078..83b959388 100644--- a/packages/core/src/proxy_configuration.ts+++ b/packages/core/src/proxy_configuration.ts@@ -31,6 +31,24 @@ interface NewUrlOptions {request?: Request;}+/**+ * Minimal contract that any object passed to a crawler as its `proxyConfiguration`+ * option must satisfy.+ *+ * Implement this interface to plug a custom proxy-provisioning strategy into any Crawlee+ * crawler — for example a remote proxy service or a thin wrapper around the built-in+ * `ProxyConfiguration` with different rotation rules. *+ *+ * @category Scaling+ */+export interface IProxyConfiguration {+ /**+ * Creates a new {@apilink ProxyInfo} object describing the proxy to use for the given+ * request. Returns `undefined` when no proxy should be used.+ */+ newProxyInfo(options?: NewUrlOptions): Promise<ProxyInfo | undefined>;+}+/*** Configures connection to a proxy server with the provided options. Proxy servers are used to prevent target websites from blocking* your crawlers based on IP address rate limits or blacklists. Setting proxy configuration in your crawlers automatically configures@@ -59,7 +77,7 @@ interface NewUrlOptions {* ```* @category Scaling*/-export class ProxyConfiguration {+export class ProxyConfiguration implements IProxyConfiguration {isManInTheMiddle = false;protected nextCustomUrlIndex = 0;protected proxyUrls?: UrlList;diff --git a/packages/core/src/storages/request_list.ts b/packages/core/src/storages/request_list.tsindex 5f0facc28..68fe272fc 100644--- a/packages/core/src/storages/request_list.ts+++ b/packages/core/src/storages/request_list.ts@@ -5,7 +5,7 @@ import ow, { ArgumentError } from 'ow';import type { Configuration } from '../configuration.js';import { EventType } from '../events/event_manager.js';import type { CrawleeLogger } from '../log.js';-import type { ProxyConfiguration } from '../proxy_configuration.js';+import type { IProxyConfiguration } from '../proxy_configuration.js';import { type InternalSource, Request, type RequestOptions, type Source } from '../request.js';import { createDeserialize, serializeArray } from '../serialization.js';import { serviceLocator } from '../service_locator.js';@@ -103,7 +103,7 @@ export interface RequestListOptions {* Takes advantage of the internal address rotation and authentication process.* If undefined, the `requestsFromUrl` requests will be made without proxy.*/- proxyConfiguration?: ProxyConfiguration;+ proxyConfiguration?: IProxyConfiguration;/*** Identifies the key in the default key-value store under which `RequestList` periodically stores its@@ -288,7 +288,7 @@ export class RequestList implements IRequestLoader {private keepDuplicateUrls: boolean;private sources: RequestListSource[];private sourcesFunction?: RequestListSourcesFunction;- private proxyConfiguration?: ProxyConfiguration;+ private proxyConfiguration?: IProxyConfiguration;private httpClient?: BaseHttpClient;/**@@ -716,7 +716,7 @@ export class RequestList implements IRequestLoader {urlsArr = await this._downloadListOfUrls({url: requestsFromUrl,urlRegExp: regex,- proxyUrl: await this.proxyConfiguration?.newUrl(),+ proxyUrl: (await this.proxyConfiguration?.newProxyInfo())?.url,});} catch (err) {throw new Error(`Cannot fetch a request list from ${requestsFromUrl}: ${err}`);diff --git a/packages/core/src/storages/request_queue.ts b/packages/core/src/storages/request_queue.tsindex e097f561c..b929f643a 100644--- a/packages/core/src/storages/request_queue.ts+++ b/packages/core/src/storages/request_queue.ts@@ -28,7 +28,7 @@ import type { Constructor } from '../typedefs.js';import type { EventManager } from '../events/event_manager.js';import { EventType } from '../events/event_manager.js';import type { CrawleeLogger } from '../log.js';-import type { ProxyConfiguration } from '../proxy_configuration.js';+import type { IProxyConfiguration } from '../proxy_configuration.js';import type { InternalSource, RequestOptions, Source } from '../request.js';import { Request } from '../request.js';import { serviceLocator } from '../service_locator.js';@@ -92,7 +92,7 @@ export class RequestQueue implements IStorage, IRequestManager {id: string;name?: string;backend: RequestQueueBackend;- protected proxyConfiguration?: ProxyConfiguration;+ protected proxyConfiguration?: IProxyConfiguration;log: CrawleeLogger;@@ -880,7 +880,7 @@ export class RequestQueue implements IStorage, IRequestManager {urlsArr = await this._downloadListOfUrls({url: requestsFromUrl,urlRegExp: regex,- proxyUrl: await this.proxyConfiguration?.newUrl(),+ proxyUrl: (await this.proxyConfiguration?.newProxyInfo())?.url,});} catch (err) {throw new Error(`Cannot fetch a request list from ${requestsFromUrl}: ${err}`);@@ -1005,7 +1005,7 @@ export interface RequestQueueOptions {* Takes advantage of the internal address rotation and authentication process.* If undefined, the `requestsFromUrl` requests will be made without proxy.*/- proxyConfiguration?: ProxyConfiguration;+ proxyConfiguration?: IProxyConfiguration;}export interface RequestQueueOperationOptions {diff --git a/packages/core/src/storages/utils.ts b/packages/core/src/storages/utils.tsindex 83c5e7dfa..1c7e81147 100644--- a/packages/core/src/storages/utils.ts+++ b/packages/core/src/storages/utils.ts@@ -3,7 +3,7 @@ import crypto from 'node:crypto';import type { BaseHttpClient, Dictionary, StorageBackend } from '@crawlee/types';import { Configuration } from '../configuration.js';-import type { ProxyConfiguration } from '../proxy_configuration.js';+import type { IProxyConfiguration } from '../proxy_configuration.js';import { serviceLocator } from '../service_locator.js';import { KeyValueStore } from './key_value_store.js';@@ -241,7 +241,7 @@ export interface StorageOpenOptions {* Takes advantage of the internal address rotation and authentication process.* If undefined, the `requestsFromUrl` requests will be made without proxy.*/- proxyConfiguration?: ProxyConfiguration;+ proxyConfiguration?: IProxyConfiguration;/*** HTTP client to be used to download the list of URLs in `RequestQueue`.diff --git a/packages/core/src/validators.ts b/packages/core/src/validators.tsindex 95095271a..b00fe36f5 100644--- a/packages/core/src/validators.ts+++ b/packages/core/src/validators.ts@@ -9,8 +9,9 @@ export const validators = {message: (label: string) => `Expected argument '${label}' to be a Puppeteer Page, got something else.`,}),proxyConfiguration: (value: Dictionary) => ({- validator: ow.isValid(value, ow.object.hasKeys('newUrl', 'newProxyInfo')),- message: (label: string) => `Expected argument '${label}' to be a ProxyConfiguration, got something else.`,+ validator: ow.isValid(value, ow.object.hasKeys('newProxyInfo')),+ message: (label: string) =>+ `Expected argument '${label}' to implement the IProxyConfiguration interface (missing 'newProxyInfo'), got something else.`,}),requestList: (value: Dictionary) => ({validator: ow.isValid(value, ow.object.hasKeys('fetchNextRequest', 'persistState')),