Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

feat: better `newUrlFunction` for ProxyConfiguration (#2392)

Jindřich Bär330598b348ad
diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.ts
index ee1e28e00..ecf34b8ac 100644
--- a/packages/browser-crawler/src/internals/browser-crawler.ts
+++ b/packages/browser-crawler/src/internals/browser-crawler.ts
@@ -494,25 +494,22 @@ export abstract class BrowserCrawler<
const experimentalContainers = this.launchContext?.experimentalContainers;
if (this.proxyConfiguration) {
- if (useIncognitoPages || experimentalContainers) {
- const { session } = crawlingContext;
+ const { session } = crawlingContext;
- const proxyInfo = await this.proxyConfiguration.newProxyInfo(session?.id, { request: crawlingContext.request });
- crawlingContext.proxyInfo = proxyInfo;
+ const proxyInfo = await this.proxyConfiguration.newProxyInfo(session?.id, { request: crawlingContext.request });
+ crawlingContext.proxyInfo = proxyInfo;
- newPageOptions.proxyUrl = proxyInfo.url;
+ newPageOptions.proxyUrl = proxyInfo?.url;
+ newPageOptions.proxyTier = proxyInfo?.proxyTier;
- if (this.proxyConfiguration.isManInTheMiddle) {
- /**
+ if (this.proxyConfiguration.isManInTheMiddle) {
+ /**
* @see https://playwright.dev/docs/api/class-browser/#browser-new-context
* @see https://github.com/puppeteer/puppeteer/blob/main/docs/api.md
*/
- newPageOptions.pageOptions = {
- ignoreHTTPSErrors: true,
- };
- }
- } else {
- newPageOptions.proxyTier = this.proxyConfiguration.getProxyTier(crawlingContext.request);
+ newPageOptions.pageOptions = {
+ ignoreHTTPSErrors: true,
+ };
}
}
@@ -706,12 +703,12 @@ export abstract class BrowserCrawler<
launchContextExtends.session = await this.sessionPool.getSession();
}
- if (this.proxyConfiguration) {
+ if (this.proxyConfiguration && !launchContext.proxyUrl) {
const proxyInfo = await this.proxyConfiguration.newProxyInfo(
launchContextExtends.session?.id,
{ proxyTier: (launchContext.proxyTier as number) ?? undefined },
);
- launchContext.proxyUrl = proxyInfo.url;
+ launchContext.proxyUrl = proxyInfo?.url;
launchContextExtends.proxyInfo = proxyInfo;
// Disable SSL verification for MITM proxies
diff --git a/packages/browser-pool/src/abstract-classes/browser-controller.ts b/packages/browser-pool/src/abstract-classes/browser-controller.ts
index 91acb1681..7cc86f390 100644
--- a/packages/browser-pool/src/abstract-classes/browser-controller.ts
+++ b/packages/browser-pool/src/abstract-classes/browser-controller.ts
@@ -61,6 +61,12 @@ export abstract class BrowserController<
*/
proxyTier?: number;
+ /**
+ * The proxy URL used by the browser controller. This is set every time the browser controller uses proxy (even the tiered one).
+ * `undefined` if no proxy is used
+ */
+ proxyUrl?: string;
+
isActive = false;
activePages = 0;
diff --git a/packages/browser-pool/src/browser-pool.ts b/packages/browser-pool/src/browser-pool.ts
index fd59d1ab0..9171fb022 100644
--- a/packages/browser-pool/src/browser-pool.ts
+++ b/packages/browser-pool/src/browser-pool.ts
@@ -1,4 +1,5 @@
import { addTimeoutToPromise, tryCancel } from '@apify/timeout';
+import type { TieredProxy } from '@crawlee/core';
import type { BrowserFingerprintWithHeaders } from 'fingerprint-generator';
import { FingerprintGenerator } from 'fingerprint-generator';
import { FingerprintInjector } from 'fingerprint-injector';
@@ -404,8 +405,9 @@ export class BrowserPool<
// Limiter is necessary - https://github.com/apify/crawlee/issues/1126
return this.limiter(async () => {
- let browserController = this._pickBrowserWithFreeCapacity(browserPlugin, { proxyTier });
- if (!browserController) browserController = await this._launchBrowser(id, { browserPlugin, proxyTier });
+ let browserController = this._pickBrowserWithFreeCapacity(browserPlugin, { proxyTier, proxyUrl });
+
+ if (!browserController) browserController = await this._launchBrowser(id, { browserPlugin, proxyTier, proxyUrl });
tryCancel();
return this._createPageForBrowser(id, browserController, pageOptions, proxyUrl);
@@ -634,6 +636,7 @@ export class BrowserPool<
browserPlugin,
launchOptions,
proxyTier,
+ proxyUrl,
} = options;
const browserController = browserPlugin.createController() as BrowserControllerReturn;
@@ -643,6 +646,7 @@ export class BrowserPool<
id: pageId,
launchOptions,
proxyTier,
+ proxyUrl,
});
try {
@@ -660,6 +664,7 @@ export class BrowserPool<
log.debug('Launched new browser.', { id: browserController.id });
browserController.proxyTier = proxyTier;
+ browserController.proxyUrl = proxyUrl;
try {
// If the launch fails on the post-launch hooks, we need to clean up
@@ -694,12 +699,23 @@ export class BrowserPool<
return this.browserPlugins[pluginIndex];
}
- private _pickBrowserWithFreeCapacity(browserPlugin: BrowserPlugin, options?: { proxyTier?: number }) {
+ private _pickBrowserWithFreeCapacity(
+ browserPlugin: BrowserPlugin,
+ options?: Partial<TieredProxy>,
+ ) {
return [...this.activeBrowserControllers].find((controller) => {
const hasCapacity = controller.activePages < this.maxOpenPagesPerBrowser;
const isCorrectPlugin = controller.browserPlugin === browserPlugin;
-
- return hasCapacity && isCorrectPlugin && (typeof options?.proxyTier !== 'number' || controller.proxyTier === options.proxyTier);
+ const isSameProxyUrl = (controller.proxyUrl === options?.proxyUrl);
+ const isCorrectProxyTier = controller.proxyTier === options?.proxyTier;
+
+ return isCorrectPlugin
+ && hasCapacity
+ && (
+ (options?.proxyTier && isCorrectProxyTier)
+ || (options?.proxyUrl && isSameProxyUrl)
+ || (!options?.proxyUrl && !options?.proxyTier && !controller.proxyUrl && !controller.proxyTier)
+ );
});
}
@@ -863,4 +879,5 @@ interface InternalLaunchBrowserOptions<BP extends BrowserPlugin> {
browserPlugin: BP;
launchOptions?: BP['launchOptions'];
proxyTier?: number;
+ proxyUrl?: string;
}
diff --git a/packages/core/src/proxy_configuration.ts b/packages/core/src/proxy_configuration.ts
index 104269f6e..a619afbd6 100644
--- a/packages/core/src/proxy_configuration.ts
+++ b/packages/core/src/proxy_configuration.ts
@@ -1,11 +1,12 @@
import log from '@apify/log';
+import { cryptoRandomObjectId } from '@apify/utilities';
import type { Dictionary } from '@crawlee/types';
import ow from 'ow';
import type { Request } from './request';
export interface ProxyConfigurationFunction {
- (sessionId: string | number): string | Promise<string>;
+ (sessionId: string | number, options?: { request?: Request }): string | null | Promise<string | null>;
}
export interface ProxyConfigurationOptions {
@@ -17,8 +18,9 @@ export interface ProxyConfigurationOptions {
proxyUrls?: string[];
/**
- * Custom function that allows you to generate the new proxy URL dynamically. It gets the `sessionId` as a parameter
- * and should always return stringified proxy URL. Can be asynchronous.
+ * Custom function that allows you to generate the new proxy URL dynamically. It gets the `sessionId` as a parameter and an optional parameter with the `Request` object when applicable.
+ * Can return either stringified proxy URL or `null` if the proxy should not be used. Can be asynchronous.
+ *
* This function is used to generate the URL when {@apilink ProxyConfiguration.newUrl} or {@apilink ProxyConfiguration.newProxyInfo} is called.
*/
newUrlFunction?: ProxyConfigurationFunction;
@@ -35,6 +37,11 @@ export interface ProxyConfigurationOptions {
tieredProxyUrls?: string[][];
}
+export interface TieredProxy {
+ proxyUrl: string;
+ proxyTier?: number;
+}
+
/**
* The main purpose of the ProxyInfo object is to provide information
* about the current proxy connection used by the crawler for the request.
@@ -95,6 +102,11 @@ export interface ProxyInfo {
* Proxy port.
*/
port: number | string;
+
+ /**
+ * Proxy tier for the current proxy, if applicable (only for `tieredProxyUrls`).
+ */
+ proxyTier?: number;
}
interface TieredProxyOptions {
@@ -250,10 +262,20 @@ export class ProxyConfiguration {
* The identifier must not be longer than 50 characters and include only the following: `0-9`, `a-z`, `A-Z`, `"."`, `"_"` and `"~"`.
* @return Represents information about used proxy and its configuration.
*/
- async newProxyInfo(sessionId?: string | number, options?: TieredProxyOptions): Promise<ProxyInfo> {
+ async newProxyInfo(sessionId?: string | number, options?: TieredProxyOptions): Promise<ProxyInfo | undefined> {
if (typeof sessionId === 'number') sessionId = `${sessionId}`;
- const url = await this.newUrl(sessionId, options);
+ let url: string | undefined;
+ let tier: number | undefined;
+ if (this.tieredProxyUrls) {
+ const { proxyUrl, proxyTier } = this._handleTieredUrl(sessionId ?? cryptoRandomObjectId(6), options);
+ url = proxyUrl;
+ tier = proxyTier;
+ } else {
+ url = await this.newUrl(sessionId, options);
+ }
+
+ if (!url) return undefined;
const { username, password, port, hostname } = new URL(url);
@@ -264,6 +286,7 @@ export class ProxyConfiguration {
password,
hostname,
port: port!,
+ proxyTier: tier,
};
}
@@ -271,25 +294,30 @@ export class ProxyConfiguration {
* Given a session identifier and a request / proxy tier, this function returns a new proxy URL based on the provided configuration options.
* @param _sessionId Session identifier
* @param options Options for the tiered proxy rotation
- * @returns A string with a proxy URL.
+ * @returns An object with the proxy URL and the proxy tier used.
*/
- protected _handleTieredUrl(_sessionId: string, options?: TieredProxyOptions): string {
+ protected _handleTieredUrl(_sessionId: string, options?: TieredProxyOptions): TieredProxy {
if (!this.tieredProxyUrls) throw new Error('Tiered proxy URLs are not set');
if (!options || (!options?.request && options?.proxyTier === undefined)) {
const allProxyUrls = this.tieredProxyUrls.flat();
- return allProxyUrls[this.nextCustomUrlIndex++ % allProxyUrls.length];
+ return {
+ proxyUrl: allProxyUrls[this.nextCustomUrlIndex++ % allProxyUrls.length],
+ };
}
let tierPrediction = options.proxyTier!;
if (typeof tierPrediction !== 'number') {
- tierPrediction = this.getProxyTier(options.request!)!;
+ tierPrediction = this.predictProxyTier(options.request!)!;
}
const proxyTier = this.tieredProxyUrls![tierPrediction];
- return proxyTier[this.nextCustomUrlIndex++ % proxyTier.length];
+ return {
+ proxyUrl: proxyTier[this.nextCustomUrlIndex++ % proxyTier.length],
+ proxyTier: tierPrediction,
+ };
}
/**
@@ -297,7 +325,7 @@ export class ProxyConfiguration {
*
* This returns `null` if `tieredProxyUrls` option is not set.
*/
- getProxyTier(request: Request): number | null {
+ protected predictProxyTier(request: Request): number | null {
if (!this.tieredProxyUrls) return null;
const domain = new URL(request.url).hostname;
@@ -334,15 +362,18 @@ export class ProxyConfiguration {
* @return A string with a proxy URL, including authentication credentials and port number.
* For example, `http://bob:[email protected]:8000`
*/
- async newUrl(sessionId?: string | number, options?: TieredProxyOptions): Promise<string> {
+ async newUrl(sessionId?: string | number, options?: TieredProxyOptions): Promise<string | undefined> {
if (typeof sessionId === 'number') sessionId = `${sessionId}`;
if (this.newUrlFunction) {
- return this._callNewUrlFunction(sessionId)!;
+ return (await this._callNewUrlFunction(sessionId, { request: options?.request }) ?? undefined);
}
if (this.tieredProxyUrls) {
- return this._handleTieredUrl(sessionId ?? Math.random().toString().slice(2, 6), options);
+ return this._handleTieredUrl(
+ sessionId ?? cryptoRandomObjectId(6),
+ options,
+ ).proxyUrl;
}
return this._handleCustomUrl(sessionId);
@@ -371,12 +402,12 @@ export class ProxyConfiguration {
/**
* Calls the custom newUrlFunction and checks format of its return value
*/
- protected async _callNewUrlFunction(sessionId?: string) {
- let proxyUrl: string;
-
+ protected async _callNewUrlFunction(sessionId?: string, options?: { request?: Request }) {
+ const proxyUrl = await this.newUrlFunction!(sessionId!, options);
try {
- proxyUrl = await this.newUrlFunction!(sessionId!);
- new URL(proxyUrl); // eslint-disable-line no-new
+ if (proxyUrl) {
+ new URL(proxyUrl); // eslint-disable-line no-new
+ }
return proxyUrl;
} catch (err) {
this._throwNewUrlFunctionInvalid(err as Error);
diff --git a/test/core/proxy_configuration.test.ts b/test/core/proxy_configuration.test.ts
index c6e98cc80..e4ead04d3 100644
--- a/test/core/proxy_configuration.test.ts
+++ b/test/core/proxy_configuration.test.ts
@@ -21,7 +21,7 @@ describe('ProxyConfiguration', () => {
password: '',
port: '1111',
};
- expect(await proxyConfiguration.newProxyInfo(sessionId)).toStrictEqual(proxyInfo);
+ expect(await proxyConfiguration.newProxyInfo(sessionId)).toEqual(proxyInfo);
});
test('should throw on invalid newUrlFunction', async () => {