Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

feat: `tieredProxyUrls` for ProxyConfiguration (#2348)

Jindřich Bär5408c7f60a5b
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
index cccf38095..2a00b012a 100644
--- a/packages/basic-crawler/src/internals/basic-crawler.ts
+++ b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -1148,7 +1148,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
this.log.debug(`Adding request ${request.url} (${request.id}) back to the queue`);
// eslint-disable-next-line dot-notation
source['inProgress'].add(request.id!);
- await source.reclaimRequest(request);
+ await source.reclaimRequest(request, { forefront: request.userData?.__crawlee?.forefront });
}, delay);
return true;
@@ -1384,7 +1384,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
{ id, url, retryCount },
);
- await source.reclaimRequest(request);
+ await source.reclaimRequest(request, { forefront: request.userData?.__crawlee?.forefront });
return;
}
}
diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.ts
index 3d07c0ac6..ee1e28e00 100644
--- a/packages/browser-crawler/src/internals/browser-crawler.ts
+++ b/packages/browser-crawler/src/internals/browser-crawler.ts
@@ -493,22 +493,26 @@ export abstract class BrowserCrawler<
const useIncognitoPages = this.launchContext?.useIncognitoPages;
const experimentalContainers = this.launchContext?.experimentalContainers;
- if (this.proxyConfiguration && (useIncognitoPages || experimentalContainers)) {
- const { session } = crawlingContext;
-
- const proxyInfo = await this.proxyConfiguration.newProxyInfo(session?.id);
- crawlingContext.proxyInfo = proxyInfo;
-
- newPageOptions.proxyUrl = proxyInfo.url;
-
- if (this.proxyConfiguration.isManInTheMiddle) {
- /**
- * @see https://playwright.dev/docs/api/class-browser/#browser-new-context
- * @see https://github.com/puppeteer/puppeteer/blob/main/docs/api.md
- */
- newPageOptions.pageOptions = {
- ignoreHTTPSErrors: true,
- };
+ if (this.proxyConfiguration) {
+ if (useIncognitoPages || experimentalContainers) {
+ const { session } = crawlingContext;
+
+ const proxyInfo = await this.proxyConfiguration.newProxyInfo(session?.id, { request: crawlingContext.request });
+ crawlingContext.proxyInfo = proxyInfo;
+
+ newPageOptions.proxyUrl = proxyInfo.url;
+
+ if (this.proxyConfiguration.isManInTheMiddle) {
+ /**
+ * @see https://playwright.dev/docs/api/class-browser/#browser-new-context
+ * @see https://github.com/puppeteer/puppeteer/blob/main/docs/api.md
+ */
+ newPageOptions.pageOptions = {
+ ignoreHTTPSErrors: true,
+ };
+ }
+ } else {
+ newPageOptions.proxyTier = this.proxyConfiguration.getProxyTier(crawlingContext.request);
}
}
@@ -703,7 +707,10 @@ export abstract class BrowserCrawler<
}
if (this.proxyConfiguration) {
- const proxyInfo = await this.proxyConfiguration.newProxyInfo(launchContextExtends.session?.id);
+ const proxyInfo = await this.proxyConfiguration.newProxyInfo(
+ launchContextExtends.session?.id,
+ { proxyTier: (launchContext.proxyTier as number) ?? undefined },
+ );
launchContext.proxyUrl = proxyInfo.url;
launchContextExtends.proxyInfo = proxyInfo;
diff --git a/packages/browser-pool/src/abstract-classes/browser-controller.ts b/packages/browser-pool/src/abstract-classes/browser-controller.ts
index 2bd3b72af..91acb1681 100644
--- a/packages/browser-pool/src/abstract-classes/browser-controller.ts
+++ b/packages/browser-pool/src/abstract-classes/browser-controller.ts
@@ -55,6 +55,12 @@ export abstract class BrowserController<
*/
launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult> = undefined!;
+ /**
+ * The proxy tier tied to this browser controller.
+ * `undefined` if no tiered proxy is used.
+ */
+ proxyTier?: number;
+
isActive = false;
activePages = 0;
diff --git a/packages/browser-pool/src/abstract-classes/browser-plugin.ts b/packages/browser-pool/src/abstract-classes/browser-plugin.ts
index 1f70d7b56..25a040410 100644
--- a/packages/browser-pool/src/abstract-classes/browser-plugin.ts
+++ b/packages/browser-pool/src/abstract-classes/browser-plugin.ts
@@ -145,6 +145,7 @@ export abstract class BrowserPlugin<
useIncognitoPages = this.useIncognitoPages,
userDataDir = this.userDataDir,
experimentalContainers = this.experimentalContainers,
+ proxyTier,
} = options;
return new LaunchContext({
@@ -155,6 +156,7 @@ export abstract class BrowserPlugin<
useIncognitoPages,
experimentalContainers,
userDataDir,
+ proxyTier,
});
}
diff --git a/packages/browser-pool/src/browser-pool.ts b/packages/browser-pool/src/browser-pool.ts
index 1d2ed50a7..fd59d1ab0 100644
--- a/packages/browser-pool/src/browser-pool.ts
+++ b/packages/browser-pool/src/browser-pool.ts
@@ -391,6 +391,7 @@ export class BrowserPool<
pageOptions,
browserPlugin = this._pickBrowserPlugin(),
proxyUrl,
+ proxyTier,
} = options;
if (this.pages.has(id)) {
@@ -403,8 +404,8 @@ export class BrowserPool<
// Limiter is necessary - https://github.com/apify/crawlee/issues/1126
return this.limiter(async () => {
- let browserController = this._pickBrowserWithFreeCapacity(browserPlugin);
- if (!browserController) browserController = await this._launchBrowser(id, { browserPlugin });
+ let browserController = this._pickBrowserWithFreeCapacity(browserPlugin, { proxyTier });
+ if (!browserController) browserController = await this._launchBrowser(id, { browserPlugin, proxyTier });
tryCancel();
return this._createPageForBrowser(id, browserController, pageOptions, proxyUrl);
@@ -632,6 +633,7 @@ export class BrowserPool<
const {
browserPlugin,
launchOptions,
+ proxyTier,
} = options;
const browserController = browserPlugin.createController() as BrowserControllerReturn;
@@ -640,6 +642,7 @@ export class BrowserPool<
const launchContext = browserPlugin.createLaunchContext({
id: pageId,
launchOptions,
+ proxyTier,
});
try {
@@ -656,6 +659,7 @@ export class BrowserPool<
}
log.debug('Launched new browser.', { id: browserController.id });
+ browserController.proxyTier = proxyTier;
try {
// If the launch fails on the post-launch hooks, we need to clean up
@@ -690,15 +694,13 @@ export class BrowserPool<
return this.browserPlugins[pluginIndex];
}
- private _pickBrowserWithFreeCapacity(browserPlugin: BrowserPlugin) {
- for (const controller of this.activeBrowserControllers) {
+ private _pickBrowserWithFreeCapacity(browserPlugin: BrowserPlugin, options?: { proxyTier?: number }) {
+ return [...this.activeBrowserControllers].find((controller) => {
const hasCapacity = controller.activePages < this.maxOpenPagesPerBrowser;
const isCorrectPlugin = controller.browserPlugin === browserPlugin;
- if (hasCapacity && isCorrectPlugin) {
- return controller;
- }
- }
- return undefined;
+
+ return hasCapacity && isCorrectPlugin && (typeof options?.proxyTier !== 'number' || controller.proxyTier === options.proxyTier);
+ });
}
private async _closeInactiveRetiredBrowsers() {
@@ -821,6 +823,10 @@ export interface BrowserPoolNewPageOptions<PageOptions, BP extends BrowserPlugin
* Proxy URL.
*/
proxyUrl?: string;
+ /**
+ * Proxy tier.
+ */
+ proxyTier?: number;
}
export interface BrowserPoolNewPageInNewBrowserOptions<PageOptions, BP extends BrowserPlugin> {
@@ -856,4 +862,5 @@ export interface BrowserPoolNewPageInNewBrowserOptions<PageOptions, BP extends B
interface InternalLaunchBrowserOptions<BP extends BrowserPlugin> {
browserPlugin: BP;
launchOptions?: BP['launchOptions'];
+ proxyTier?: number;
}
diff --git a/packages/browser-pool/src/launch-context.ts b/packages/browser-pool/src/launch-context.ts
index e4c61ec5a..a89d31edf 100644
--- a/packages/browser-pool/src/launch-context.ts
+++ b/packages/browser-pool/src/launch-context.ts
@@ -50,6 +50,7 @@ export interface LaunchContextOptions<
*/
userDataDir?: string;
proxyUrl?: string;
+ proxyTier?: number;
}
export class LaunchContext<
@@ -65,6 +66,7 @@ export class LaunchContext<
useIncognitoPages: boolean;
experimentalContainers: boolean;
userDataDir: string;
+ proxyTier?: number;
private _proxyUrl?: string;
private readonly _reservedFieldNames = [...Reflect.ownKeys(this), 'extend'];
@@ -81,6 +83,7 @@ export class LaunchContext<
useIncognitoPages,
experimentalContainers,
userDataDir = '',
+ proxyTier,
} = options;
this.id = id;
@@ -89,6 +92,7 @@ export class LaunchContext<
this.useIncognitoPages = useIncognitoPages ?? false;
this.experimentalContainers = experimentalContainers ?? false;
this.userDataDir = userDataDir;
+ this.proxyTier = proxyTier;
this._proxyUrl = proxyUrl;
}
diff --git a/packages/core/src/proxy_configuration.ts b/packages/core/src/proxy_configuration.ts
index 8bc03d6e5..104269f6e 100644
--- a/packages/core/src/proxy_configuration.ts
+++ b/packages/core/src/proxy_configuration.ts
@@ -2,6 +2,8 @@ import log from '@apify/log';
import type { Dictionary } from '@crawlee/types';
import ow from 'ow';
+import type { Request } from './request';
+
export interface ProxyConfigurationFunction {
(sessionId: string | number): string | Promise<string>;
}
@@ -20,6 +22,17 @@ export interface ProxyConfigurationOptions {
* This function is used to generate the URL when {@apilink ProxyConfiguration.newUrl} or {@apilink ProxyConfiguration.newProxyInfo} is called.
*/
newUrlFunction?: ProxyConfigurationFunction;
+
+ /**
+ * An array of custom proxy URLs to be rotated stratified in tiers.
+ * This is a more advanced version of `proxyUrls` that allows you to define a hierarchy of proxy URLs
+ * If everything goes well, all the requests will be sent through the first proxy URL in the list.
+ * Whenever the crawler encounters a problem with the current proxy on the given domain, it will switch to the higher tier for this domain.
+ * The crawler probes lower-level proxies at intervals to check if it can make the tier downshift.
+ *
+ * This feature is useful when you have a set of proxies with different performance characteristics (speed, price, antibot performance etc.) and you want to use the best one for each domain.
+ */
+ tieredProxyUrls?: string[][];
}
/**
@@ -84,6 +97,66 @@ export interface ProxyInfo {
port: number | string;
}
+interface TieredProxyOptions {
+ request?: Request;
+ proxyTier?: number;
+}
+
+/**
+ * Internal class for tracking the proxy tier history for a specific domain.
+ *
+ * Predicts the best proxy tier for the next request based on the error history for different proxy tiers.
+ */
+class ProxyTierTracker {
+ private histogram: number[];
+ private currentTier: number;
+
+ constructor(tieredProxyUrls: string[][]) {
+ this.histogram = tieredProxyUrls.map(() => 0);
+ this.currentTier = 0;
+ }
+
+ /**
+ * Processes a single step of the algorithm and updates the current tier prediction based on the error history.
+ */
+ private processStep(): void {
+ this.histogram.forEach((x, i) => {
+ if (this.currentTier === i) return;
+ if (x > 0) this.histogram[i]--;
+ });
+
+ const left = this.currentTier > 0 ? this.histogram[this.currentTier - 1] : Infinity;
+ const right = this.currentTier < this.histogram.length - 1 ? this.histogram[this.currentTier + 1] : Infinity;
+
+ if (this.histogram[this.currentTier] > Math.min(left, right)) {
+ this.currentTier = left <= right ? this.currentTier - 1 : this.currentTier + 1;
+ }
+
+ if (this.histogram[this.currentTier] === left) {
+ this.currentTier--;
+ }
+ }
+
+ /**
+ * Increases the error score for the given proxy tier. This raises the chance of picking a different proxy tier for the subsequent requests.
+ *
+ * The error score is increased by 10 for the given tier. This means that this tier will be disadvantaged for the next 10 requests (every new request prediction decreases the error score by 1).
+ * @param tier The proxy tier to mark as problematic.
+ */
+ addError(tier: number) {
+ this.histogram[tier] += 10;
+ }
+
+ /**
+ * Returns the best proxy tier for the next request based on the error history for different proxy tiers.
+ * @returns The proxy tier prediction
+ */
+ predictTier() {
+ this.processStep();
+ return this.currentTier;
+ }
+}
+
/**
* Configures connection to a proxy server with the provided options. Proxy servers are used to prevent target websites from blocking
* your crawlers based on IP address rate limits or blacklists. Setting proxy configuration in your crawlers automatically configures
@@ -116,9 +189,11 @@ export class ProxyConfiguration {
isManInTheMiddle = false;
protected nextCustomUrlIndex = 0;
protected proxyUrls?: string[];
+ protected tieredProxyUrls?: string[][];
protected usedProxyUrls = new Map<string, string>();
protected newUrlFunction?: ProxyConfigurationFunction;
protected log = log.child({ prefix: 'ProxyConfiguration' });
+ protected domainTiers = new Map<string, ProxyTierTracker>();
/**
* Creates a {@apilink ProxyConfiguration} instance based on the provided options. Proxy servers are used to prevent target websites from
@@ -145,15 +220,17 @@ export class ProxyConfiguration {
ow(rest, ow.object.exactShape({
proxyUrls: ow.optional.array.nonEmpty.ofType(ow.string.url),
newUrlFunction: ow.optional.function,
+ tieredProxyUrls: ow.optional.array.nonEmpty.ofType(ow.array.nonEmpty.ofType(ow.string.url)),
}));
- const { proxyUrls, newUrlFunction } = options;
+ const { proxyUrls, newUrlFunction, tieredProxyUrls } = options;
- if (proxyUrls && newUrlFunction) this._throwCannotCombineCustomMethods();
+ if ([proxyUrls, newUrlFunction, tieredProxyUrls].filter((x) => x).length > 1) this._throwCannotCombineCustomMethods();
if (!proxyUrls && !newUrlFunction && validateRequired) this._throwNoOptionsProvided();
this.proxyUrls = proxyUrls;
this.newUrlFunction = newUrlFunction;
+ this.tieredProxyUrls = tieredProxyUrls;
}
/**
@@ -173,9 +250,10 @@ export class ProxyConfiguration {
* The identifier must not be longer than 50 characters and include only the following: `0-9`, `a-z`, `A-Z`, `"."`, `"_"` and `"~"`.
* @return Represents information about used proxy and its configuration.
*/
- async newProxyInfo(sessionId?: string | number): Promise<ProxyInfo> {
+ async newProxyInfo(sessionId?: string | number, options?: TieredProxyOptions): Promise<ProxyInfo> {
if (typeof sessionId === 'number') sessionId = `${sessionId}`;
- const url = await this.newUrl(sessionId);
+
+ const url = await this.newUrl(sessionId, options);
const { username, password, port, hostname } = new URL(url);
@@ -189,6 +267,60 @@ export class ProxyConfiguration {
};
}
+ /**
+ * Given a session identifier and a request / proxy tier, this function returns a new proxy URL based on the provided configuration options.
+ * @param _sessionId Session identifier
+ * @param options Options for the tiered proxy rotation
+ * @returns A string with a proxy URL.
+ */
+ protected _handleTieredUrl(_sessionId: string, options?: TieredProxyOptions): string {
+ if (!this.tieredProxyUrls) throw new Error('Tiered proxy URLs are not set');
+
+ if (!options || (!options?.request && options?.proxyTier === undefined)) {
+ const allProxyUrls = this.tieredProxyUrls.flat();
+ return allProxyUrls[this.nextCustomUrlIndex++ % allProxyUrls.length];
+ }
+
+ let tierPrediction = options.proxyTier!;
+
+ if (typeof tierPrediction !== 'number') {
+ tierPrediction = this.getProxyTier(options.request!)!;
+ }
+
+ const proxyTier = this.tieredProxyUrls![tierPrediction];
+
+ return proxyTier[this.nextCustomUrlIndex++ % proxyTier.length];
+ }
+
+ /**
+ * Given a `Request` object, this function returns the tier of the proxy that should be used for the request.
+ *
+ * This returns `null` if `tieredProxyUrls` option is not set.
+ */
+ getProxyTier(request: Request): number | null {
+ if (!this.tieredProxyUrls) return null;
+
+ const domain = new URL(request.url).hostname;
+ if (!this.domainTiers.has(domain)) {
+ this.domainTiers.set(domain, new ProxyTierTracker(this.tieredProxyUrls));
+ }
+
+ request.userData.__crawlee ??= {};
+
+ const tracker = this.domainTiers.get(domain)!;
+
+ if (typeof request.userData.__crawlee.lastProxyTier === 'number') {
+ tracker.addError(request.userData.__crawlee.lastProxyTier);
+ }
+
+ const tierPrediction = tracker.predictTier();
+
+ request.userData.__crawlee.lastProxyTier = tierPrediction;
+ request.userData.__crawlee.forefront = true;
+
+ return tierPrediction;
+ }
+
/**
* Returns a new proxy URL based on provided configuration options and the `sessionId` parameter.
* @param [sessionId]
@@ -202,13 +334,17 @@ export class ProxyConfiguration {
* @return A string with a proxy URL, including authentication credentials and port number.
* For example, `http://bob:[email protected]:8000`
*/
- async newUrl(sessionId?: string | number): Promise<string> {
+ async newUrl(sessionId?: string | number, options?: TieredProxyOptions): Promise<string> {
if (typeof sessionId === 'number') sessionId = `${sessionId}`;
if (this.newUrlFunction) {
return this._callNewUrlFunction(sessionId)!;
}
+ if (this.tieredProxyUrls) {
+ return this._handleTieredUrl(sessionId ?? Math.random().toString().slice(2, 6), options);
+ }
+
return this._handleCustomUrl(sessionId);
}
diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.ts
index 5549d0925..a5788957e 100644
--- a/packages/http-crawler/src/internals/http-crawler.ts
+++ b/packages/http-crawler/src/internals/http-crawler.ts
@@ -455,7 +455,7 @@ export class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any, H
if (this.proxyConfiguration) {
const sessionId = session ? session.id : undefined;
- crawlingContext.proxyInfo = await this.proxyConfiguration.newProxyInfo(sessionId);
+ crawlingContext.proxyInfo = await this.proxyConfiguration.newProxyInfo(sessionId, { request });
}
if (!request.skipNavigation) {
diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.ts
index 055b05b92..0e5832caa 100644
--- a/test/core/crawlers/basic_crawler.test.ts
+++ b/test/core/crawlers/basic_crawler.test.ts
@@ -695,7 +695,7 @@ describe('BasicCrawler', () => {
// 1st try
- expect(reclaimReq).toBeCalledWith(request1);
+ expect(reclaimReq).toBeCalledWith(request1, expect.objectContaining({}));
expect(reclaimReq).toBeCalledTimes(3);
expect(processed['http://example.com/0'].userData.foo).toBe('bar');
diff --git a/test/core/crawlers/cheerio_crawler.test.ts b/test/core/crawlers/cheerio_crawler.test.ts
index aa0dc169e..0b9a7c29b 100644
--- a/test/core/crawlers/cheerio_crawler.test.ts
+++ b/test/core/crawlers/cheerio_crawler.test.ts
@@ -1174,7 +1174,7 @@ describe('CheerioCrawler', () => {
// localhost proxy causes proxy errors, session rotations and finally throws, but we don't care
}
- expect(newUrlSpy).toBeCalledWith(usedSession.id);
+ expect(newUrlSpy).toBeCalledWith(usedSession.id, expect.objectContaining({ request: expect.any(Request) }));
});
});
diff --git a/test/core/proxy_configuration.test.ts b/test/core/proxy_configuration.test.ts
index 4cdea896a..c6e98cc80 100644
--- a/test/core/proxy_configuration.test.ts
+++ b/test/core/proxy_configuration.test.ts
@@ -1,4 +1,4 @@
-import { ProxyConfiguration } from '@crawlee/core';
+import { ProxyConfiguration, Request } from '@crawlee/core';
const sessionId = 538909250932;
@@ -176,4 +176,92 @@ describe('ProxyConfiguration', () => {
}
});
});
+
+ describe('with tieredProxyUrls', () => {
+ test('without Request rotates the urls uniformly', async () => {
+ const proxyConfiguration = new ProxyConfiguration({
+ tieredProxyUrls: [
+ ['http://proxy.com:1111', 'http://proxy.com:2222'],
+ ['http://proxy.com:3333', 'http://proxy.com:4444'],
+ ],
+ });
+
+ // @ts-expect-error protected property
+ const { tieredProxyUrls } = proxyConfiguration;
+ expect(await proxyConfiguration.newUrl()).toEqual(tieredProxyUrls[0][0]);
+ expect(await proxyConfiguration.newUrl()).toEqual(tieredProxyUrls[0][1]);
+ expect(await proxyConfiguration.newUrl()).toEqual(tieredProxyUrls[1][0]);
+ expect(await proxyConfiguration.newUrl()).toEqual(tieredProxyUrls[1][1]);
+ expect(await proxyConfiguration.newUrl()).toEqual(tieredProxyUrls[0][0]);
+ });
+
+ test('rotating a request results in higher-level proxies', async () => {
+ const proxyConfiguration = new ProxyConfiguration({
+ tieredProxyUrls: [
+ ['http://proxy.com:1111'],
+ ['http://proxy.com:2222'],
+ ['http://proxy.com:3333'],
+ ],
+ });
+
+ const request = new Request({
+ url: 'http://example.com',
+ });
+
+ // @ts-expect-error protected property
+ const { tieredProxyUrls } = proxyConfiguration;
+ expect(await proxyConfiguration.newUrl('session-id', { request })).toEqual(tieredProxyUrls[0][0]);
+ expect(await proxyConfiguration.newUrl('session-id', { request })).toEqual(tieredProxyUrls[1][0]);
+ expect(await proxyConfiguration.newUrl('session-id', { request })).toEqual(tieredProxyUrls[2][0]);
+
+ // we still get the same (higher) proxy tier even with a new request
+ const request2 = new Request({
+ url: 'http://example.com/another-resource',
+ });
+
+ expect(await proxyConfiguration.newUrl('session-id', { request: request2 })).toEqual(tieredProxyUrls[2][0]);
+ });
+
+ test('upshifts and downshifts properly', async () => {
+ const tieredProxyUrls = [
+ ['http://proxy.com:1111'],
+ ['http://proxy.com:2222'],
+ ['http://proxy.com:3333'],
+ ];
+
+ const proxyConfiguration = new ProxyConfiguration({
+ tieredProxyUrls,
+ });
+
+ const request = new Request({
+ url: 'http://example.com',
+ });
+
+ let gotToTheHighestProxy = false;
+ for (let i = 0; i < 10; i++) {
+ const lastProxyUrl = await proxyConfiguration.newUrl('session-id', { request });
+ if (lastProxyUrl === tieredProxyUrls[2][0]) {
+ gotToTheHighestProxy = true;
+ break;
+ }
+ request.sessionRotationCount++;
+ }
+
+ expect(gotToTheHighestProxy).toBe(true);
+
+ // now let's go back down
+ let gotToTheLowestProxy = false;
+
+ for (let i = 0; i < 20; i++) {
+ const lastProxyUrl = await proxyConfiguration.newUrl('session-id', { request });
+ if (lastProxyUrl === tieredProxyUrls[0][0]) {
+ gotToTheLowestProxy = true;
+ break;
+ }
+ // We don't increment the sessionRotationCount here - this causes the proxy tier to go down (current proxy is ok, so it tries to downshift in some time)
+ }
+
+ expect(gotToTheLowestProxy).toBe(true);
+ });
+ });
});