Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat: `tieredProxyUrls` accept `null` for switching the proxy off (#2743)
Jindřich Bär82f4ea99f632
diff --git a/packages/core/src/proxy_configuration.ts b/packages/core/src/proxy_configuration.tsindex 6e0b8a8b3..3444e0ed3 100644--- a/packages/core/src/proxy_configuration.ts+++ b/packages/core/src/proxy_configuration.ts@@ -33,12 +33,14 @@ export interface ProxyConfigurationOptions {* The crawler probes lower-level proxies at intervals to check if it can make the tier downshift.** This feature is useful when you have a set of proxies with different performance characteristics (speed, price, antibot performance etc.) and you want to use the best one for each domain.+ *+ * Use `null` as a proxy URL to disable the proxy for the given tier.*/- tieredProxyUrls?: string[][];+ tieredProxyUrls?: (string | null)[][];}export interface TieredProxy {- proxyUrl: string;+ proxyUrl: string | null;proxyTier?: number;}@@ -123,7 +125,7 @@ class ProxyTierTracker {private histogram: number[];private currentTier: number;- constructor(tieredProxyUrls: string[][]) {+ constructor(tieredProxyUrls: (string | null)[][]) {this.histogram = tieredProxyUrls.map(() => 0);this.currentTier = 0;}@@ -199,7 +201,7 @@ export class ProxyConfiguration {isManInTheMiddle = false;protected nextCustomUrlIndex = 0;protected proxyUrls?: string[];- protected tieredProxyUrls?: string[][];+ protected tieredProxyUrls?: (string | null)[][];protected usedProxyUrls = new Map<string, string>();protected newUrlFunction?: ProxyConfigurationFunction;protected log = log.child({ prefix: 'ProxyConfiguration' });@@ -232,7 +234,9 @@ export class ProxyConfiguration {ow.object.exactShape({proxyUrls: ow.optional.array.nonEmpty.ofType(ow.string.url),newUrlFunction: ow.optional.function,- tieredProxyUrls: ow.optional.array.nonEmpty.ofType(ow.array.nonEmpty.ofType(ow.string.url)),+ tieredProxyUrls: ow.optional.array.nonEmpty.ofType(+ ow.array.nonEmpty.ofType(ow.any(ow.string.url, ow.null)),+ ),}),);@@ -271,7 +275,7 @@ export class ProxyConfiguration {let tier: number | undefined;if (this.tieredProxyUrls) {const { proxyUrl, proxyTier } = this._handleTieredUrl(sessionId ?? cryptoRandomObjectId(6), options);- url = proxyUrl;+ url = proxyUrl ?? undefined;tier = proxyTier;} else {url = await this.newUrl(sessionId, options);@@ -381,7 +385,7 @@ export class ProxyConfiguration {}if (this.tieredProxyUrls) {- return this._handleTieredUrl(sessionId ?? cryptoRandomObjectId(6), options).proxyUrl;+ return this._handleTieredUrl(sessionId ?? cryptoRandomObjectId(6), options).proxyUrl ?? undefined;}return this._handleCustomUrl(sessionId);