Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: `proxyUrls` list can contain `null` (#3142)

Jindřich Bärdc39cc223a90
diff --git a/packages/core/src/proxy_configuration.ts b/packages/core/src/proxy_configuration.ts
index 5bbada6f8..132d923ac 100644
--- a/packages/core/src/proxy_configuration.ts
+++ b/packages/core/src/proxy_configuration.ts
@@ -10,13 +10,15 @@ export interface ProxyConfigurationFunction {
(sessionId: string | number, options?: { request?: Request }): string | null | Promise<string | null>;
}
+type UrlList = (string | null)[];
+
export interface ProxyConfigurationOptions {
/**
* An array of custom proxy URLs to be rotated.
* Custom proxies are not compatible with Apify Proxy and an attempt to use both
* configuration options will cause an error to be thrown on initialize.
*/
- proxyUrls?: string[];
+ proxyUrls?: UrlList;
/**
* Custom function that allows you to generate the new proxy URL dynamically. It gets the `sessionId` as a parameter and an optional parameter with the `Request` object when applicable.
@@ -37,7 +39,7 @@ export interface ProxyConfigurationOptions {
*
* Use `null` as a proxy URL to disable the proxy for the given tier.
*/
- tieredProxyUrls?: (string | null)[][];
+ tieredProxyUrls?: UrlList[];
}
export interface TieredProxy {
@@ -201,9 +203,9 @@ class ProxyTierTracker {
export class ProxyConfiguration {
isManInTheMiddle = false;
protected nextCustomUrlIndex = 0;
- protected proxyUrls?: string[];
- protected tieredProxyUrls?: (string | null)[][];
- protected usedProxyUrls = new Map<string, string>();
+ protected proxyUrls?: UrlList;
+ protected tieredProxyUrls?: UrlList[];
+ protected usedProxyUrls = new Map<string, string | null>();
protected newUrlFunction?: ProxyConfigurationFunction;
protected log = log.child({ prefix: 'ProxyConfiguration' });
protected domainTiers = new Map<string, ProxyTierTracker>();
@@ -233,7 +235,7 @@ export class ProxyConfiguration {
ow(
rest,
ow.object.exactShape({
- proxyUrls: ow.optional.array.nonEmpty.ofType(ow.string.url),
+ proxyUrls: ow.optional.array.nonEmpty.ofType(ow.any(ow.string.url, ow.null)),
newUrlFunction: ow.optional.function,
tieredProxyUrls: ow.optional.array.nonEmpty.ofType(
ow.array.nonEmpty.ofType(ow.any(ow.string.url, ow.null)),
@@ -389,14 +391,14 @@ export class ProxyConfiguration {
return this._handleTieredUrl(sessionId ?? cryptoRandomObjectId(6), options).proxyUrl ?? undefined;
}
- return this._handleCustomUrl(sessionId);
+ return this._handleCustomUrl(sessionId) ?? undefined;
}
/**
* Handles custom url rotation with session
*/
- protected _handleCustomUrl(sessionId?: string): string {
- let customUrlToUse: string;
+ protected _handleCustomUrl(sessionId?: string): string | null {
+ let customUrlToUse: string | null;
if (!sessionId) {
return this.proxyUrls![this.nextCustomUrlIndex++ % this.proxyUrls!.length];
diff --git a/test/core/crawlers/browser_crawler.test.ts b/test/core/crawlers/browser_crawler.test.ts
index 738bab4a7..6075c1c2c 100644
--- a/test/core/crawlers/browser_crawler.test.ts
+++ b/test/core/crawlers/browser_crawler.test.ts
@@ -849,7 +849,7 @@ describe('BrowserCrawler', () => {
// @ts-expect-error Accessing private property
const proxiesToUse = proxyConfiguration.proxyUrls!;
for (const proxyUrl of proxiesToUse) {
- expect(browserProxies.includes(new URL(proxyUrl).href.slice(0, -1))).toBeTruthy();
+ expect(browserProxies.includes(new URL(proxyUrl!).href.slice(0, -1))).toBeTruthy();
}
delete process.env[ENV_VARS.PROXY_PASSWORD];