Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat: newUrlFunction receives Request and can return false
Jindřich Bär7bfc15e9be3f
diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.tsindex d88106684..67cccfaa7 100644--- a/packages/browser-crawler/src/internals/browser-crawler.ts+++ b/packages/browser-crawler/src/internals/browser-crawler.ts@@ -467,19 +467,21 @@ export abstract class BrowserCrawler<if (this.proxyConfiguration && (useIncognitoPages || experimentalContainers)) {const { session } = crawlingContext;- const proxyInfo = await this.proxyConfiguration.newProxyInfo(session?.id);+ const proxyInfo = await this.proxyConfiguration.newProxyInfo(session?.id, crawlingContext.request);crawlingContext.proxyInfo = proxyInfo;- newPageOptions.proxyUrl = proxyInfo.url;-- if (this.proxyConfiguration.isManInTheMiddle) {- /**- * @see https://playwright.dev/docs/api/class-browser/#browser-new-context- * @see https://github.com/puppeteer/puppeteer/blob/main/docs/api.md- */- newPageOptions.pageOptions = {- ignoreHTTPSErrors: true,- };+ if (proxyInfo) {+ newPageOptions.proxyUrl = proxyInfo.url;++ if (this.proxyConfiguration.isManInTheMiddle) {+ /**+ * @see https://playwright.dev/docs/api/class-browser/#browser-new-context+ * @see https://github.com/puppeteer/puppeteer/blob/main/docs/api.md+ */+ newPageOptions.pageOptions = {+ ignoreHTTPSErrors: true,+ };+ }}}@@ -664,16 +666,19 @@ export abstract class BrowserCrawler<if (this.proxyConfiguration) {const proxyInfo = await this.proxyConfiguration.newProxyInfo(launchContextExtends.session?.id);- launchContext.proxyUrl = proxyInfo.url;- launchContextExtends.proxyInfo = proxyInfo;-- // Disable SSL verification for MITM proxies- if (this.proxyConfiguration.isManInTheMiddle) {- /**- * @see https://playwright.dev/docs/api/class-browser/#browser-new-context- * @see https://github.com/puppeteer/puppeteer/blob/main/docs/api.md- */- (launchContext.launchOptions as Dictionary).ignoreHTTPSErrors = true;++ if (proxyInfo) {+ launchContext.proxyUrl = proxyInfo.url;+ launchContextExtends.proxyInfo = proxyInfo;++ // Disable SSL verification for MITM proxies+ if (this.proxyConfiguration.isManInTheMiddle) {+ /**+ * @see https://playwright.dev/docs/api/class-browser/#browser-new-context+ * @see https://github.com/puppeteer/puppeteer/blob/main/docs/api.md+ */+ (launchContext.launchOptions as Dictionary).ignoreHTTPSErrors = true;+ }}}diff --git a/packages/core/src/proxy_configuration.ts b/packages/core/src/proxy_configuration.tsindex 8bc03d6e5..b644f751d 100644--- a/packages/core/src/proxy_configuration.ts+++ b/packages/core/src/proxy_configuration.ts@@ -2,8 +2,10 @@ import log from '@apify/log';import type { Dictionary } from '@crawlee/types';import ow from 'ow';+import type { Request } from './request';+export interface ProxyConfigurationFunction {- (sessionId: string | number): string | Promise<string>;+ (sessionId: string | number, request?: Request): string | Promise<string | false>;}export interface ProxyConfigurationOptions {@@ -15,8 +17,8 @@ export interface ProxyConfigurationOptions {proxyUrls?: string[];/**- * Custom function that allows you to generate the new proxy URL dynamically. It gets the `sessionId` as a parameter- * and should always return stringified proxy URL. Can be asynchronous.+ * Custom function that allows you to generate the new proxy URL dynamically. It gets the `sessionId` and the current Request object as parameters+ * and should return either stringified proxy URL or `false` if proxy shouldn't be used. Can be asynchronous.* This function is used to generate the URL when {@apilink ProxyConfiguration.newUrl} or {@apilink ProxyConfiguration.newProxyInfo} is called.*/newUrlFunction?: ProxyConfigurationFunction;@@ -173,9 +175,11 @@ export class ProxyConfiguration {* The identifier must not be longer than 50 characters and include only the following: `0-9`, `a-z`, `A-Z`, `"."`, `"_"` and `"~"`.* @return Represents information about used proxy and its configuration.*/- async newProxyInfo(sessionId?: string | number): Promise<ProxyInfo> {+ async newProxyInfo(sessionId?: string | number, request?: Request): Promise<ProxyInfo | undefined> {if (typeof sessionId === 'number') sessionId = `${sessionId}`;- const url = await this.newUrl(sessionId);+ const url = await this.newUrl(sessionId, request);++ if (!url) return undefined;const { username, password, port, hostname } = new URL(url);@@ -202,11 +206,11 @@ export class ProxyConfiguration {* @return A string with a proxy URL, including authentication credentials and port number.* For example, `http://bob:[email protected]:8000`*/- async newUrl(sessionId?: string | number): Promise<string> {+ async newUrl(sessionId?: string | number, request?: Request): Promise<string | false> {if (typeof sessionId === 'number') sessionId = `${sessionId}`;if (this.newUrlFunction) {- return this._callNewUrlFunction(sessionId)!;+ return this._callNewUrlFunction(sessionId, request)!;}return this._handleCustomUrl(sessionId);@@ -233,13 +237,15 @@ export class ProxyConfiguration {}/**- * Calls the custom newUrlFunction and checks format of its return value+ * Calls the custom newUrlFunction and checks format of its return value.*/- protected async _callNewUrlFunction(sessionId?: string) {- let proxyUrl: string;+ protected async _callNewUrlFunction(sessionId?: string, request?: Request) {+ let proxyUrl: string | false;try {- proxyUrl = await this.newUrlFunction!(sessionId!);+ proxyUrl = await this.newUrlFunction!(sessionId!, request);+ if (!proxyUrl) return false;+new URL(proxyUrl); // eslint-disable-line no-newreturn proxyUrl;} catch (err) {diff --git a/packages/core/src/storages/request_list.ts b/packages/core/src/storages/request_list.tsindex fc4cf5b08..b5aee21c6 100644--- a/packages/core/src/storages/request_list.ts+++ b/packages/core/src/storages/request_list.ts@@ -677,7 +677,11 @@ export class RequestList {// Download remote resource and parse URLs.let urlsArr;try {- urlsArr = await this._downloadListOfUrls({ url: requestsFromUrl, urlRegExp: regex, proxyUrl: await this.proxyConfiguration?.newUrl() });+ urlsArr = await this._downloadListOfUrls({+ url: requestsFromUrl,+ urlRegExp: regex,+ proxyUrl: (await this.proxyConfiguration?.newUrl() || undefined),+ });} catch (err) {throw new Error(`Cannot fetch a request list from ${requestsFromUrl}: ${err}`);}diff --git a/packages/core/src/storages/request_queue.ts b/packages/core/src/storages/request_queue.tsindex 36a2ee4b7..c6583d110 100644--- a/packages/core/src/storages/request_queue.ts+++ b/packages/core/src/storages/request_queue.ts@@ -887,7 +887,11 @@ export class RequestQueue {// Download remote resource and parse URLs.let urlsArr;try {- urlsArr = await this._downloadListOfUrls({ url: requestsFromUrl, urlRegExp: regex, proxyUrl: await this.proxyConfiguration?.newUrl() });+ urlsArr = await this._downloadListOfUrls({+ url: requestsFromUrl,+ urlRegExp: regex,+ proxyUrl: await this.proxyConfiguration?.newUrl() || undefined,+ });} catch (err) {throw new Error(`Cannot fetch a request list from ${requestsFromUrl}: ${err}`);}diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.tsindex 2451506f0..4a96a3151 100644--- a/packages/http-crawler/src/internals/http-crawler.ts+++ b/packages/http-crawler/src/internals/http-crawler.ts@@ -453,7 +453,7 @@ export class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any, Hif (this.proxyConfiguration) {const sessionId = session ? session.id : undefined;- crawlingContext.proxyInfo = await this.proxyConfiguration.newProxyInfo(sessionId);+ crawlingContext.proxyInfo = await this.proxyConfiguration.newProxyInfo(sessionId, crawlingContext.request);}if (!request.skipNavigation) {await this._handleNavigation(crawlingContext);