Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Revert "feat: newUrlFunction receives Request and can return false"
Jindřich Bär27a0fe7d018d
diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.tsindex 67cccfaa7..d88106684 100644--- a/packages/browser-crawler/src/internals/browser-crawler.ts+++ b/packages/browser-crawler/src/internals/browser-crawler.ts@@ -467,21 +467,19 @@ export abstract class BrowserCrawler<if (this.proxyConfiguration && (useIncognitoPages || experimentalContainers)) {const { session } = crawlingContext;- const proxyInfo = await this.proxyConfiguration.newProxyInfo(session?.id, crawlingContext.request);+ const proxyInfo = await this.proxyConfiguration.newProxyInfo(session?.id);crawlingContext.proxyInfo = proxyInfo;- if (proxyInfo) {- newPageOptions.proxyUrl = proxyInfo.url;-- if (this.proxyConfiguration.isManInTheMiddle) {- /**- * @see https://playwright.dev/docs/api/class-browser/#browser-new-context- * @see https://github.com/puppeteer/puppeteer/blob/main/docs/api.md- */- newPageOptions.pageOptions = {- ignoreHTTPSErrors: true,- };- }+ newPageOptions.proxyUrl = proxyInfo.url;++ if (this.proxyConfiguration.isManInTheMiddle) {+ /**+ * @see https://playwright.dev/docs/api/class-browser/#browser-new-context+ * @see https://github.com/puppeteer/puppeteer/blob/main/docs/api.md+ */+ newPageOptions.pageOptions = {+ ignoreHTTPSErrors: true,+ };}}@@ -666,19 +664,16 @@ export abstract class BrowserCrawler<if (this.proxyConfiguration) {const proxyInfo = await this.proxyConfiguration.newProxyInfo(launchContextExtends.session?.id);-- if (proxyInfo) {- launchContext.proxyUrl = proxyInfo.url;- launchContextExtends.proxyInfo = proxyInfo;-- // Disable SSL verification for MITM proxies- if (this.proxyConfiguration.isManInTheMiddle) {- /**- * @see https://playwright.dev/docs/api/class-browser/#browser-new-context- * @see https://github.com/puppeteer/puppeteer/blob/main/docs/api.md- */- (launchContext.launchOptions as Dictionary).ignoreHTTPSErrors = true;- }+ launchContext.proxyUrl = proxyInfo.url;+ launchContextExtends.proxyInfo = proxyInfo;++ // Disable SSL verification for MITM proxies+ if (this.proxyConfiguration.isManInTheMiddle) {+ /**+ * @see https://playwright.dev/docs/api/class-browser/#browser-new-context+ * @see https://github.com/puppeteer/puppeteer/blob/main/docs/api.md+ */+ (launchContext.launchOptions as Dictionary).ignoreHTTPSErrors = true;}}diff --git a/packages/core/src/proxy_configuration.ts b/packages/core/src/proxy_configuration.tsindex b644f751d..8bc03d6e5 100644--- a/packages/core/src/proxy_configuration.ts+++ b/packages/core/src/proxy_configuration.ts@@ -2,10 +2,8 @@ import log from '@apify/log';import type { Dictionary } from '@crawlee/types';import ow from 'ow';-import type { Request } from './request';-export interface ProxyConfigurationFunction {- (sessionId: string | number, request?: Request): string | Promise<string | false>;+ (sessionId: string | number): string | Promise<string>;}export interface ProxyConfigurationOptions {@@ -17,8 +15,8 @@ export interface ProxyConfigurationOptions {proxyUrls?: string[];/**- * Custom function that allows you to generate the new proxy URL dynamically. It gets the `sessionId` and the current Request object as parameters- * and should return either stringified proxy URL or `false` if proxy shouldn't be used. Can be asynchronous.+ * Custom function that allows you to generate the new proxy URL dynamically. It gets the `sessionId` as a parameter+ * and should always return stringified proxy URL. Can be asynchronous.* This function is used to generate the URL when {@apilink ProxyConfiguration.newUrl} or {@apilink ProxyConfiguration.newProxyInfo} is called.*/newUrlFunction?: ProxyConfigurationFunction;@@ -175,11 +173,9 @@ export class ProxyConfiguration {* The identifier must not be longer than 50 characters and include only the following: `0-9`, `a-z`, `A-Z`, `"."`, `"_"` and `"~"`.* @return Represents information about used proxy and its configuration.*/- async newProxyInfo(sessionId?: string | number, request?: Request): Promise<ProxyInfo | undefined> {+ async newProxyInfo(sessionId?: string | number): Promise<ProxyInfo> {if (typeof sessionId === 'number') sessionId = `${sessionId}`;- const url = await this.newUrl(sessionId, request);-- if (!url) return undefined;+ const url = await this.newUrl(sessionId);const { username, password, port, hostname } = new URL(url);@@ -206,11 +202,11 @@ export class ProxyConfiguration {* @return A string with a proxy URL, including authentication credentials and port number.* For example, `http://bob:[email protected]:8000`*/- async newUrl(sessionId?: string | number, request?: Request): Promise<string | false> {+ async newUrl(sessionId?: string | number): Promise<string> {if (typeof sessionId === 'number') sessionId = `${sessionId}`;if (this.newUrlFunction) {- return this._callNewUrlFunction(sessionId, request)!;+ return this._callNewUrlFunction(sessionId)!;}return this._handleCustomUrl(sessionId);@@ -237,15 +233,13 @@ export class ProxyConfiguration {}/**- * Calls the custom newUrlFunction and checks format of its return value.+ * Calls the custom newUrlFunction and checks format of its return value*/- protected async _callNewUrlFunction(sessionId?: string, request?: Request) {- let proxyUrl: string | false;+ protected async _callNewUrlFunction(sessionId?: string) {+ let proxyUrl: string;try {- proxyUrl = await this.newUrlFunction!(sessionId!, request);- if (!proxyUrl) return false;-+ proxyUrl = await this.newUrlFunction!(sessionId!);new URL(proxyUrl); // eslint-disable-line no-newreturn proxyUrl;} catch (err) {diff --git a/packages/core/src/storages/request_list.ts b/packages/core/src/storages/request_list.tsindex b5aee21c6..fc4cf5b08 100644--- a/packages/core/src/storages/request_list.ts+++ b/packages/core/src/storages/request_list.ts@@ -677,11 +677,7 @@ export class RequestList {// Download remote resource and parse URLs.let urlsArr;try {- urlsArr = await this._downloadListOfUrls({- url: requestsFromUrl,- urlRegExp: regex,- proxyUrl: (await this.proxyConfiguration?.newUrl() || undefined),- });+ urlsArr = await this._downloadListOfUrls({ url: requestsFromUrl, urlRegExp: regex, proxyUrl: await this.proxyConfiguration?.newUrl() });} catch (err) {throw new Error(`Cannot fetch a request list from ${requestsFromUrl}: ${err}`);}diff --git a/packages/core/src/storages/request_queue.ts b/packages/core/src/storages/request_queue.tsindex c6583d110..36a2ee4b7 100644--- a/packages/core/src/storages/request_queue.ts+++ b/packages/core/src/storages/request_queue.ts@@ -887,11 +887,7 @@ export class RequestQueue {// Download remote resource and parse URLs.let urlsArr;try {- urlsArr = await this._downloadListOfUrls({- url: requestsFromUrl,- urlRegExp: regex,- proxyUrl: await this.proxyConfiguration?.newUrl() || undefined,- });+ urlsArr = await this._downloadListOfUrls({ url: requestsFromUrl, urlRegExp: regex, proxyUrl: await this.proxyConfiguration?.newUrl() });} catch (err) {throw new Error(`Cannot fetch a request list from ${requestsFromUrl}: ${err}`);}diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.tsindex 4a96a3151..2451506f0 100644--- a/packages/http-crawler/src/internals/http-crawler.ts+++ b/packages/http-crawler/src/internals/http-crawler.ts@@ -453,7 +453,7 @@ export class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any, Hif (this.proxyConfiguration) {const sessionId = session ? session.id : undefined;- crawlingContext.proxyInfo = await this.proxyConfiguration.newProxyInfo(sessionId, crawlingContext.request);+ crawlingContext.proxyInfo = await this.proxyConfiguration.newProxyInfo(sessionId);}if (!request.skipNavigation) {await this._handleNavigation(crawlingContext);