Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: pass restored `ProxyInfo` to `newProxyInfo` (#4198)
Jindřich Bärd401355e3810
diff --git a/docs/public-api/crawlee-core.api.md b/docs/public-api/crawlee-core.api.mdindex e09ce53fe..7d55bfd05 100644--- a/docs/public-api/crawlee-core.api.md+++ b/docs/public-api/crawlee-core.api.md@@ -370,7 +370,7 @@ interface Intervals {// @publicexport interface IProxyConfiguration {- newProxyInfo(): Promise<ProxyInfo | undefined>;+ newProxyInfo(proxyInfo?: ProxyInfo): Promise<ProxyInfo | undefined>;}// @public@@ -562,7 +562,7 @@ export class ProxyConfiguration implements IProxyConfiguration {constructor(options?: ProxyConfigurationOptions);// (undocumented)readonly isManInTheMiddle = false;- newProxyInfo(): Promise<ProxyInfo | undefined>;+ newProxyInfo(proxyInfo?: ProxyInfo): Promise<ProxyInfo | undefined>;newUrl(): Promise<string | undefined>;}diff --git a/docs/upgrading/upgrading_v4.md b/docs/upgrading/upgrading_v4.mdindex 95fc60d19..816212e5e 100644--- a/docs/upgrading/upgrading_v4.md+++ b/docs/upgrading/upgrading_v4.md@@ -1208,7 +1208,7 @@ More complex routing (more tiers, weighted draws, sticky assignment, cooldowns)#### `ProxyConfiguration.newUrl` / `newProxyInfo` signatures changed-Because proxy tiers are gone, the leading `sessionId` positional argument was dropped from `ProxyConfiguration.newUrl()` and `ProxyConfiguration.newProxyInfo()`. Both now take no arguments instead of `(sessionId?, options?)`. The `ProxyConfigurationFunction` callback (the `newUrlFunction` option) was likewise simplified — it no longer receives any arguments, neither a `sessionId` nor a `{ request }` object. The proxy is resolved once per session, so to route specific requests through specific proxies, pin them to named sessions (see [Pinning a request to a specific session](../guides/session-management#pinning-a-request-to-a-specific-session)). The `TieredProxy` interface and the `TieredProxyOptions` type have been removed.+Because proxy tiers are gone, the leading `sessionId` positional argument was dropped from `ProxyConfiguration.newUrl()` and `ProxyConfiguration.newProxyInfo()`. `newUrl()` now takes no arguments, and `newProxyInfo()` takes only an optional, previously created `ProxyInfo` (e.g. one restored with a persisted session) instead of `(sessionId?, options?)`. Custom `IProxyConfiguration` implementations should return that `ProxyInfo` (refreshed if it depends on the environment) rather than a new one, otherwise the crawler replaces the session's proxy. The `ProxyConfigurationFunction` callback (the `newUrlFunction` option) was likewise simplified — it no longer receives any arguments, neither a `sessionId` nor a `{ request }` object. The proxy is resolved once per session, so to route specific requests through specific proxies, pin them to named sessions (see [Pinning a request to a specific session](../guides/session-management#pinning-a-request-to-a-specific-session)). The `TieredProxy` interface and the `TieredProxyOptions` type have been removed.**Before:**```typescriptdiff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex d00cc0e10..27b02683f 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -1195,8 +1195,9 @@ export class BasicCrawler<createSessionFunction: async (opts) =>new Session({...opts?.sessionOptions,- proxyInfo:- opts?.sessionOptions?.proxyInfo ?? (await this.proxyConfiguration?.newProxyInfo()),+ proxyInfo: this.proxyConfiguration+ ? await this.proxyConfiguration.newProxyInfo(opts?.sessionOptions?.proxyInfo)+ : opts?.sessionOptions?.proxyInfo,}),}),);diff --git a/packages/core/src/proxy_configuration.ts b/packages/core/src/proxy_configuration.tsindex ae62afee9..21515e3fe 100644--- a/packages/core/src/proxy_configuration.ts+++ b/packages/core/src/proxy_configuration.ts@@ -54,8 +54,12 @@ export interface IProxyConfiguration {/*** Creates a new {@apilink ProxyInfo} object describing the proxy to use.* Returns `undefined` when no proxy should be used.+ *+ * @param proxyInfo A previously created `ProxyInfo`, e.g. one restored with a persisted session. Implementations+ * should return an equivalent `ProxyInfo` that is usable in the current environment, or the argument itself when+ * there is nothing to refresh.*/- newProxyInfo(): Promise<ProxyInfo | undefined>;+ newProxyInfo(proxyInfo?: ProxyInfo): Promise<ProxyInfo | undefined>;}/**@@ -144,9 +148,12 @@ export class ProxyConfiguration implements IProxyConfiguration {* Use it if you want to work with a rich representation of a proxy URL.* If you need the URL string only, use {@apilink ProxyConfiguration.newUrl}.*+ * @param proxyInfo A previously created `ProxyInfo`, returned unchanged.* @return Represents information about used proxy and its configuration.*/- async newProxyInfo(): Promise<ProxyInfo | undefined> {+ async newProxyInfo(proxyInfo?: ProxyInfo): Promise<ProxyInfo | undefined> {+ if (proxyInfo) return proxyInfo;+const url = await this.newUrl();if (!url) return undefined;diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.tsindex 67610e299..a744740f6 100644--- a/test/core/crawlers/basic_crawler.test.ts+++ b/test/core/crawlers/basic_crawler.test.ts@@ -2754,6 +2754,45 @@ describe('BasicCrawler', () => {}expect(sessions[0].usageCount).toBe(3);});++ describe('restored sessions', () => {+ const persistedProxyInfo: ProxyInfo = {+ url: 'http://user:[email protected]:8011',+ username: 'user',+ password: 'pass',+ hostname: '10.0.0.1',+ port: '8011',+ };++ async function restoreSession(proxyConfiguration: BasicCrawlerOptions['proxyConfiguration']) {+ const crawler = new BasicCrawler({ proxyConfiguration, requestHandler: async () => {} });+ const sessionPool = crawler.sessionPool as SessionPool;++ const previousRun = new SessionPool({+ persistStateKey: `CRAWLEE_SESSION_POOL_STATE_${sessionPool.id}`,+ });+ await previousRun.addSession({ id: 'restored', proxyInfo: persistedProxyInfo });+ await previousRun.teardown();++ return sessionPool.getSession('restored');+ }++ it('passes the persisted proxyInfo to newProxyInfo and uses the result', async () => {+ const refreshedProxyInfo = { ...persistedProxyInfo, url: 'http://user:[email protected]:8011' };+ const newProxyInfo = vitest.fn(async () => refreshedProxyInfo);++ const session = await restoreSession({ newProxyInfo });++ expect(newProxyInfo).toHaveBeenCalledWith(persistedProxyInfo);+ expect(session?.proxyInfo).toEqual(refreshedProxyInfo);+ });++ it('drops the persisted proxyInfo when newProxyInfo returns undefined', async () => {+ const session = await restoreSession({ newProxyInfo: async () => undefined });++ expect(session?.proxyInfo).toBeUndefined();+ });+ });});test('extendContext', async () => {diff --git a/test/core/proxy_configuration.test.ts b/test/core/proxy_configuration.test.tsindex 0fc4c270d..b05ffa520 100644--- a/test/core/proxy_configuration.test.ts+++ b/test/core/proxy_configuration.test.ts@@ -130,6 +130,15 @@ describe('ProxyConfiguration', () => {expect((await proxyConfiguration.newProxyInfo())!.url).toEqual(proxyUrls[2]);});+ test('newProxyInfo() should return a passed ProxyInfo without advancing the rotation', async () => {+ const proxyUrls = ['http://proxy.com:1111', 'http://proxy.com:2222'];+ const proxyConfiguration = new ProxyConfiguration({ proxyUrls });++ const proxyInfo = await proxyConfiguration.newProxyInfo();+ expect(await proxyConfiguration.newProxyInfo(proxyInfo)).toBe(proxyInfo);+ expect((await proxyConfiguration.newProxyInfo())!.url).toEqual(proxyUrls[1]);+ });+test('should throw cannot combine custom methods', async () => {const proxyUrls = ['http://proxy.com:1111', 'http://proxy.com:2222', 'http://proxy.com:3333'];const newUrlFunction = () => {