Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Fix proxy configuration to support dynamic proxy status url (#940)
Andrey Bykov7eeefede9ca3
diff --git a/src/proxy_configuration.js b/src/proxy_configuration.jsindex 6932e7f07..c46bd0bcb 100644--- a/src/proxy_configuration.js+++ b/src/proxy_configuration.js@@ -8,7 +8,6 @@ import defaultLog from './utils_log';// CONSTANTSconst PROTOCOL = 'http';-const APIFY_PROXY_STATUS_URL = 'http://proxy.apify.com/?format=json';// https://docs.apify.com/proxy/datacenter-proxy#username-parametersconst MAX_SESSION_ID_LENGTH = 50;const CHECK_ACCESS_REQUEST_TIMEOUT_SECS = 4;@@ -301,6 +300,15 @@ export class ProxyConfiguration {return username;}+ /**+ * Returns Apify proxy status url.+ * @return {string} the Apify proxy status url+ * @ignore+ */+ _getApifyProxyStatusUrl() {+ return process.env[ENV_VARS.PROXY_STATUS_URL] || `${PROTOCOL}://${LOCAL_ENV_VARS[ENV_VARS.PROXY_HOSTNAME]}`;+ }+/*** Checks if Apify Token is provided in env* and gets the password via API and sets it to env@@ -353,7 +361,7 @@ export class ProxyConfiguration {*/async _fetchStatus() {const requestOpts = {- url: APIFY_PROXY_STATUS_URL,+ url: `${this._getApifyProxyStatusUrl()}/?format=json`,proxyUrl: this.newUrl(),json: true,timeoutSecs: CHECK_ACCESS_REQUEST_TIMEOUT_SECS,diff --git a/test/proxy_configuration.test.js b/test/proxy_configuration.test.jsindex fddef0ddd..1332dc005 100644--- a/test/proxy_configuration.test.js+++ b/test/proxy_configuration.test.js@@ -24,6 +24,8 @@ const proxyUrlNoSession = 'http://groups-GROUP1+GROUP2,country-CZ:test12345@proxafterEach(() => {delete process.env[ENV_VARS.TOKEN];delete process.env[ENV_VARS.PROXY_PASSWORD];+ delete process.env[ENV_VARS.PROXY_HOSTNAME];+ delete process.env[ENV_VARS.PROXY_STATUS_URL];});describe('ProxyConfiguration', () => {@@ -147,7 +149,6 @@ describe('ProxyConfiguration', () => {newUrlFunction,});try {- // eslint-disable-next-line no-unused-varsproxyConfiguration.newUrl();throw new Error('wrong error');} catch (err) {@@ -396,8 +397,7 @@ describe('Apify.createProxyConfiguration()', () => {const stub = sinon.stub(requestUtils, 'requestAsBrowser').callsFake(fakeCall);try {- // eslint-disable-next-line no-unused-vars- const proxyConfiguration = await Apify.createProxyConfiguration();+ await Apify.createProxyConfiguration();throw new Error('wrong error');} catch (err) {expect(err.message).toMatch('Apify Proxy password must be provided');@@ -422,8 +422,7 @@ describe('Apify.createProxyConfiguration()', () => {.returns({ get: async () => userData });try {- // eslint-disable-next-line no-unused-vars- const proxyConfiguration = await Apify.createProxyConfiguration({ groups });+ await Apify.createProxyConfiguration({ groups });throw new Error('wrong error');} catch (err) {expect(err.message).toMatch(connectionError);@@ -449,4 +448,23 @@ describe('Apify.createProxyConfiguration()', () => {requestUtilsMock.verify();logMock.verify();});++ test('should connect to proxy in environments other than production', async () => {+ process.env.APIFY_PROXY_STATUS_URL = 'http://proxy-domain.apify.com';+ process.env.APIFY_PROXY_HOSTNAME = 'proxy-domain.apify.com';+ process.env.APIFY_PROXY_PASSWORD = password;++ const mock = sinon.mock(requestUtils);+ mock.expects('requestAsBrowser')+ .once()+ .withArgs(sinon.match({+ url: `${process.env.APIFY_PROXY_STATUS_URL}/?format=json`,+ proxyUrl: `http://auto:${password}@${process.env.APIFY_PROXY_HOSTNAME}:8000`,+ }))+ .resolves({ body: { connected: true } });++ await Apify.createProxyConfiguration();++ mock.verify();+ });});