Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Fix proxy configuration to support dynamic proxy status url (#940)

Andrey Bykov7eeefede9ca3
diff --git a/src/proxy_configuration.js b/src/proxy_configuration.js
index 6932e7f07..c46bd0bcb 100644
--- a/src/proxy_configuration.js
+++ b/src/proxy_configuration.js
@@ -8,7 +8,6 @@ import defaultLog from './utils_log';
// CONSTANTS
const PROTOCOL = 'http';
-const APIFY_PROXY_STATUS_URL = 'http://proxy.apify.com/?format=json';
// https://docs.apify.com/proxy/datacenter-proxy#username-parameters
const MAX_SESSION_ID_LENGTH = 50;
const CHECK_ACCESS_REQUEST_TIMEOUT_SECS = 4;
@@ -301,6 +300,15 @@ export class ProxyConfiguration {
return username;
}
+ /**
+ * Returns Apify proxy status url.
+ * @return {string} the Apify proxy status url
+ * @ignore
+ */
+ _getApifyProxyStatusUrl() {
+ return process.env[ENV_VARS.PROXY_STATUS_URL] || `${PROTOCOL}://${LOCAL_ENV_VARS[ENV_VARS.PROXY_HOSTNAME]}`;
+ }
+
/**
* Checks if Apify Token is provided in env
* and gets the password via API and sets it to env
@@ -353,7 +361,7 @@ export class ProxyConfiguration {
*/
async _fetchStatus() {
const requestOpts = {
- url: APIFY_PROXY_STATUS_URL,
+ url: `${this._getApifyProxyStatusUrl()}/?format=json`,
proxyUrl: this.newUrl(),
json: true,
timeoutSecs: CHECK_ACCESS_REQUEST_TIMEOUT_SECS,
diff --git a/test/proxy_configuration.test.js b/test/proxy_configuration.test.js
index fddef0ddd..1332dc005 100644
--- a/test/proxy_configuration.test.js
+++ b/test/proxy_configuration.test.js
@@ -24,6 +24,8 @@ const proxyUrlNoSession = 'http://groups-GROUP1+GROUP2,country-CZ:test12345@prox
afterEach(() => {
delete process.env[ENV_VARS.TOKEN];
delete process.env[ENV_VARS.PROXY_PASSWORD];
+ delete process.env[ENV_VARS.PROXY_HOSTNAME];
+ delete process.env[ENV_VARS.PROXY_STATUS_URL];
});
describe('ProxyConfiguration', () => {
@@ -147,7 +149,6 @@ describe('ProxyConfiguration', () => {
newUrlFunction,
});
try {
- // eslint-disable-next-line no-unused-vars
proxyConfiguration.newUrl();
throw new Error('wrong error');
} catch (err) {
@@ -396,8 +397,7 @@ describe('Apify.createProxyConfiguration()', () => {
const stub = sinon.stub(requestUtils, 'requestAsBrowser').callsFake(fakeCall);
try {
- // eslint-disable-next-line no-unused-vars
- const proxyConfiguration = await Apify.createProxyConfiguration();
+ await Apify.createProxyConfiguration();
throw new Error('wrong error');
} catch (err) {
expect(err.message).toMatch('Apify Proxy password must be provided');
@@ -422,8 +422,7 @@ describe('Apify.createProxyConfiguration()', () => {
.returns({ get: async () => userData });
try {
- // eslint-disable-next-line no-unused-vars
- const proxyConfiguration = await Apify.createProxyConfiguration({ groups });
+ await Apify.createProxyConfiguration({ groups });
throw new Error('wrong error');
} catch (err) {
expect(err.message).toMatch(connectionError);
@@ -449,4 +448,23 @@ describe('Apify.createProxyConfiguration()', () => {
requestUtilsMock.verify();
logMock.verify();
});
+
+ test('should connect to proxy in environments other than production', async () => {
+ process.env.APIFY_PROXY_STATUS_URL = 'http://proxy-domain.apify.com';
+ process.env.APIFY_PROXY_HOSTNAME = 'proxy-domain.apify.com';
+ process.env.APIFY_PROXY_PASSWORD = password;
+
+ const mock = sinon.mock(requestUtils);
+ mock.expects('requestAsBrowser')
+ .once()
+ .withArgs(sinon.match({
+ url: `${process.env.APIFY_PROXY_STATUS_URL}/?format=json`,
+ proxyUrl: `http://auto:${password}@${process.env.APIFY_PROXY_HOSTNAME}:8000`,
+ }))
+ .resolves({ body: { connected: true } });
+
+ await Apify.createProxyConfiguration();
+
+ mock.verify();
+ });
});