Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Fix proxy check crashes (#762)

Ondra Urban42aa9933be5e
diff --git a/src/proxy_configuration.js b/src/proxy_configuration.js
index 681b3e28a..f8cfd11e7 100644
--- a/src/proxy_configuration.js
+++ b/src/proxy_configuration.js
@@ -1,17 +1,18 @@
import { checkParamOrThrow } from 'apify-client/build/utils';
import { ENV_VARS, LOCAL_ENV_VARS } from 'apify-shared/consts';
import { APIFY_PROXY_VALUE_REGEX } from 'apify-shared/regexs';
-import { URL } from 'url';
import { COUNTRY_CODE_REGEX } from './constants';
import { apifyClient } from './utils';
import { requestAsBrowser } from './utils_request';
-import log from './utils_log';
+import defaultLog from './utils_log';
// CONSTANTS
const PROTOCOL = 'http';
const APIFY_PROXY_STATUS_URL = 'http://proxy.apify.com/?format=json';
// https://docs.apify.com/proxy/datacenter-proxy#username-parameters
const MAX_SESSION_ID_LENGTH = 50;
+const CHECK_ACCESS_REQUEST_TIMEOUT_SECS = 4;
+const CHECK_ACCESS_MAX_ATTEMPTS = 2;
/**
* @typedef ProxyConfigurationOptions
@@ -184,6 +185,7 @@ export class ProxyConfiguration {
this.usedProxyUrls = new Map();
this.newUrlFunction = newUrlFunction;
this.usesApifyProxy = !this.proxyUrls && !this.newUrlFunction;
+ this.log = defaultLog.child({ prefix: 'ProxyConfiguration' });
}
/**
@@ -302,7 +304,7 @@ export class ProxyConfiguration {
const { proxy: { password } } = await apifyClient.users.getUser({ token, userId: 'me' });
if (this.password) {
if (this.password !== password) {
- log.warning('The Apify Proxy password you provided belongs to'
+ this.log.warning('The Apify Proxy password you provided belongs to'
+ ' a different user than the Apify token you are using. Are you sure this is correct?');
}
} else {
@@ -316,16 +318,45 @@ export class ProxyConfiguration {
}
/**
- * Checks the status of Apify Proxy and throws an error if the status is not "connected".
+ * Checks whether the user has access to the proxies specified in the provided ProxyConfigurationOptions.
+ * If the check can not be made, it only prints a warning and allows the program to continue. This is to
+ * prevent program crashes caused by short downtimes of Proxy.
+ *
* @returns {Promise<void>}
* @ignore
*/
async _checkAccess() {
- const url = APIFY_PROXY_STATUS_URL;
- const proxyUrl = this.newUrl();
- const { countryCode } = this;
- const { body: { connected, connectionError } } = await requestAsBrowser({ url, proxyUrl, countryCode, json: true });
- if (!connected) this._throwApifyProxyConnectionError(connectionError);
+ const status = await this._fetchStatus();
+ if (status) {
+ const { connected, connectionError } = status;
+ if (!connected) this._throwApifyProxyConnectionError(connectionError);
+ } else {
+ this.log.warning('Apify Proxy access check timed out. Watch out for errors with status code 407. '
+ + 'If you see some, it most likely means you don\'t have access to either all or some of the proxies you\'re trying to use.');
+ }
+ }
+
+ /**
+ * Apify Proxy can be down for a second or a minute, but this should not crash processes.
+ *
+ * @return {Promise<?{ connected: boolean, connectionError: string }>}
+ * @ignore
+ */
+ async _fetchStatus() {
+ const requestOpts = {
+ url: APIFY_PROXY_STATUS_URL,
+ proxyUrl: this.newUrl(),
+ json: true,
+ timeoutSecs: CHECK_ACCESS_REQUEST_TIMEOUT_SECS,
+ };
+ for (let attempt = 1; attempt <= CHECK_ACCESS_MAX_ATTEMPTS; attempt++) {
+ try {
+ const response = await requestAsBrowser(requestOpts);
+ return response.body;
+ } catch (err) {
+ // retry connection errors
+ }
+ }
}
/**
diff --git a/test/proxy_configuration.test.js b/test/proxy_configuration.test.js
index 98e0608e3..44ecdbf6e 100644
--- a/test/proxy_configuration.test.js
+++ b/test/proxy_configuration.test.js
@@ -4,7 +4,6 @@ import Apify from '../build/index';
import * as requestUtils from '../build/utils_request';
import * as utils from '../build/utils';
import { ProxyConfiguration } from '../build/proxy_configuration';
-import log from '../build/utils_log';
const { apifyClient } = utils;
@@ -22,6 +21,11 @@ const basicOpts = {
const basicOptsProxyUrl = 'http://groups-GROUP1+GROUP2,session-538909250932,country-CZ:[email protected]:8000';
const proxyUrlNoSession = 'http://groups-GROUP1+GROUP2,country-CZ:[email protected]:8000';
+afterEach(() => {
+ delete process.env[ENV_VARS.TOKEN];
+ delete process.env[ENV_VARS.PROXY_PASSWORD];
+});
+
describe('ProxyConfiguration', () => {
test('should accept all options', () => {
const proxyConfiguration = new ProxyConfiguration(basicOpts);
@@ -306,7 +310,7 @@ describe('Apify.createProxyConfiguration()', () => {
mock.expects('requestAsBrowser')
.once()
- .withArgs({ url, proxyUrl, countryCode, json: true })
+ .withArgs(sinon.match({ url, proxyUrl }))
.resolves({ body: status });
const proxyConfiguration = await Apify.createProxyConfiguration(basicOpts);
@@ -322,8 +326,10 @@ describe('Apify.createProxyConfiguration()', () => {
});
test('should work without password (with token)', async () => {
- const opts = basicOpts;
- opts.password = null;
+ const token = '123456789';
+ process.env.APIFY_TOKEN = token;
+ const opts = { ...basicOpts };
+ delete opts.password;
const requestUtilsMock = sinon.mock(requestUtils);
const status = { connected: true };
@@ -332,12 +338,10 @@ describe('Apify.createProxyConfiguration()', () => {
requestUtilsMock.expects('requestAsBrowser')
.once()
- .withArgs({ url, proxyUrl, countryCode, json: true })
+ .withArgs(sinon.match({ url, proxyUrl }))
.resolves({ body: status });
const clientUsersMock = sinon.mock(apifyClient.users);
- const token = '123456789';
- process.env.APIFY_TOKEN = token;
const data = { proxy: { password } };
clientUsersMock.expects('getUser')
@@ -360,30 +364,25 @@ describe('Apify.createProxyConfiguration()', () => {
});
test('should show warning log', async () => {
- const logMock = sinon.mock(log);
process.env.APIFY_TOKEN = '123456789';
const status = { connected: true };
- const fakeUserObjectProxyData = { password: '987654321' };
+ const fakeUserObjectProxyData = { password: 'some-other-users-password' };
- const fakeRequestAsBrowser = async () => {
- return { body: status };
- };
- const stub1 = sinon.stub(requestUtils, 'requestAsBrowser').callsFake(fakeRequestAsBrowser);
+ const stub1 = sinon.stub(requestUtils, 'requestAsBrowser').resolves({ body: status });
- const fakeGetUser = async () => {
- return { proxy: fakeUserObjectProxyData };
- };
- const stub2 = sinon.stub(apifyClient.users, 'getUser').callsFake(fakeGetUser);
+ const stub2 = sinon.stub(apifyClient.users, 'getUser').resolves({ proxy: fakeUserObjectProxyData });
// eslint-disable-next-line no-unused-vars
- const proxyConfiguration = await Apify.createProxyConfiguration(basicOpts);
-
+ const proxyConfiguration = new ProxyConfiguration(basicOpts);
+ const logMock = sinon.mock(proxyConfiguration.log);
logMock.expects('warning').once();
+ await proxyConfiguration.initialize();
+
stub1.restore();
stub2.restore();
- logMock.restore();
+ logMock.verify();
});
test('should throw missing password', async () => {
@@ -411,7 +410,7 @@ describe('Apify.createProxyConfiguration()', () => {
test('should throw when group is not available', async () => {
delete process.env[ENV_VARS.PROXY_PASSWORD];
process.env.APIFY_TOKEN = '123456789';
- const connectionError = 'Invalid username: proxy group &quot;GROUP2&quot; not found or not accessible.';
+ const connectionError = 'Invalid username: proxy group "GROUP2"; not found or not accessible.';
const status = { connected: false, connectionError };
const fakeUserObjectProxyData = { password };
@@ -436,4 +435,22 @@ describe('Apify.createProxyConfiguration()', () => {
stub1.restore();
stub2.restore();
});
+
+ test('should not throw when access check is unresponsive', async () => {
+ process.env.APIFY_PROXY_PASSWORD = '123456789';
+ const requestUtilsMock = sinon.mock(requestUtils);
+
+ requestUtilsMock.expects('requestAsBrowser')
+ .twice()
+ .rejects(new Error('some error'));
+
+ const proxyConfiguration = new ProxyConfiguration();
+ const logMock = sinon.mock(proxyConfiguration.log);
+ logMock.expects('warning').once();
+
+ await proxyConfiguration.initialize();
+
+ requestUtilsMock.verify();
+ logMock.verify();
+ });
});