Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Fixed Apify.getApifyProxyUrl() to use proper regex for checks (#84)

Jan Čurn237ce8d6b4c2
diff --git a/package.json b/package.json
index 0854be302..376c62273 100644
--- a/package.json
+++ b/package.json
@@ -45,7 +45,7 @@
},
"dependencies": {
"apify-client": "^0.2.8",
- "apify-shared": "^0.0.32",
+ "apify-shared": "^0.0.34",
"bluebird": "^3.5.0",
"content-type": "^1.0.3",
"fs-extra": "^5.0.0",
diff --git a/src/actor.js b/src/actor.js
index 85ce53791..e2ec216c2 100644
--- a/src/actor.js
+++ b/src/actor.js
@@ -3,7 +3,7 @@ import Promise from 'bluebird';
import fs from 'fs';
import log from 'apify-shared/log';
import { checkParamOrThrow } from 'apify-client/build/utils';
-import { PROXY_GROUP_NAME_REGEX, PROXY_SESSION_ID_REGEX } from 'apify-shared/regexs';
+import { APIFY_PROXY_VALUE_REGEX } from 'apify-shared/regexs';
import { ENV_VARS, EXIT_CODES, ACT_TASK_TERMINAL_STATUSES, ACT_TASK_STATUSES, DEFAULT_PROXY_HOSTNAME, DEFAULT_PROXY_PORT } from './constants';
import { initializeEvents, stopEvents } from './events';
import { newPromise, apifyClient, addCharsetToContentType } from './utils';
@@ -416,7 +416,9 @@ export const getApifyProxyUrl = (opts = {}) => {
} = opts;
const getMissingParamErrorMgs = (param, env) => `Apify Proxy ${param} must be provided as parameter or "${env}" environment variable!`;
- const throwNonAlphanumericError = (param) => { throw new Error(`Only alphanumeric characters and underscore are allowed in ${param}`); };
+ const throwInvalidProxyValueError = (param) => {
+ throw new Error(`The "${param}" option can only contain the following characters: 0-9, a-z, A-Z, ".", "_" and "~"`);
+ };
checkParamOrThrow(apifyProxyGroups, 'opts.apifyProxyGroups', 'Maybe [String]');
checkParamOrThrow(apifyProxySession, 'opts.apifyProxySession', 'Maybe Number | String');
@@ -430,11 +432,11 @@ export const getApifyProxyUrl = (opts = {}) => {
const parts = [];
if (apifyProxyGroups && apifyProxyGroups.length) {
- if (!apifyProxyGroups.every(group => PROXY_GROUP_NAME_REGEX.test(group))) throwNonAlphanumericError('proxy group name');
+ if (!apifyProxyGroups.every(group => APIFY_PROXY_VALUE_REGEX.test(group))) throwInvalidProxyValueError('apifyProxyGroups');
parts.push(`GROUPS-${apifyProxyGroups.join('+')}`);
}
if (apifyProxySession) {
- if (!PROXY_SESSION_ID_REGEX.test(apifyProxySession)) throwNonAlphanumericError('proxy session ID');
+ if (!APIFY_PROXY_VALUE_REGEX.test(apifyProxySession)) throwInvalidProxyValueError('apifyProxySession');
parts.push(`SESSION-${apifyProxySession}`);
}
diff --git a/test/actor.js b/test/actor.js
index 7ea1f35a6..e0fd098b8 100644
--- a/test/actor.js
+++ b/test/actor.js
@@ -774,12 +774,14 @@ describe('Apify.getApifyProxyUrl()', () => {
process.env[ENV_VARS.PROXY_HOSTNAME] = 'my.host.com';
process.env[ENV_VARS.PROXY_PORT] = 123;
- expect(() => Apify.getApifyProxyUrl({ session: 'a.b' })).to.throw();
expect(() => Apify.getApifyProxyUrl({ session: 'a-b' })).to.throw();
- expect(() => Apify.getApifyProxyUrl({ session: 'ab_' })).to.throw();
expect(() => Apify.getApifyProxyUrl({ session: 'a$b' })).to.throw();
+ expect(() => Apify.getApifyProxyUrl({ session: {} })).to.throw();
+ expect(() => Apify.getApifyProxyUrl({ session: new Date() })).to.throw();
expect(() => Apify.getApifyProxyUrl({ session: 'a_b' })).to.not.throw();
+ expect(() => Apify.getApifyProxyUrl({ session: '0.34252352' })).to.not.throw();
+ expect(() => Apify.getApifyProxyUrl({ session: 'aaa~BBB' })).to.not.throw();
expect(() => Apify.getApifyProxyUrl({ session: 'a_1_b' })).to.not.throw();
expect(() => Apify.getApifyProxyUrl({ session: 'a_2' })).to.not.throw();
expect(() => Apify.getApifyProxyUrl({ session: 'a' })).to.not.throw();