Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Fixed Apify.getApifyProxyUrl() to use proper regex for checks (#84)
Jan Čurn237ce8d6b4c2
diff --git a/package.json b/package.jsonindex 0854be302..376c62273 100644--- a/package.json+++ b/package.json@@ -45,7 +45,7 @@},"dependencies": {"apify-client": "^0.2.8",- "apify-shared": "^0.0.32",+ "apify-shared": "^0.0.34","bluebird": "^3.5.0","content-type": "^1.0.3","fs-extra": "^5.0.0",diff --git a/src/actor.js b/src/actor.jsindex 85ce53791..e2ec216c2 100644--- a/src/actor.js+++ b/src/actor.js@@ -3,7 +3,7 @@ import Promise from 'bluebird';import fs from 'fs';import log from 'apify-shared/log';import { checkParamOrThrow } from 'apify-client/build/utils';-import { PROXY_GROUP_NAME_REGEX, PROXY_SESSION_ID_REGEX } from 'apify-shared/regexs';+import { APIFY_PROXY_VALUE_REGEX } from 'apify-shared/regexs';import { ENV_VARS, EXIT_CODES, ACT_TASK_TERMINAL_STATUSES, ACT_TASK_STATUSES, DEFAULT_PROXY_HOSTNAME, DEFAULT_PROXY_PORT } from './constants';import { initializeEvents, stopEvents } from './events';import { newPromise, apifyClient, addCharsetToContentType } from './utils';@@ -416,7 +416,9 @@ export const getApifyProxyUrl = (opts = {}) => {} = opts;const getMissingParamErrorMgs = (param, env) => `Apify Proxy ${param} must be provided as parameter or "${env}" environment variable!`;- const throwNonAlphanumericError = (param) => { throw new Error(`Only alphanumeric characters and underscore are allowed in ${param}`); };+ const throwInvalidProxyValueError = (param) => {+ throw new Error(`The "${param}" option can only contain the following characters: 0-9, a-z, A-Z, ".", "_" and "~"`);+ };checkParamOrThrow(apifyProxyGroups, 'opts.apifyProxyGroups', 'Maybe [String]');checkParamOrThrow(apifyProxySession, 'opts.apifyProxySession', 'Maybe Number | String');@@ -430,11 +432,11 @@ export const getApifyProxyUrl = (opts = {}) => {const parts = [];if (apifyProxyGroups && apifyProxyGroups.length) {- if (!apifyProxyGroups.every(group => PROXY_GROUP_NAME_REGEX.test(group))) throwNonAlphanumericError('proxy group name');+ if (!apifyProxyGroups.every(group => APIFY_PROXY_VALUE_REGEX.test(group))) throwInvalidProxyValueError('apifyProxyGroups');parts.push(`GROUPS-${apifyProxyGroups.join('+')}`);}if (apifyProxySession) {- if (!PROXY_SESSION_ID_REGEX.test(apifyProxySession)) throwNonAlphanumericError('proxy session ID');+ if (!APIFY_PROXY_VALUE_REGEX.test(apifyProxySession)) throwInvalidProxyValueError('apifyProxySession');parts.push(`SESSION-${apifyProxySession}`);}diff --git a/test/actor.js b/test/actor.jsindex 7ea1f35a6..e0fd098b8 100644--- a/test/actor.js+++ b/test/actor.js@@ -774,12 +774,14 @@ describe('Apify.getApifyProxyUrl()', () => {process.env[ENV_VARS.PROXY_HOSTNAME] = 'my.host.com';process.env[ENV_VARS.PROXY_PORT] = 123;- expect(() => Apify.getApifyProxyUrl({ session: 'a.b' })).to.throw();expect(() => Apify.getApifyProxyUrl({ session: 'a-b' })).to.throw();- expect(() => Apify.getApifyProxyUrl({ session: 'ab_' })).to.throw();expect(() => Apify.getApifyProxyUrl({ session: 'a$b' })).to.throw();+ expect(() => Apify.getApifyProxyUrl({ session: {} })).to.throw();+ expect(() => Apify.getApifyProxyUrl({ session: new Date() })).to.throw();expect(() => Apify.getApifyProxyUrl({ session: 'a_b' })).to.not.throw();+ expect(() => Apify.getApifyProxyUrl({ session: '0.34252352' })).to.not.throw();+ expect(() => Apify.getApifyProxyUrl({ session: 'aaa~BBB' })).to.not.throw();expect(() => Apify.getApifyProxyUrl({ session: 'a_1_b' })).to.not.throw();expect(() => Apify.getApifyProxyUrl({ session: 'a_2' })).to.not.throw();expect(() => Apify.getApifyProxyUrl({ session: 'a' })).to.not.throw();