Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
For more clarity Apify.getApifyProxyUrl() accepts again 'session' and 'groups' options instead of 'apifyProxySession' and 'apifyProxyGroups' (#111)
Jan Čurn94e2fa5bbd41
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 925242250..2eb57f124 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -1,5 +1,7 @@-xxxxxxxxxxxxxxxxxxx+XXX===================+- Revereted back - `Apify.getApifyProxyUrl()` accepts again `session` and `groups` options instead of+ `apifyProxySession` and `apifyProxyGroups`- Parameter `memory` added to `Apify.call()`.0.5.37 / 2018-06-07diff --git a/src/actor.js b/src/actor.jsindex b2d0e83ea..04388b799 100644--- a/src/actor.js+++ b/src/actor.js@@ -384,9 +384,9 @@ export const call = (actId, input, opts = {}) => {* @param {String} opts.password User's password for the proxy.* By default, it is taken from the `APIFY_PROXY_PASSWORD` environment variable,* which is automatically set by the system when running the acts on the Apify cloud.- * @param {String[]} [opts.apifyProxyGroups] Array of Apify Proxy groups to be used.+ * @param {String[]} [opts.groups] Array of Apify Proxy groups to be used.* If not provided, the proxy will select the groups automatically.- * @param {String} [opts.apifyProxySession] Apify Proxy session identifier to be used by the Chrome browser.+ * @param {String} [opts.session] Apify Proxy session identifier to be used by the Chrome browser.* All HTTP requests going through the proxy with the same session identifier* will use the same target proxy server (i.e. the same IP address).* The identifier can only contain the following characters: `0-9`, `a-z`, `A-Z`, `"."`, `"_"` and `"~"`.@@ -400,18 +400,18 @@ export const call = (actId, input, opts = {}) => {export const getApifyProxyUrl = (opts = {}) => {// For backwards compatibility.// TODO: remove this when we release v1.0.0- if (!opts.apifyProxyGroups && opts.groups) {- log.warning('Parameter `groups` of Apify.getApifyProxyUrl() is deprecated!!! Use `apifyProxyGroups` instead!');- opts.apifyProxyGroups = opts.groups;+ if (!opts.groups && opts.apifyProxyGroups) {+ log.warning('Parameter `apifyProxyGroups` of Apify.getApifyProxyUrl() is deprecated!!! Use `groups` instead!');+ opts.groups = opts.apifyProxyGroups;}- if (!opts.apifyProxySession && opts.session) {- log.warning('Parameter `session` of Apify.getApifyProxyUrl() is deprecated!!! Use `apifyProxySession` instead!');- opts.apifyProxySession = opts.session;+ if (!opts.session && opts.apifyProxySession) {+ log.warning('Parameter `apifyProxySession` of Apify.getApifyProxyUrl() is deprecated!!! Use `session` instead!');+ opts.session = opts.apifyProxySession;}const {- apifyProxyGroups,- apifyProxySession,+ groups,+ session,password = process.env[ENV_VARS.PROXY_PASSWORD],hostname = process.env[ENV_VARS.PROXY_HOSTNAME] || DEFAULT_PROXY_HOSTNAME,port = parseInt(process.env[ENV_VARS.PROXY_PORT], 10) || DEFAULT_PROXY_PORT,@@ -422,24 +422,24 @@ export const getApifyProxyUrl = (opts = {}) => {throw new Error(`The "${param}" option can only contain the following characters: 0-9, a-z, A-Z, ".", "_" and "~"`);};- checkParamOrThrow(apifyProxyGroups, 'opts.apifyProxyGroups', 'Maybe [String]');- checkParamOrThrow(apifyProxySession, 'opts.apifyProxySession', 'Maybe Number | String');+ checkParamOrThrow(groups, 'opts.groups', 'Maybe [String]');+ checkParamOrThrow(session, 'opts.session', 'Maybe Number | String');checkParamOrThrow(password, 'opts.password', 'String', getMissingParamErrorMgs('password', ENV_VARS.PROXY_PASSWORD));checkParamOrThrow(hostname, 'opts.hostname', 'String', getMissingParamErrorMgs('hostname', ENV_VARS.PROXY_HOSTNAME));checkParamOrThrow(port, 'opts.port', 'Number', getMissingParamErrorMgs('port', ENV_VARS.PROXY_PORT));let username;- if (apifyProxyGroups || apifyProxySession) {+ if (groups || session) {const parts = [];- if (apifyProxyGroups && apifyProxyGroups.length) {- if (!apifyProxyGroups.every(group => APIFY_PROXY_VALUE_REGEX.test(group))) throwInvalidProxyValueError('apifyProxyGroups');- parts.push(`GROUPS-${apifyProxyGroups.join('+')}`);+ if (groups && groups.length) {+ if (!groups.every(group => APIFY_PROXY_VALUE_REGEX.test(group))) throwInvalidProxyValueError('groups');+ parts.push(`GROUPS-${groups.join('+')}`);}- if (apifyProxySession) {- if (!APIFY_PROXY_VALUE_REGEX.test(apifyProxySession)) throwInvalidProxyValueError('apifyProxySession');- parts.push(`SESSION-${apifyProxySession}`);+ if (session) {+ if (!APIFY_PROXY_VALUE_REGEX.test(session)) throwInvalidProxyValueError('session');+ parts.push(`SESSION-${session}`);}username = parts.join(',');diff --git a/src/puppeteer.js b/src/puppeteer.jsindex 3d613bfe1..053c234f3 100644--- a/src/puppeteer.js+++ b/src/puppeteer.js@@ -170,9 +170,10 @@ export const launchPuppeteer = (opts = {}) => {optsCopy.executablePath = process.env[ENV_VARS.CHROME_EXECUTABLE_PATH] || getTypicalChromeExecutablePath();}if (optsCopy.useApifyProxy) {- const { apifyProxyGroups, apifyProxySession } = optsCopy;-- optsCopy.proxyUrl = getApifyProxyUrl({ apifyProxyGroups, apifyProxySession });+ optsCopy.proxyUrl = getApifyProxyUrl({+ groups: optsCopy.apifyProxyGroups,+ session: optsCopy.apifyProxySession,+ });}// When User-Agent is not set and we're using Chromium or headless mode,diff --git a/test/actor.js b/test/actor.jsindex 0473ec038..4a1febd5d 100644--- a/test/actor.js+++ b/test/actor.js@@ -705,16 +705,16 @@ describe('Apify.getApifyProxyUrl()', () => {process.env[ENV_VARS.PROXY_PORT] = 123;expect(Apify.getApifyProxyUrl({- apifyProxySession: 'XYZ',- apifyProxyGroups: ['g1', 'g2', 'g3'],+ session: 'XYZ',+ groups: ['g1', 'g2', 'g3'],})).to.be.eql('http://GROUPS-g1+g2+g3,SESSION-XYZ:[email protected]:123');expect(Apify.getApifyProxyUrl({- apifyProxyGroups: ['g1', 'g2', 'g3'],+ groups: ['g1', 'g2', 'g3'],})).to.be.eql('http://GROUPS-g1+g2+g3:[email protected]:123');expect(Apify.getApifyProxyUrl({- apifyProxySession: 'XYZ',+ session: 'XYZ',})).to.be.eql('http://SESSION-XYZ:[email protected]:123');expect(Apify.getApifyProxyUrl()).to.be.eql('http://auto:[email protected]:123');@@ -741,16 +741,16 @@ describe('Apify.getApifyProxyUrl()', () => {process.env[ENV_VARS.PROXY_PORT] = 123;expect(Apify.getApifyProxyUrl({- session: 'XYZ',- groups: ['g1', 'g2', 'g3'],+ apifyProxySession: 'XYZ',+ apifyProxyGroups: ['g1', 'g2', 'g3'],})).to.be.eql('http://GROUPS-g1+g2+g3,SESSION-XYZ:[email protected]:123');expect(Apify.getApifyProxyUrl({- groups: ['g1', 'g2', 'g3'],+ apifyProxyGroups: ['g1', 'g2', 'g3'],})).to.be.eql('http://GROUPS-g1+g2+g3:[email protected]:123');expect(Apify.getApifyProxyUrl({- session: 'XYZ',+ apifyProxySession: 'XYZ',})).to.be.eql('http://SESSION-XYZ:[email protected]:123');expect(Apify.getApifyProxyUrl()).to.be.eql('http://auto:[email protected]:123');@@ -779,6 +779,7 @@ describe('Apify.getApifyProxyUrl()', () => {expect(() => Apify.getApifyProxyUrl({ session: 'a$b' })).to.throw();expect(() => Apify.getApifyProxyUrl({ session: {} })).to.throw();expect(() => Apify.getApifyProxyUrl({ session: new Date() })).to.throw();+ expect(() => Apify.getApifyProxyUrl({ apifyProxySession: new Date() })).to.throw();expect(() => Apify.getApifyProxyUrl({ session: 'a_b' })).to.not.throw();expect(() => Apify.getApifyProxyUrl({ session: '0.34252352' })).to.not.throw();@@ -788,6 +789,12 @@ describe('Apify.getApifyProxyUrl()', () => {expect(() => Apify.getApifyProxyUrl({ session: 'a' })).to.not.throw();expect(() => Apify.getApifyProxyUrl({ session: '1' })).to.not.throw();+ expect(() => Apify.getApifyProxyUrl({ groups: [new Date()] })).to.throw();+ expect(() => Apify.getApifyProxyUrl({ groups: [{}, 'fff', 'ccc'] })).to.throw();+ expect(() => Apify.getApifyProxyUrl({ groups: ['ffff', 'ff-hf', 'ccc'] })).to.throw();+ expect(() => Apify.getApifyProxyUrl({ groups: ['ffff', 'fff', 'cc$c'] })).to.throw();+ expect(() => Apify.getApifyProxyUrl({ apifyProxyGroups: [new Date()] })).to.throw();+delete process.env[ENV_VARS.PROXY_PASSWORD];delete process.env[ENV_VARS.PROXY_HOSTNAME];delete process.env[ENV_VARS.PROXY_PORT];diff --git a/test/puppeteer.js b/test/puppeteer.jsindex 249fcba1b..7be25fb8e 100644--- a/test/puppeteer.js+++ b/test/puppeteer.js@@ -198,8 +198,8 @@ describe('Apify.launchPuppeteer()', () => {mock.expects('getApifyProxyUrl').once().withArgs({- apifyProxySession: 'xxx',- apifyProxyGroups: ['yyy'],+ session: 'xxx',+ groups: ['yyy'],}).returns(null); // Return null so that it doesn't start proxy-chain