Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Add sessionId length validation (#720)
Ondra Urbanf0d1b01cac00
diff --git a/docs/guides/proxy_management.md b/docs/guides/proxy_management.mdindex 22653fd3d..3d1c3f6b0 100644--- a/docs/guides/proxy_management.md+++ b/docs/guides/proxy_management.md@@ -141,3 +141,9 @@ const proxyUrl = proxyConfiguration.newUrl();Now your crawlers will use only Residential proxies from the US. Note that you must first get accessto a proxy group before you are able to use it. You can find your available proxy groupsin the [proxy dashboard](https://my.apify.com/proxy).++## Inspecting current proxy in Crawlers+`CheerioCrawler` and `PuppeteerCrawler` grant access to information about the currently used proxy+in their `handlePageFunction` using a [`proxyInfo`](/docs/typedefs/proxy-info) object.+With the object, you can easily access the proxy URL. If you're using Apify Proxy, the other+configuration parameters will also be available in the `proxyInfo` object.diff --git a/src/crawlers/cheerio_crawler.js b/src/crawlers/cheerio_crawler.jsindex 17e9ea50e..aa2fa8ede 100644--- a/src/crawlers/cheerio_crawler.js+++ b/src/crawlers/cheerio_crawler.js@@ -456,13 +456,10 @@ class CheerioCrawler {* @param {Object} options* @param {Request} options.request* @param {AutoscaledPool} options.autoscaledPool- * @param {Session} options.session- * @param {ProxyInfo} options.proxyInfo+ * @param {Session} [options.session]* @ignore*/async _handleRequestFunction({ request, autoscaledPool, session }) {- if (this.prepareRequestFunction) await this.prepareRequestFunction({ request, session });-let proxyInfo;let proxyUrl;if (this.proxyConfiguration) {@@ -470,6 +467,8 @@ class CheerioCrawler {proxyUrl = proxyInfo.url;}+ if (this.prepareRequestFunction) await this.prepareRequestFunction({ request, session, proxyInfo });+const { dom, isXml, body, contentType, response } = await addTimeoutToPromise(this._requestFunction({ request, session, proxyUrl }),this.requestTimeoutMillis,diff --git a/src/proxy_configuration.js b/src/proxy_configuration.jsindex c53e95d61..984d6c83f 100644--- a/src/proxy_configuration.js+++ b/src/proxy_configuration.js@@ -10,6 +10,8 @@ import log from './utils_log';// CONSTANTSconst PROTOCOL = 'http';const APIFY_PROXY_STATUS_URL = 'http://proxy.apify.com/?format=json';+// https://docs.apify.com/proxy/datacenter-proxy#username-parameters+const MAX_SESSION_ID_LENGTH = 50;/*** @typedef ProxyConfigurationOptions@@ -368,7 +370,10 @@ export class ProxyConfiguration {* @ignore*/_validateSessionArgumentStructure(sessionId) {- if (!APIFY_PROXY_VALUE_REGEX.test(sessionId)) this._throwInvalidProxyValueError(sessionId);+ if (!APIFY_PROXY_VALUE_REGEX.test(sessionId)) this._throwInvalidProxyValueError('session', sessionId);+ if (sessionId.length > MAX_SESSION_ID_LENGTH) {+ throw new Error(`The provided sessionId must not be longer than ${MAX_SESSION_ID_LENGTH} characters. Received: ${sessionId.length}`);+ }}/**@@ -378,7 +383,7 @@ export class ProxyConfiguration {*/_validateGroupsStructure(groups) {for (const group of groups) {- if (!APIFY_PROXY_VALUE_REGEX.test(group)) this._throwInvalidProxyValueError(group);+ if (!APIFY_PROXY_VALUE_REGEX.test(group)) this._throwInvalidProxyValueError('group', group);}}@@ -420,10 +425,11 @@ export class ProxyConfiguration {/*** Throws invalid proxy value error* @param {string} param+ * @param {string} value* @ignore*/- _throwInvalidProxyValueError(param) {- throw new Error(`The provided proxy group name "${param}" can only contain the following characters: 0-9, a-z, A-Z, ".", "_" and "~"`);+ _throwInvalidProxyValueError(param, value) {+ throw new Error(`The provided proxy ${param} name "${value}" can only contain the following characters: 0-9, a-z, A-Z, ".", "_" and "~"`);}/**diff --git a/test/proxy_configuration.test.js b/test/proxy_configuration.test.jsindex 69c4448d9..98e0608e3 100644--- a/test/proxy_configuration.test.js+++ b/test/proxy_configuration.test.js@@ -116,13 +116,14 @@ describe('ProxyConfiguration', () => {expect(() => new ProxyConfiguration({ countryCode: 1111 })).toThrow();});- test('newUrl() should throw invalid session argument', () => {+ test('newUrl() should throw on invalid session argument', () => {const proxyConfiguration = new ProxyConfiguration();expect(() => proxyConfiguration.newUrl('a-b')).toThrowError();expect(() => proxyConfiguration.newUrl('a$b')).toThrowError();expect(() => proxyConfiguration.newUrl({})).toThrowError();expect(() => proxyConfiguration.newUrl(new Date())).toThrowError();+ expect(() => proxyConfiguration.newUrl(Array(51).fill('x').join(''))).toThrowError();expect(() => proxyConfiguration.newUrl('a_b')).not.toThrowError();expect(() => proxyConfiguration.newUrl('0.34252352')).not.toThrowError();@@ -132,6 +133,7 @@ describe('ProxyConfiguration', () => {expect(() => proxyConfiguration.newUrl('a')).not.toThrowError();expect(() => proxyConfiguration.newUrl('1')).not.toThrowError();expect(() => proxyConfiguration.newUrl(123456)).not.toThrowError();+ expect(() => proxyConfiguration.newUrl(Array(50).fill('x').join(''))).not.toThrowError();});test('should throw on invalid newUrlFunction', async () => {