Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Added redactUrl() and redactParsedUrl() functions to utils with unit tests, bumped package.json version
Jan Curne4e0c55653e4
diff --git a/package.json b/package.jsonindex d0b5a1647..4490c5933 100644--- a/package.json+++ b/package.json@@ -1,6 +1,6 @@{"name": "apify",- "version": "0.4.8",+ "version": "0.4.9","description": "Apify runtime for JavaScript","main": "build/index.js","keywords": [diff --git a/src/proxy_chain.js b/src/proxy_chain.jsindex ec69fd249..55f4810a7 100644--- a/src/proxy_chain.js+++ b/src/proxy_chain.js@@ -5,7 +5,7 @@ import tmp from 'tmp';import portastic from 'portastic';import Promise from 'bluebird';import rimraf from 'rimraf';-import { parseUrl } from './utils';+import { parseUrl, redactParsedUrl } from './utils';/* globals process */@@ -87,8 +87,7 @@ export class ProxyChain {}_getParentProxyUrl() {- const p = this.parsedProxyUrl;- return `${p.protocol}//${p.username || ''}:${p.password ? '<redacted>' : ''}@${p.host}`;+ return redactParsedUrl(this.parsedProxyUrl);}/**diff --git a/src/utils.js b/src/utils.jsindex 0f6a8ef11..38921a9b2 100644--- a/src/utils.js+++ b/src/utils.js@@ -111,3 +111,30 @@ export const parseUrl = (url) => {return parsed;};+++/**+ * Redacts password from a URL, so that it can be shown in logs, results etc.+ * For example, converts URL such as+ * 'https://username:[email protected]/path#hash'+ * to 'https://username:<redacted>@www.example.com/path#hash'+ * @param url URL, it must contain at least protocol and hostname+ * @param passwordReplacement The string that replaces password, by default it is '<redacted>'+ * @return {string}+ */+export const redactUrl = (url, passwordReplacement) => {+ return redactParsedUrl(parseUrl(url), passwordReplacement);+};++export const redactParsedUrl = (parsedUrl, passwordReplacement = '<redacted>') => {+ const p = parsedUrl;+ let auth = null;+ if (p.username) {+ if (p.password) {+ auth = `${p.username}:${passwordReplacement}`;+ } else {+ auth = `${p.username}`;+ }+ }+ return `${p.protocol}//${auth || ''}${auth ? '@' : ''}${p.host}${p.path || ''}${p.hash || ''}`;+};diff --git a/test/utils.js b/test/utils.jsindex 336cfe432..ec31dcc21 100644--- a/test/utils.js+++ b/test/utils.js@@ -111,3 +111,26 @@ describe('utils.parseUrl()', () => {});});});+++describe('utils.redactUrl()', () => {+ it('works', () => {+ expect(utils.redactUrl('https://username:[email protected]:1234/path#hash'))+ .to.eql('https://username:<redacted>@www.example.com:1234/path#hash');++ expect(utils.redactUrl('https://[email protected]:1234/path#hash'))+ .to.eql('https://[email protected]:1234/path#hash');++ expect(utils.redactUrl('https://username:[email protected]:1234/path#hash', '<xxx>'))+ .to.eql('https://username:<xxx>@www.example.com:1234/path#hash');++ expect(utils.redactUrl('ftp://@www.example.com/path/path2'))+ .to.eql('ftp://www.example.com/path/path2');++ expect(utils.redactUrl('ftp://www.example.com'))+ .to.eql('ftp://www.example.com/');++ expect(utils.redactUrl('ftp://example.com/'))+ .to.eql('ftp://example.com/');+ });+});