Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix(utils): read mailto links with a query or several recipients (#4335)
SwastikTripathib18c587f83d2
diff --git a/packages/utils/src/internals/social.ts b/packages/utils/src/internals/social.tsindex 0cb983e37..cbbc3354a 100644--- a/packages/utils/src/internals/social.ts+++ b/packages/utils/src/internals/social.ts@@ -36,6 +36,8 @@ export function emailsFromText(text: string): string[] {/*** The function extracts email addresses from a list of URLs.* Basically it looks for all `mailto:` URLs and returns valid email addresses from them.+ * A `mailto:` URL can list several comma-separated, percent-encoded addresses and end with a query+ * such as `?subject=Hello`, see [RFC 6068](https://www.rfc-editor.org/rfc/rfc6068).* Note that the function preserves the order of emails and keep duplicates.* @param urls Array of URLs.* @return Array of emails addresses found.@@ -49,8 +51,18 @@ export function emailsFromUrls(urls: string[]): string[] {if (!url) continue;if (!EMAIL_URL_PREFIX_REGEX.test(url)) continue;- const email = url.replace(EMAIL_URL_PREFIX_REGEX, '').trim();- if (EMAIL_REGEX.test(email)) emails.push(email);+ // Characters like `?` and `,` inside an address must be percent-encoded, so it's safe+ // to split on them before decoding.+ const [addresses] = url.replace(EMAIL_URL_PREFIX_REGEX, '').split('?');+ for (const address of addresses.split(',')) {+ let email = address.trim();+ try {+ email = decodeURIComponent(email).trim();+ } catch {+ // Keep the raw value if it isn't valid percent-encoding+ }+ if (EMAIL_REGEX.test(email)) emails.push(email);+ }}return emails;diff --git a/test/utils/social.test.ts b/test/utils/social.test.tsindex 49dd5e35a..2ce73fd72 100644--- a/test/utils/social.test.ts+++ b/test/utils/social.test.ts@@ -141,6 +141,23 @@ describe('utils.social', () => {]),).toEqual(['[email protected]', '[email protected]']);});++ test('handles queries, several recipients and percent-encoding', () => {+ expect(+ emailsFromUrls([+ 'mailto:[email protected]?subject=Hello%20there',+ 'mailto:david%40example.com',+ 'MAILTO:[email protected]?',+ ]),+ ).toEqual([+ '[email protected]',+ '[email protected]',+ '[email protected]',+ '[email protected]',+ '[email protected]',+ ]);+ });});describe('phonesFromText()', () => {