Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Development of new Apify.utils.social functions

Jan Curn73ef60dabd1f
diff --git a/src/utils_social.js b/src/utils_social.js
index 2ea33f029..7caa14117 100644
--- a/src/utils_social.js
+++ b/src/utils_social.js
@@ -27,7 +27,7 @@ const emailsFromText = (text) => {
* The function extracts email addresses from a list of URLs.
* Basically it looks for all `mailto:` URLs and returns valid email addresses from them.
* Note that the function preserves the order of emails and keep duplicates.
- * @param {String[]} Array of URLs.
+ * @param {String[]} urls Array of URLs.
* @return {String[]} Array of emails addresses found.
* If no emails are found, the function returns an empty array.
* @memberOf utils.social
@@ -47,6 +47,30 @@ const emailsFromUrls = (urls) => {
};
+/**
+ * The function extracts email addresses from a HTML text.
+ * It looks for `mailto:` links as well as email addresses in the text content.
+ * Note that the function preserves the order of emails and keep duplicates.
+ * @param {String} html HTML document
+ * @return {String[]} Array of emails addresses found.
+ * If no emails are found, the function returns an empty array.
+ * @memberOf utils.social
+ */
+const emailsFromHtml = (html) => {
+ if (!Array.isArray(urls)) throw new Error('The "urls" parameter must be an array');
+
+ const emails = [];
+ for (const url of urls) {
+ if (!url) continue;
+ if (!EMAIL_URL_PREFIX_REGEX.test(url)) continue;
+
+ const email = url.replace(EMAIL_URL_PREFIX_REGEX, '').trim();
+ if (EMAIL_REGEX.test(email)) emails.push(email);
+ }
+ return emails;
+};
+
+
const LINKEDIN_URL_REGEX = /http(s)?:\/\/[a-zA-Z]+\.linkedin\.com\/in\/[a-zA-Z0-9\-_%]+/g;
const INSTAGRAM_URL_REGEX = /(?:(^|[^0-9a-z]))(((http|https):\/\/)?((www\.)?(?:instagram.com|instagr.am)\/([A-Za-z0-9_.]{2,30})))/ig;
// eslint-disable-next-line max-len, no-useless-escape
@@ -108,7 +132,7 @@ const SKIP_PHONE_REGEXS = [
const PHONE_REGEX_GLOBAL = new RegExp(`(${PHONE_REGEXS_STRINGS.join('|')})`, 'ig');
-const PHONE_REGEX = new RegExp(`^(${PHONE_REGEXS_STRINGS.join('|')})$`, 'ig');
+const PHONE_REGEX = new RegExp(`^(${PHONE_REGEXS_STRINGS.join('|')})$`, 'i');
const SKIP_PHONE_REGEX = new RegExp(`^(${SKIP_PHONE_REGEXS.join('|')})$`, 'i');
@@ -150,10 +174,12 @@ const phonesFromText = (text) => {
* @memberOf utils
*/
const phonesFromUrls = (urls) => {
+ if (!Array.isArray(urls)) throw new Error('The "urls" parameter must be an array');
+
const phones = [];
for (const url of urls) {
if (!url) continue;
- if (!EMAIL_URL_PREFIX_REGEX.test(url)) continue;
+ if (!PHONE_URL_PREFIX_REGEX.test(url)) continue;
const phone = url.replace(PHONE_URL_PREFIX_REGEX, '').trim();
if (PHONE_REGEX.test(phone)) phones.push(phone);
diff --git a/test/utils_social.js b/test/utils_social.js
index a0ce85695..5f6e2b927 100644
--- a/test/utils_social.js
+++ b/test/utils_social.js
@@ -267,48 +267,37 @@ describe('utils.social.phonesFromUrls()', () => {
expect(social.phonesFromUrls([
'tel:12345678',
- 'tel:/12345678',
- 'tel://12345678',
- 'phone:12345678',
- 'phone:/12345678',
- 'phone://12345678',
- 'telephone:12345678',
- 'telephone:/12345678',
- 'telephone://12345678',
+ 'tel:/22345678', //
+ 'tel://32345678',
+ 'phone:42345678', //
+ 'phone:/52345678',
+ 'phone://62345678',
+ 'telephone:72345678',
+ 'telephone:/82345678',
+ 'telephone://92345678',
])).to.eql([
'12345678',
- '12345678',
- '12345678',
- '12345678',
- '12345678',
- '12345678',
- '12345678',
- '12345678',
- ]);
-
- expect(social.emailsFromUrls([
- 'http://www.example.com',
- 'mailto:[email protected]',
- 'mailto:[email protected]',
- '',
- '\n\n\n',
- ])).to.eql([
+ '22345678',
+ '32345678',
+ '42345678',
+ '52345678',
+ '62345678',
+ '72345678',
+ '82345678',
+ '92345678',
]);
- expect(social.emailsFromUrls([
+ expect(social.phonesFromUrls([
+ 'https://www.example.com',
+ 'ftp://www.example.com',
+ '1234567',
+ '+42055555567',
+ 'tel://+42012345678',
+ 'tel://+420.123.456',
'http://www.example.com',
- 'mailto:[email protected]',
- 'mailto:[email protected]',
- '',
- '\n\n\n',
])).to.eql([
+ '+42012345678',
+ '+420.123.456',
]);
});
});
-