Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Development of new Apify.utils.social functions
Jan Curn209316db2336
diff --git a/src/utils_social.js b/src/utils_social.jsindex 7caa14117..a2b601cce 100644--- a/src/utils_social.js+++ b/src/utils_social.js@@ -1,5 +1,7 @@/* eslint-disable no-continue */import _ from 'underscore';+import htmlToText from 'html-to-text';+import cheerio from 'cheerio';// Regex inspired by https://zapier.com/blog/extract-links-email-phone-regex/// eslint-disable-next-line max-len@@ -47,30 +49,6 @@ const emailsFromUrls = (urls) => {};-/**- * The function extracts email addresses from a HTML text.- * It looks for `mailto:` links as well as email addresses in the text content.- * Note that the function preserves the order of emails and keep duplicates.- * @param {String} html HTML document- * @return {String[]} Array of emails addresses found.- * If no emails are found, the function returns an empty array.- * @memberOf utils.social- */-const emailsFromHtml = (html) => {- if (!Array.isArray(urls)) throw new Error('The "urls" parameter must be an array');-- const emails = [];- for (const url of urls) {- if (!url) continue;- if (!EMAIL_URL_PREFIX_REGEX.test(url)) continue;-- const email = url.replace(EMAIL_URL_PREFIX_REGEX, '').trim();- if (EMAIL_REGEX.test(email)) emails.push(email);- }- return emails;-};--const LINKEDIN_URL_REGEX = /http(s)?:\/\/[a-zA-Z]+\.linkedin\.com\/in\/[a-zA-Z0-9\-_%]+/g;const INSTAGRAM_URL_REGEX = /(?:(^|[^0-9a-z]))(((http|https):\/\/)?((www\.)?(?:instagram.com|instagr.am)\/([A-Za-z0-9_.]{2,30})))/ig;// eslint-disable-next-line max-len, no-useless-escape@@ -188,6 +166,53 @@ const phonesFromUrls = (urls) => {};+/**+ * The functions attempts to extract the following social handles from a HTML document:+ * emails, phones. Note that the function removes duplicates.+ * @param {String} html HTML document+ * @return {*} An object with social handles. It has the following strucute:+ * ```+ * {+ * emails: String[],+ * phones: String[],+ * }+ * ```+ */+const handlesFromHtml = (html) => {+ const result = {+ emails: [],+ phones: [],+ };++ if (!_.isString(html)) return result;++ // We use ignoreHref and ignoreImage options so that the text doesn't contain links,+ // since their parts can be interpreted as e.g. phone numbers.+ const text = htmlToText.fromString(html, { ignoreHref: true, ignoreImage: true });++ // TODO: Both html-to-text and cheerio use htmlparser2, the parsing could be done only once to improve performance+ const $ = cheerio.load(html, { decodeEntities: true });++ // Find all <a> links with href tag+ const linkUrls = [];+ $('a[href]').each((index, elem) => {+ if (elem) linkUrls.push($(elem).attr('href'));+ });++ // TODO: We should probably normalize all the handles to lower-case++ result.emails = emailsFromUrls(linkUrls).concat(emailsFromText(text));+ result.emails.sort();+ result.emails = _.uniq(result.emails, true);++ result.phones = phonesFromUrls(linkUrls).concat(phonesFromText(text));+ result.phones.sort();+ result.phones = _.uniq(result.phones, true);++ return result;+};++/*** A namespace that contains various Puppeteer utilities.*@@ -205,4 +230,5 @@ export const socialUtils = {emailsFromUrls,phonesFromText,phonesFromUrls,+ handlesFromHtml,};diff --git a/test/utils_social.js b/test/utils_social.jsindex 5f6e2b927..bf45c13f6 100644--- a/test/utils_social.js+++ b/test/utils_social.js@@ -301,3 +301,53 @@ describe('utils.social.phonesFromUrls()', () => {]);});});+++++describe('utils.social.handlesFromHtml()', () => {+ const EMPTY_RESULT = {+ emails: [],+ phones: [],+ };++ it('handles invalid arg', () => {+ expect(social.handlesFromHtml()).to.eql(EMPTY_RESULT);+ expect(social.handlesFromHtml(undefined)).to.eql(EMPTY_RESULT);+ expect(social.handlesFromHtml(null)).to.eql(EMPTY_RESULT);+ expect(social.handlesFromHtml({})).to.eql(EMPTY_RESULT);+ expect(social.handlesFromHtml(1234)).to.eql(EMPTY_RESULT);+ });++ it('works', () => {+ expect(social.handlesFromHtml('')).to.eql(EMPTY_RESULT);+ expect(social.handlesFromHtml(' ')).to.eql(EMPTY_RESULT);++ expect(social.handlesFromHtml(`+ <html>+ <head>+ <title>Bla</title>+ </head>+ <a>+ <p>[email protected]</p>+ <p>carl@example.com</p>+ <a href="mailto:[email protected]"></a>+ <a href="mailto:david@example.com"></a>++ <a href="[email protected]"></a>+ <img src="http://somewhere.com/ [email protected] " />+ <a href="http://somewhere.com/ [email protected] "></a>++ +420775222222+ +4207751111111+ <a href="skip.this.one: +42099999999"></a>+ <a href="tel://+42077533333"></a>++ </body>+ </html>+ `)).to.eql({+ phones: ['+4207751111111', '+420775222222', '+42077533333'],+ });+ });+});