Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Development of new Apify.utils.social functions
Jan Curn9561473c8641
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex f67d4187e..def1d3c1e 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -1,6 +1,7 @@xxx==================-- Added new `Apify.utils.social` namespace with new functions and constants+- Added new `Apify.utils.social` namespace with function to extract emails, phone and social profile URLs+ from HTML and text documents. Specifically, it supports Twitter, LinkedIn, Instagram and Facebook profiles.- Updated NPM dependencies0.9.7 / 2018-11-20diff --git a/src/utils_social.js b/src/utils_social.jsindex d4fefb018..d15ed71f6 100644--- a/src/utils_social.js+++ b/src/utils_social.js@@ -3,6 +3,8 @@ import _ from 'underscore';import htmlToText from 'html-to-text';import cheerio from 'cheerio';+// TODO: Finish docs and examples !!!+// Regex inspired by https://zapier.com/blog/extract-links-email-phone-regex/// eslint-disable-next-line max-lenconst EMAIL_REGEX_STRING = '(?:[a-z0-9!#$%&\'*+/=?^_`{|}~-]+(?:\\.[a-z0-9!#$%&\'*+/=?^_`{|}~-]+)*|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\\[(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\\])';@@ -176,24 +178,39 @@ const phonesFromUrls = (urls) => {return phones;};+// NOTEs about the regular expressions+// - They have just a single matching group for the profile username, all other groups are non-matching+// - They use a negative lookbehind and lookahead assertions, which are only supported in Node 8+.+// They are used to prevent matching URLs in strings like "blahttps://www.example.com"++// eslint-disable-next-line max-len+const LINKEDIN_PROFILE_REGEX_STRING = '(?<!\\w)(?:http(?:s)?:\\/\\/)?(?:(?:[a-z]+\\.)?linkedin\\.com\\/in\\/)([a-z0-9\\-_%]{2,60})(?![a-z0-9\\-_%])(?:/)?';++// eslint-disable-next-line max-len+const INSTAGRAM_PROFILE_REGEX_STRING = '(?<!\\w)(?:http(?:s)?:\\/\\/)?(?:(?:www\\.)?(?:instagram\\.com|instagr\\.am)\\/)([a-z0-9_.]{2,30})(?![a-z0-9_.])(?:/)?';++const TWITTER_RESERVED_PATHS = 'oauth|account|tos|privacy|signup|home|hashtag|search|login|widgets|i|settings|start|share|intent|oct';+// eslint-disable-next-line max-len, quotes+const TWITTER_PROFILE_REGEX_STRING = `(?<!\\w)(?:http(?:s)?:\\/\\/)?(?:www.)?(?:twitter.com)\\/(?!(?:${TWITTER_RESERVED_PATHS})(?:[\\'\\"\\?\\.\\/]|$))([a-z0-9_]{1,15})(?![a-z0-9_])(?:/)?`;-const LINKEDIN_URL_REGEX_STRING = '(http(s)?:\\/\\/)?([a-z]+\\.)?linkedin\\.com\\/in\\/[a-zA-Z0-9\\-_%]+';-const INSTAGRAM_URL_REGEX_STRING = '(http(s)?:\\/\\)?([a-z]+\\.)?(instagram\\.com|instagr\\.am)\\/[a-z0-9_.]{2,30}';-// const INSTAGRAM_URL_REGEX_STRING = '(?:(^|[^0-9a-z]))(((http|https):\\/\\/)?((www\\.)?(?:instagram.com|instagr.am)\\/([A-Za-z0-9_.]{2,30})))';// eslint-disable-next-line max-len, quotes-const TWITTER_URL_REGEX_STRING = `(?:(?:http|https):\\/\\/)?(?:www.)?(?:twitter.com)\\/(?!(oauth|account|tos|privacy|signup|home|hashtag|search|login|widgets|i|settings|start|share|intent|oct)([\\'\\"\\?\\.\\/]|$))([A-Za-z0-9_]{1,15})`;+const FACEBOOK_RESERVED_PATHS = 'rsrc\\.php|apps|groups|events|l\\.php|friends|images|photo.php|chat|ajax|dyi|common|policies|login|recover|reg|help|security|messages|marketplace|pages|live|bookmarks|games|fundraisers|saved|gaming|salesgroups|jobs|people|ads|ad_campaign|weather|offers|recommendations|crisisresponse|onthisday|developers|settings';+// eslint-disable-next-line max-len, quotes+const FACEBOOK_PROFILE_REGEX_STRING = `(?<!\\w)(?:http(?:s)?:\\/\\/)?(?:www.)?(?:facebook.com|fb.com)\\/(?!(?:${FACEBOOK_RESERVED_PATHS})(?:[\\'\\"\\?\\.\\/]|$))(profile\\.php\\?id\\=[0-9]{3,20}|(?!profile\\.php)[a-z0-9\\.]{5,51})(?![a-z0-9\\.])(?:/)?`;-// https://www.facebook.com/profile.php?id=1153222087-// https://www.facebook.com/julianwaldthaler-///**- * Regular expression to exactly match a single LinkedIn profile URL.- * It has the following form: `/^...$/i`.+ * Regular expression to exactly match a single LinkedIn profile URL, without any additional+ * subdirectories or query parameters. The regular expression has the following form: `/^...$/i`.+ *+ * Example usage:+ * ```+ * xx+ * ```* @type {RegExp}* @memberOf utils.social*/-const LINKEDIN_URL_REGEX = new RegExp(`^${LINKEDIN_URL_REGEX_STRING}$`, 'i');+const LINKEDIN_PROFILE_REGEX = new RegExp(`^${LINKEDIN_PROFILE_REGEX_STRING}$`, 'i');/*** Regular expression to find multiple LinkedIn profile URLs in a text.@@ -201,7 +218,7 @@ const LINKEDIN_URL_REGEX = new RegExp(`^${LINKEDIN_URL_REGEX_STRING}$`, 'i');* @type {RegExp}* @memberOf utils.social*/-const LINKEDIN_URL_REGEX_GLOBAL = new RegExp(LINKEDIN_URL_REGEX_STRING, 'ig');+const LINKEDIN_PROFILE_REGEX_GLOBAL = new RegExp(LINKEDIN_PROFILE_REGEX_STRING, 'ig');/**@@ -210,7 +227,7 @@ const LINKEDIN_URL_REGEX_GLOBAL = new RegExp(LINKEDIN_URL_REGEX_STRING, 'ig');* @type {RegExp}* @memberOf utils.social*/-const INSTAGRAM_URL_REGEX = new RegExp(`^${INSTAGRAM_URL_REGEX_STRING}$`, 'i');+const INSTAGRAM_PROFILE_REGEX = new RegExp(`^${INSTAGRAM_PROFILE_REGEX_STRING}$`, 'i');/*** Regular expression to find multiple Instagram profile URLs in a text.@@ -218,7 +235,7 @@ const INSTAGRAM_URL_REGEX = new RegExp(`^${INSTAGRAM_URL_REGEX_STRING}$`, 'i');* @type {RegExp}* @memberOf utils.social*/-const INSTAGRAM_URL_REGEX_GLOBAL = new RegExp(INSTAGRAM_URL_REGEX_STRING, 'ig');+const INSTAGRAM_PROFILE_REGEX_GLOBAL = new RegExp(INSTAGRAM_PROFILE_REGEX_STRING, 'ig');/**@@ -227,7 +244,23 @@ const INSTAGRAM_URL_REGEX_GLOBAL = new RegExp(INSTAGRAM_URL_REGEX_STRING, 'ig');* @type {RegExp}* @memberOf utils.social*/-const TWITTER_URL_REGEX = new RegExp(`^${TWITTER_URL_REGEX_STRING}$`, 'i');+const TWITTER_PROFILE_REGEX = new RegExp(`^${TWITTER_PROFILE_REGEX_STRING}$`, 'i');++/**+ * Regular expression to find multiple Instagram profile URLs in a text.+ * It has the following form: `/.../ig`.+ * @type {RegExp}+ * @memberOf utils.social+ */+const TWITTER_PROFILE_REGEX_GLOBAL = new RegExp(TWITTER_PROFILE_REGEX_STRING, 'ig');++/**+ * Regular expression to exactly match a single Facebook user profile URL.+ * It has the following form: `/^...$/i`.+ * @type {RegExp}+ * @memberOf utils.social+ */+const FACEBOOK_PROFILE_REGEX = new RegExp(`^${FACEBOOK_PROFILE_REGEX_STRING}$`, 'i');/*** Regular expression to find multiple Instagram profile URLs in a text.@@ -235,29 +268,37 @@ const TWITTER_URL_REGEX = new RegExp(`^${TWITTER_URL_REGEX_STRING}$`, 'i');* @type {RegExp}* @memberOf utils.social*/-const TWITTER_URL_REGEX_GLOBAL = new RegExp(TWITTER_URL_REGEX_STRING, 'ig');+const FACEBOOK_PROFILE_REGEX_GLOBAL = new RegExp(FACEBOOK_PROFILE_REGEX_STRING, 'ig');/**- * The functions attempts to extract the following social handles from a HTML document:- * emails, phones. Note that the function removes duplicates.- * @param {String} html HTML document- * @return {*} An object with social handles. It has the following strucute:+ * The functions attempts to extract emails, phones and social profile URLs from a HTML document,+ * specifically LinkedIn, Twitter, Instagram and Facebook profile URLs.+ * The function removes duplicates from the resulting arrays and sorts the items alphabetically.+ * @param {String} html HTML text+ * @param {Object} data Optional object which will receive the `text` and `$` properties+ * that contain text content of the HTML and `cheerio` object, respectively. This is an optimization+ * so that the caller doesn't need to parse the HTML document again, if needed.+ * @return {*} An object with social handles. It has the following structure:* ```* {* emails: String[],* phones: String[],* linkedInUrls: String[],+ * twitterProfiles: String[],+ * instagramProfiles: String[],+ * facebookProfiles: String[],* }* ```*/-const handlesFromHtml = (html) => {+const parseHandlesFromHtml = (html, data = null) => {const result = {emails: [],phones: [],- linkedIns: [],- twitters: [],- instagrams: [],+ linkedInProfiles: [],+ twitterProfiles: [],+ instagramProfiles: [],+ facebookProfiles: [],};if (!_.isString(html)) return result;@@ -280,20 +321,26 @@ const handlesFromHtml = (html) => {result.emails = emailsFromUrls(linkUrls).concat(emailsFromText(text));result.phones = phonesFromUrls(linkUrls).concat(phonesFromText(text));- // Note that these regexps extract just the base profile path. For example, for URL:+ // Note that these regexps extract just the base profile path. For example for// https://www.linkedin.com/in/carl-newman-123456a/detail/recent-activity/// they match just:// https://www.linkedin.com/in/carl-newman-123456a- result.linkedIns = html.match(LINKEDIN_URL_REGEX_GLOBAL) || [];- result.twitters = html.match(TWITTER_URL_REGEX_GLOBAL) || [];- result.instagrams = html.match(INSTAGRAM_URL_REGEX_GLOBAL) || [];- // result.facebooks = html.match(INSTAGRAM_URL_REGEX_GLOBAL) || [];+ result.linkedInProfiles = html.match(LINKEDIN_PROFILE_REGEX_GLOBAL) || [];+ result.twitterProfiles = html.match(TWITTER_PROFILE_REGEX_GLOBAL) || [];+ result.instagramProfiles = html.match(INSTAGRAM_PROFILE_REGEX_GLOBAL) || [];+ result.facebookProfiles = html.match(FACEBOOK_PROFILE_REGEX_GLOBAL) || [];// Sort and deduplicate handles- ['emails', 'phones', 'linkedIns', 'instagrams'].forEach((property) => {- result[property].sort();- result[property] = _.uniq(result[property], true);- });+ // eslint-disable-next-line guard-for-in, no-restricted-syntax+ for (const key in result) {+ result[key].sort();+ result[key] = _.uniq(result[key], true);+ }++ if (data) {+ data.text = text;+ data.$ = $;+ }return result;};@@ -301,7 +348,6 @@ const handlesFromHtml = (html) => {/*** A namespace that contains various utilities to help you extract social handles- * lie* from text, URLs and and HTML documents.** **Example usage:**@@ -318,17 +364,20 @@ export const socialUtils = {emailsFromUrls,phonesFromText,phonesFromUrls,- handlesFromHtml,+ parseHandlesFromHtml,EMAIL_REGEX,EMAIL_REGEX_GLOBAL,- LINKEDIN_URL_REGEX,- LINKEDIN_URL_REGEX_GLOBAL,+ LINKEDIN_PROFILE_REGEX,+ LINKEDIN_PROFILE_REGEX_GLOBAL,++ INSTAGRAM_PROFILE_REGEX,+ INSTAGRAM_PROFILE_REGEX_GLOBAL,- INSTAGRAM_URL_REGEX,- INSTAGRAM_URL_REGEX_GLOBAL,+ TWITTER_PROFILE_REGEX,+ TWITTER_PROFILE_REGEX_GLOBAL,- TWITTER_URL_REGEX,- TWITTER_URL_REGEX_GLOBAL,+ FACEBOOK_PROFILE_REGEX,+ FACEBOOK_PROFILE_REGEX_GLOBAL,};diff --git a/test/utils_social.js b/test/utils_social.jsindex 1353c6efb..59dfeb986 100644--- a/test/utils_social.js+++ b/test/utils_social.js@@ -303,28 +303,29 @@ describe('utils.social.phonesFromUrls()', () => {});});-describe('utils.social.handlesFromHtml()', () => {+describe('utils.social.parseHandlesFromHtml()', () => {const EMPTY_RESULT = {emails: [],phones: [],- linkedIns: [],- instagrams: [],- twitters: [],+ linkedInProfiles: [],+ twitterProfiles: [],+ instagramProfiles: [],+ facebookProfiles: [],};it('handles invalid arg', () => {- expect(social.handlesFromHtml()).to.eql(EMPTY_RESULT);- expect(social.handlesFromHtml(undefined)).to.eql(EMPTY_RESULT);- expect(social.handlesFromHtml(null)).to.eql(EMPTY_RESULT);- expect(social.handlesFromHtml({})).to.eql(EMPTY_RESULT);- expect(social.handlesFromHtml(1234)).to.eql(EMPTY_RESULT);+ expect(social.parseHandlesFromHtml()).to.eql(EMPTY_RESULT);+ expect(social.parseHandlesFromHtml(undefined)).to.eql(EMPTY_RESULT);+ expect(social.parseHandlesFromHtml(null)).to.eql(EMPTY_RESULT);+ expect(social.parseHandlesFromHtml({})).to.eql(EMPTY_RESULT);+ expect(social.parseHandlesFromHtml(1234)).to.eql(EMPTY_RESULT);});it('works', () => {- expect(social.handlesFromHtml('')).to.eql(EMPTY_RESULT);- expect(social.handlesFromHtml(' ')).to.eql(EMPTY_RESULT);+ expect(social.parseHandlesFromHtml('')).to.eql(EMPTY_RESULT);+ expect(social.parseHandlesFromHtml(' ')).to.eql(EMPTY_RESULT);- expect(social.handlesFromHtml(`+ expect(social.parseHandlesFromHtml(`<html><head><title>Bla</title>@@ -347,7 +348,7 @@ describe('utils.social.handlesFromHtml()', () => {<a href="skip.this.one: +42099999999"></a><a href="tel://+42077533333"></a>- https://www.linkedin.com/in/bobnewman/+ https://www.linkedin.com/in/bobnewmanhttps://www.linkedin.com/in/alicenewman/https://www.linkedin.com/in/alicenewman/ duplicatehttp://www.linkedin.com/in/nohttps/@@ -358,49 +359,381 @@ describe('utils.social.handlesFromHtml()', () => {<a href="https://www.linkedin.com/in/carl-newman-5555555a/detail/recent-activity/">Sub-link</a>https://www.instagram.com/old_prague/- https://www.instagram.com/old_prague+ https://www.instagram.com/old_prague/ duplicate+ instagram.com/old_prague+ https://www.instagram.com/newyorkarea/+ <a href="https://www.instagram.com/york">link</a>+ <a href="https://www.instagram.com/york2/something">sub-link</a>- https://www.instagram.com/newyorkarea/+ <a href="twitter.com/betasomething">link</a>+ https://www.twitter.com/apify+ https://www.twitter.com/apify duplicate+ <a href="twitter.com/cblabla/sub-dir/">link</a>- <a href="https://www.instagram.com/york">link</a>+ <a href="facebook.com/carl.username123/sub-dir/">link</a>+ https://www.facebook.com/bob.username123/+ https://www.facebook.com/bob.username123/ duplicate+ http://www.facebook.com/alice.username123+ <a href="https://www.facebook.com/profile.php?id=1155802082&xxx=5">link x</a>+ <a href="fb.com/dada5678?query=1">link</a></body></html>`)).to.eql({phones: ['+4207751111111', '+420775222222', '+42077533333'],- linkedIns: [- 'http://www.linkedin.com/in/nohttps',+ linkedInProfiles: [+ 'http://www.linkedin.com/in/nohttps/','https://cz.linkedin.com/in/somecountry',- 'https://www.linkedin.com/in/alicenewman',+ 'https://www.linkedin.com/in/alicenewman/','https://www.linkedin.com/in/bobnewman',- 'https://www.linkedin.com/in/carl-newman',- 'https://www.linkedin.com/in/carl-newman-5555555a',+ 'https://www.linkedin.com/in/carl-newman-5555555a/',+ 'https://www.linkedin.com/in/carl-newman/','https://www.linkedin.com/in/first-last-123456a','https://www.linkedin.com/in/jancurn',],- instagrams: [- 'https://www.instagram.com/newyorkarea',- 'https://www.instagram.com/old_prague',+ instagramProfiles: [+ 'https://www.instagram.com/newyorkarea/',+ 'https://www.instagram.com/old_prague/','https://www.instagram.com/york',+ 'https://www.instagram.com/york2/',+ 'instagram.com/old_prague',+ ],+ twitterProfiles: [+ 'https://www.twitter.com/apify',+ 'twitter.com/betasomething',+ 'twitter.com/cblabla/',+ ],+ facebookProfiles: [+ 'facebook.com/carl.username123/',+ 'fb.com/dada5678',+ 'http://www.facebook.com/alice.username123',+ 'https://www.facebook.com/bob.username123/',+ 'https://www.facebook.com/profile.php?id=1155802082',],- twitters: [],});});++ it('data is set correctly', () => {+ const data = {};+ social.parseHandlesFromHtml(`+ <html>+ <head>+ <title>Bla</title>+ </head>+ <body>+ Body content+ </body>+ </html>+ `, data);++ expect(data.$('body').text().trim()).to.eql('Body content');+ expect(data.text.trim()).to.eql('Body content');+ });});-describe('utils.social REGEXPES', () => {- it('exist', () => {+describe('utils.social', () => {+ it('EMAIL_REGEX', () => {expect(_.isRegExp(social.EMAIL_REGEX)).to.eql(true);expect(_.isRegExp(social.EMAIL_REGEX_GLOBAL)).to.eql(true);- expect(_.isRegExp(social.LINKEDIN_URL_REGEX)).to.eql(true);- expect(_.isRegExp(social.LINKEDIN_URL_REGEX_GLOBAL)).to.eql(true);+ expect(social.EMAIL_REGEX.flags).to.eql('i');+ expect(social.EMAIL_REGEX_GLOBAL.flags).to.eql('gi');- expect(_.isRegExp(social.INSTAGRAM_URL_REGEX)).to.eql(true);- expect(_.isRegExp(social.INSTAGRAM_URL_REGEX_GLOBAL)).to.eql(true);+ expect(social.EMAIL_REGEX.test('[email protected]')).to.eql(true);+ expect(social.EMAIL_REGEX.test('[email protected]')).to.eql(true);++ expect(social.EMAIL_REGEX.test('[email protected]')).to.eql(true);+ expect(social.EMAIL_REGEX.test('[email protected]')).to.eql(true);+ expect(social.EMAIL_REGEX.test('[email protected]')).to.eql(true);+ expect(social.EMAIL_REGEX.test('[email protected]')).to.eql(true);++ expect(social.EMAIL_REGEX.test('a [email protected]')).to.eql(false);+ expect(social.EMAIL_REGEX.test('[email protected] [email protected]')).to.eql(false);+ expect(social.EMAIL_REGEX.test('')).to.eql(false);+ expect(social.EMAIL_REGEX.test('dummy')).to.eql(false);++ expect(social.EMAIL_REGEX_GLOBAL.test('[email protected]')).to.eql(true);+ expect('[email protected] [email protected]'.match(social.EMAIL_REGEX_GLOBAL))+ .to.eql(['[email protected]', '[email protected]']);++ expect(''.match(social.EMAIL_REGEX_GLOBAL)).to.eql(null);+ expect(' dummy '.match(social.EMAIL_REGEX_GLOBAL)).to.eql(null);+ });++ it('LINKEDIN_PROFILE_REGEX', () => {+ expect(_.isRegExp(social.LINKEDIN_PROFILE_REGEX)).to.eql(true);+ expect(_.isRegExp(social.LINKEDIN_PROFILE_REGEX_GLOBAL)).to.eql(true);++ expect(social.LINKEDIN_PROFILE_REGEX.flags).to.eql('i');+ expect(social.LINKEDIN_PROFILE_REGEX_GLOBAL.flags).to.eql('gi');++ expect(social.LINKEDIN_PROFILE_REGEX.test('https://www.linkedin.com/in/bobnewman')).to.eql(true);+ expect(social.LINKEDIN_PROFILE_REGEX.test('https://www.linkedin.com/in/bobnewman/')).to.eql(true);+ expect(social.LINKEDIN_PROFILE_REGEX.test('http://www.linkedin.com/in/bobnewman')).to.eql(true);+ expect(social.LINKEDIN_PROFILE_REGEX.test('http://ie.linkedin.com/in/bobnewman')).to.eql(true);+ expect(social.LINKEDIN_PROFILE_REGEX.test('https://linkedin.com/in/bobnewman')).to.eql(true);+ expect(social.LINKEDIN_PROFILE_REGEX.test('https://www.linkedin.com/in/carl-newman')).to.eql(true);+ expect(social.LINKEDIN_PROFILE_REGEX.test('https://www.linkedin.com/in/first-last-123456a')).to.eql(true);+ expect(social.LINKEDIN_PROFILE_REGEX.test('https://www.linkedin.com/in/first_last_1%23456a')).to.eql(true);+ expect(social.LINKEDIN_PROFILE_REGEX.test('HTTPS://WWW.LINKEDIN.COM/IN/CARL-NEWMAN')).to.eql(true);+ expect(social.LINKEDIN_PROFILE_REGEX.test('www.linkedin.com/in/bobnewman')).to.eql(true);+ expect(social.LINKEDIN_PROFILE_REGEX.test('linkedin.com/in/bobnewman')).to.eql(true);++ // Test there is just on matching group for the username+ expect('https://www.linkedin.com/in/bobnewman/'.match(social.LINKEDIN_PROFILE_REGEX)[1]).to.eql('bobnewman');+ expect('http://www.linkedin.com/in/bobnewman'.match(social.LINKEDIN_PROFILE_REGEX)[1]).to.eql('bobnewman');+ expect('www.linkedin.com/in/bobnewman/'.match(social.LINKEDIN_PROFILE_REGEX)[1]).to.eql('bobnewman');+ expect('linkedin.com/in/bobnewman'.match(social.LINKEDIN_PROFILE_REGEX)[1]).to.eql('bobnewman');++ expect(social.LINKEDIN_PROFILE_REGEX.test('')).to.eql(false);+ expect(social.LINKEDIN_PROFILE_REGEX.test('dummy')).to.eql(false);+ expect(social.LINKEDIN_PROFILE_REGEX.test('a https://www.linkedin.com/in/bobnewman')).to.eql(false);+ expect(social.LINKEDIN_PROFILE_REGEX.test('https://linkedin.com/in/bobnewman/sub-page')).to.eql(false);+ expect(social.LINKEDIN_PROFILE_REGEX.test('xhttps://www.linkedin.com/in/carl-newman')).to.eql(false);+ expect(social.LINKEDIN_PROFILE_REGEX.test('0https://www.linkedin.com/in/carl-newman')).to.eql(false);+ expect(social.LINKEDIN_PROFILE_REGEX.test('_https://www.linkedin.com/in/carl-newman')).to.eql(false);+ expect(social.LINKEDIN_PROFILE_REGEX.test('xlinkedin.com/in/bobnewman')).to.eql(false);+ expect(social.LINKEDIN_PROFILE_REGEX.test('_linkedin.com/in/bobnewman')).to.eql(false);+ expect(social.LINKEDIN_PROFILE_REGEX.test('0linkedin.com/in/bobnewman')).to.eql(false);+ expect(social.LINKEDIN_PROFILE_REGEX.test('https://www.linkedin.com/in/bobnewman/?param=bla')).to.eql(false);+ expect(social.LINKEDIN_PROFILE_REGEX.test('://linkedin.com/in/bobnewman')).to.eql(false);+ expect(social.LINKEDIN_PROFILE_REGEX.test('https://www.linkedin.com/in/bob https://www.linkedin.com/in/alice')).to.eql(false);++ expect(social.LINKEDIN_PROFILE_REGEX_GLOBAL.test('https://www.linkedin.com/in/bobnewman')).to.eql(true);+ expect(`+ https://www.linkedin.com/in/bobnewman+ "http://ie.linkedin.com/in/alicenewman"+ https://www.linkedin.com/in/someverylongnamesomeverylongnamesomeverylongnamesomeverylongnamesomeverylongnamesomeverylongname+ linkedin.com/in/carlnewman+ `.match(social.LINKEDIN_PROFILE_REGEX_GLOBAL))+ .to.eql([+ 'https://www.linkedin.com/in/bobnewman',+ 'http://ie.linkedin.com/in/alicenewman',+ 'linkedin.com/in/carlnewman',+ ]);+ expect(`+ -https://www.linkedin.com/in/bobnewman/sub-dir+ :http://ie.linkedin.com/in/alicenewman?param=1+ xlinkedin.com/in/carlnewman+ alinkedin.com/in/carlnewman+ _linkedin.com/in/carlnewman+ `.match(social.LINKEDIN_PROFILE_REGEX_GLOBAL))+ .to.eql([+ 'https://www.linkedin.com/in/bobnewman/',+ 'http://ie.linkedin.com/in/alicenewman',+ ]);+ expect(''.match(social.LINKEDIN_PROFILE_REGEX_GLOBAL)).to.eql(null);+ });++ it('INSTAGRAM_PROFILE_REGEX', () => {+ expect(_.isRegExp(social.INSTAGRAM_PROFILE_REGEX)).to.eql(true);+ expect(_.isRegExp(social.INSTAGRAM_PROFILE_REGEX_GLOBAL)).to.eql(true);++ expect(social.INSTAGRAM_PROFILE_REGEX.flags).to.eql('i');+ expect(social.INSTAGRAM_PROFILE_REGEX_GLOBAL.flags).to.eql('gi');++ expect(social.INSTAGRAM_PROFILE_REGEX.test('https://www.instagram.com/old_prague')).to.eql(true);+ expect(social.INSTAGRAM_PROFILE_REGEX.test('https://www.instagram.com/old_prague/')).to.eql(true);+ expect(social.INSTAGRAM_PROFILE_REGEX.test('http://www.instagram.com/old_prague/')).to.eql(true);+ expect(social.INSTAGRAM_PROFILE_REGEX.test('https://instagram.com/old_prague/')).to.eql(true);+ expect(social.INSTAGRAM_PROFILE_REGEX.test('HTTPS://INSTAGR.AM/OLD_PRAGUE/')).to.eql(true);+ expect(social.INSTAGRAM_PROFILE_REGEX.test('http://instagr.am/old_prague/')).to.eql(true);+ expect(social.INSTAGRAM_PROFILE_REGEX.test('https://www.instagr.am/old_prague/')).to.eql(true);+ expect(social.INSTAGRAM_PROFILE_REGEX.test('www.instagram.com/old_prague/')).to.eql(true);+ expect(social.INSTAGRAM_PROFILE_REGEX.test('instagram.com/old_prague/')).to.eql(true);+ expect(social.INSTAGRAM_PROFILE_REGEX.test('www.instagr.am/old_prague/')).to.eql(true);+ expect(social.INSTAGRAM_PROFILE_REGEX.test('instagr.am/old_prague/')).to.eql(true);++ // Test there is just on matching group for the username+ expect('https://www.instagram.com/old_prague/'.match(social.INSTAGRAM_PROFILE_REGEX)[1]).to.eql('old_prague');+ expect('http://www.instagram.com/old_prague/'.match(social.INSTAGRAM_PROFILE_REGEX)[1]).to.eql('old_prague');+ expect('www.instagram.com/old_prague'.match(social.INSTAGRAM_PROFILE_REGEX)[1]).to.eql('old_prague');+ expect('instagram.com/old_prague'.match(social.INSTAGRAM_PROFILE_REGEX)[1]).to.eql('old_prague');++ expect(social.INSTAGRAM_PROFILE_REGEX.test('')).to.eql(false);+ expect(social.INSTAGRAM_PROFILE_REGEX.test('dummy')).to.eql(false);+ expect(social.INSTAGRAM_PROFILE_REGEX.test('a https://www.instagram.com/old_prague')).to.eql(false);+ expect(social.INSTAGRAM_PROFILE_REGEX.test('https://www.instagram.com/a')).to.eql(false);+ expect(social.INSTAGRAM_PROFILE_REGEX.test('https://www.instagram.com/old_prague/sub-page')).to.eql(false);+ expect(social.INSTAGRAM_PROFILE_REGEX.test('xhttps://www.instagram.com/old_prague')).to.eql(false);+ expect(social.INSTAGRAM_PROFILE_REGEX.test('0https://www.instagram.com/old_prague')).to.eql(false);+ expect(social.INSTAGRAM_PROFILE_REGEX.test('_https://www.instagram.com/old_prague')).to.eql(false);+ expect(social.INSTAGRAM_PROFILE_REGEX.test('xinstagram.com/old_prague')).to.eql(false);+ expect(social.INSTAGRAM_PROFILE_REGEX.test('_instagram.com/old_prague')).to.eql(false);+ expect(social.INSTAGRAM_PROFILE_REGEX.test('0instagram.com/old_prague')).to.eql(false);+ expect(social.INSTAGRAM_PROFILE_REGEX.test('https://www.instagram.com/old_prague/?param=bla')).to.eql(false);+ expect(social.INSTAGRAM_PROFILE_REGEX.test('://www.instagram.com/old_prague')).to.eql(false);+ expect(social.INSTAGRAM_PROFILE_REGEX.test('http://www.instagram.com/old_prague http://www.instagram.com/old_brno')).to.eql(false);++ expect(social.INSTAGRAM_PROFILE_REGEX_GLOBAL.test('https://www.instagram.com/old_prague')).to.eql(true);+ expect(`+ https://www.instagram.com/old_prague+ https://www.instagram.com/someverylongusernamethatisnotgood+ "instagram.com/old_brno"+ http://instagr.am/old_plzen+ `.match(social.INSTAGRAM_PROFILE_REGEX_GLOBAL))+ .to.eql([+ 'https://www.instagram.com/old_prague',+ 'instagram.com/old_brno',+ 'http://instagr.am/old_plzen',+ ]);+ expect(`+ -https://www.instagram.com/old_prague/sub-dir+ instagr.am/old_plzen?param=1+ xinstagram.com/old_brno+ ainstagram.com/old_brno+ _instagram.com/old_brno+ `.match(social.INSTAGRAM_PROFILE_REGEX_GLOBAL))+ .to.eql([+ 'https://www.instagram.com/old_prague/',+ 'instagr.am/old_plzen',+ ]);+ expect(''.match(social.INSTAGRAM_PROFILE_REGEX_GLOBAL)).to.eql(null);+ });++ it('TWITTER_PROFILE_REGEX', () => {+ expect(_.isRegExp(social.TWITTER_PROFILE_REGEX)).to.eql(true);+ expect(_.isRegExp(social.TWITTER_PROFILE_REGEX_GLOBAL)).to.eql(true);++ expect(social.TWITTER_PROFILE_REGEX.flags).to.eql('i');+ expect(social.TWITTER_PROFILE_REGEX_GLOBAL.flags).to.eql('gi');++ expect(social.TWITTER_PROFILE_REGEX.test('https://www.twitter.com/apify')).to.eql(true);+ expect(social.TWITTER_PROFILE_REGEX.test('https://www.twitter.com/apify/')).to.eql(true);+ expect(social.TWITTER_PROFILE_REGEX.test('https://www.twitter.com/aa_bb_123')).to.eql(true);+ expect(social.TWITTER_PROFILE_REGEX.test('http://www.twitter.com/apify')).to.eql(true);+ expect(social.TWITTER_PROFILE_REGEX.test('https://twitter.com/apify')).to.eql(true);+ expect(social.TWITTER_PROFILE_REGEX.test('http://twitter.com/apify')).to.eql(true);+ expect(social.TWITTER_PROFILE_REGEX.test('www.twitter.com/apify')).to.eql(true);+ expect(social.TWITTER_PROFILE_REGEX.test('twitter.com/apify')).to.eql(true);++ // Test there is just on matching group for the username+ expect('https://www.twitter.com/apify/'.match(social.TWITTER_PROFILE_REGEX)[1]).to.eql('apify');+ expect('http://www.twitter.com/apify'.match(social.TWITTER_PROFILE_REGEX)[1]).to.eql('apify');+ expect('www.twitter.com/apify'.match(social.TWITTER_PROFILE_REGEX)[1]).to.eql('apify');+ expect('twitter.com/apify'.match(social.TWITTER_PROFILE_REGEX)[1]).to.eql('apify');++ expect(social.TWITTER_PROFILE_REGEX.test('')).to.eql(false);+ expect(social.TWITTER_PROFILE_REGEX.test('dummy')).to.eql(false);+ expect(social.TWITTER_PROFILE_REGEX.test('a https://www.twitter.com/apify')).to.eql(false);+ expect(social.TWITTER_PROFILE_REGEX.test('https://www.twitter.com/apify/sub-page')).to.eql(false);+ expect(social.TWITTER_PROFILE_REGEX.test('xhttps://www.twitter.com/apify')).to.eql(false);+ expect(social.TWITTER_PROFILE_REGEX.test('0https://www.twitter.com/apify')).to.eql(false);+ expect(social.TWITTER_PROFILE_REGEX.test('_https://www.twitter.com/apify')).to.eql(false);+ expect(social.TWITTER_PROFILE_REGEX.test('xtwitter.com/apify')).to.eql(false);+ expect(social.TWITTER_PROFILE_REGEX.test('_twitter.com/apify')).to.eql(false);+ expect(social.TWITTER_PROFILE_REGEX.test('0twitter.com/apify')).to.eql(false);+ expect(social.TWITTER_PROFILE_REGEX.test('https://www.twitter.com/apify?param=bla')).to.eql(false);+ expect(social.TWITTER_PROFILE_REGEX.test('://www.twitter.com/apify')).to.eql(false);+ expect(social.TWITTER_PROFILE_REGEX.test('https://www.twitter.com/apify https://www.twitter.com/jack')).to.eql(false);+ expect(social.TWITTER_PROFILE_REGEX.test('https://www.twitter.com/oauth')).to.eql(false);+ expect(social.TWITTER_PROFILE_REGEX.test('https://www.twitter.com/account')).to.eql(false);+ expect(social.TWITTER_PROFILE_REGEX.test('https://www.twitter.com/privacy/')).to.eql(false);++ expect(social.TWITTER_PROFILE_REGEX_GLOBAL.test('https://www.twitter.com/apify')).to.eql(true);+ expect(`+ https://www.twitter.com/apify+ www.twitter.com/jack/sub-dir+ www.twitter.com/invalidverylongtwitterhandlenotgood+ twitter.com/bob123?param=1+ `.match(social.TWITTER_PROFILE_REGEX_GLOBAL))+ .to.eql([+ 'https://www.twitter.com/apify',+ 'www.twitter.com/jack/',+ 'twitter.com/bob123',+ ]);+ expect(`+ -https://www.twitter.com/apify+ twitter.com/jack+ twitter.com/carl123+ xtwitter.com/bob+ atwitter.com/bob+ _twitter.com/bob+ `.match(social.TWITTER_PROFILE_REGEX_GLOBAL))+ .to.eql([+ 'https://www.twitter.com/apify',+ 'twitter.com/jack',+ 'twitter.com/carl123',+ ]);+ expect(''.match(social.TWITTER_PROFILE_REGEX_GLOBAL)).to.eql(null);+ });- expect(_.isRegExp(social.TWITTER_URL_REGEX)).to.eql(true);- expect(_.isRegExp(social.TWITTER_URL_REGEX_GLOBAL)).to.eql(true);+ it('FACEBOOK_PROFILE_REGEX', () => {+ expect(_.isRegExp(social.FACEBOOK_PROFILE_REGEX)).to.eql(true);+ expect(_.isRegExp(social.FACEBOOK_PROFILE_REGEX_GLOBAL)).to.eql(true);++ expect(social.FACEBOOK_PROFILE_REGEX.flags).to.eql('i');+ expect(social.FACEBOOK_PROFILE_REGEX_GLOBAL.flags).to.eql('gi');++ expect(social.FACEBOOK_PROFILE_REGEX.test('https://www.facebook.com/someusername')).to.eql(true);+ expect(social.FACEBOOK_PROFILE_REGEX.test('https://www.facebook.com/someusername/')).to.eql(true);+ expect(social.FACEBOOK_PROFILE_REGEX.test('http://www.facebook.com/some.username123')).to.eql(true);+ expect(social.FACEBOOK_PROFILE_REGEX.test('www.facebook.com/someusername')).to.eql(true);+ expect(social.FACEBOOK_PROFILE_REGEX.test('facebook.com/someusername')).to.eql(true);+ expect(social.FACEBOOK_PROFILE_REGEX.test('https://www.fb.com/someusername')).to.eql(true);+ expect(social.FACEBOOK_PROFILE_REGEX.test('https://www.fb.com/someusername/')).to.eql(true);+ expect(social.FACEBOOK_PROFILE_REGEX.test('http://www.fb.com/some.username123')).to.eql(true);+ expect(social.FACEBOOK_PROFILE_REGEX.test('www.fb.com/someusername')).to.eql(true);+ expect(social.FACEBOOK_PROFILE_REGEX.test('fb.com/someusername')).to.eql(true);++ expect(social.FACEBOOK_PROFILE_REGEX.test('https://www.facebook.com/profile.php?id=1155802082')).to.eql(true);+ expect(social.FACEBOOK_PROFILE_REGEX.test('http://www.facebook.com/profile.php?id=1155802082')).to.eql(true);+ expect(social.FACEBOOK_PROFILE_REGEX.test('www.facebook.com/profile.php?id=1155802082')).to.eql(true);+ expect(social.FACEBOOK_PROFILE_REGEX.test('facebook.com/profile.php?id=1155802082')).to.eql(true);+ expect(social.FACEBOOK_PROFILE_REGEX.test('fb.com/profile.php?id=1155802082')).to.eql(true);++ // Test there is just on matching group for the username+ expect('https://www.facebook.com/someusername/'.match(social.FACEBOOK_PROFILE_REGEX)[1]).to.eql('someusername');+ expect('https://www.facebook.com/someusername'.match(social.FACEBOOK_PROFILE_REGEX)[1]).to.eql('someusername');+ expect('https://www.facebook.com/profile.php?id=1155802082'.match(social.FACEBOOK_PROFILE_REGEX)[1]).to.eql('profile.php?id=1155802082');+ expect('fb.com/someusername'.match(social.FACEBOOK_PROFILE_REGEX)[1]).to.eql('someusername');++ expect(social.FACEBOOK_PROFILE_REGEX.test('')).to.eql(false);+ expect(social.FACEBOOK_PROFILE_REGEX.test('dummy')).to.eql(false);+ expect(social.FACEBOOK_PROFILE_REGEX.test('a https://www.facebook.com/someusername')).to.eql(false);+ expect(social.FACEBOOK_PROFILE_REGEX.test('https://www.facebook.com/a')).to.eql(false);+ expect(social.FACEBOOK_PROFILE_REGEX.test('https://www.facebook.com/someusername/sub-page')).to.eql(false);+ expect(social.FACEBOOK_PROFILE_REGEX.test('http://www.facebook.com/profile.php')).to.eql(false);+ expect(social.FACEBOOK_PROFILE_REGEX.test('xhttps://www.facebook.com/someusername')).to.eql(false);+ expect(social.FACEBOOK_PROFILE_REGEX.test('0https://www.facebook.com/someusername')).to.eql(false);+ expect(social.FACEBOOK_PROFILE_REGEX.test('_https://www.facebook.com/someusername')).to.eql(false);+ expect(social.FACEBOOK_PROFILE_REGEX.test('xfacebook.com/someusername')).to.eql(false);+ expect(social.FACEBOOK_PROFILE_REGEX.test('_facebook.com/someusername')).to.eql(false);+ expect(social.FACEBOOK_PROFILE_REGEX.test('0facebook.com/someusername')).to.eql(false);+ expect(social.FACEBOOK_PROFILE_REGEX.test('https://www.facebook.com/someusername?param=bla')).to.eql(false);++ expect(social.FACEBOOK_PROFILE_REGEX.test('://www.facebook.com/someusername')).to.eql(false);+ expect(social.FACEBOOK_PROFILE_REGEX.test('https://www.facebook.com/someusername https://www.facebook.com/jack')).to.eql(false);+ expect(social.FACEBOOK_PROFILE_REGEX.test('https://www.facebook.com/groups')).to.eql(false);+ expect(social.FACEBOOK_PROFILE_REGEX.test('https://www.facebook.com/events')).to.eql(false);+ expect(social.FACEBOOK_PROFILE_REGEX.test('https://www.facebook.com/policies/')).to.eql(false);++ expect(social.FACEBOOK_PROFILE_REGEX_GLOBAL.test('https://www.facebook.com/someusername')).to.eql(true);+ expect(`+ https://www.facebook.com/someusername?param=123+ www.facebook.com/another123/sub-dir+ https://www.facebook.com/waytoolongusernamewaytoolongusernamewaytoolongusernamewaytoolongusernamewaytoolongusername+ fb.com/bob123+ `.match(social.FACEBOOK_PROFILE_REGEX_GLOBAL))+ .to.eql([+ 'https://www.facebook.com/someusername',+ 'www.facebook.com/another123/',+ 'fb.com/bob123',+ ]);+ expect(`+ -https://www.facebook.com/someusername/+ facebook.com/jack4567+ fb.com/carl123+ xfacebook.com/bob+ afacebook.com/bob+ _facebook.com/bob+ `.match(social.FACEBOOK_PROFILE_REGEX_GLOBAL))+ .to.eql([+ 'https://www.facebook.com/someusername/',+ 'facebook.com/jack4567',+ 'fb.com/carl123',+ ]);+ expect(''.match(social.FACEBOOK_PROFILE_REGEX_GLOBAL)).to.eql(null);});});