Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Added regexp for Youtube video URLs to utils.social
drobnikj45a420ddb486
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 08830b7d2..69aa260a9 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -1,3 +1,7 @@+NEXT+====================+- Added regexps for Youtube videos (`YOUTUBE_REGEX`, `YOUTUBE_REGEX_GLOBAL`) to `utils.social`+0.16.1 / 2019-10-31====================- Bump Puppeteer to 2.0.0 and use `{ pipe: true }` again because upstream bug has been fixed.diff --git a/docs/api/social.md b/docs/api/social.mdindex 5f576b038..d75953642 100644--- a/docs/api/social.md+++ b/docs/api/social.md@@ -24,6 +24,8 @@ const emails = Apify.utils.social.emailsFromText('[email protected] bob@example.- [`.TWITTER_REGEX_GLOBAL`](#social.TWITTER_REGEX_GLOBAL) : `RegExp`- [`.FACEBOOK_REGEX`](#social.FACEBOOK_REGEX) : `RegExp`- [`.FACEBOOK_REGEX_GLOBAL`](#social.FACEBOOK_REGEX_GLOBAL) : `RegExp`+ - [`.YOUTUBE_REGEX`](#social.YOUTUBE_REGEX) : `RegExp`+ - [`.YOUTUBE_REGEX_GLOBAL`](#social.YOUTUBE_REGEX_GLOBAL) : `RegExp`- [`.EMAIL_REGEX`](#social.EMAIL_REGEX) : `RegExp`- [`.EMAIL_REGEX_GLOBAL`](#social.EMAIL_REGEX_GLOBAL) : `RegExp`- [`.emailsFromText(text)`](#social.emailsFromText) ⇒ `Array<String>`@@ -111,7 +113,7 @@ https://www.instagram.com/cristiano/followersExample usage:```-if (Apify.utils.social.INSTAGRAM_REGEX_STRING.test('https://www.instagram.com/old_prague')) {+if (Apify.utils.social.INSTAGRAM_REGEX.test('https://www.instagram.com/old_prague')) {console.log('Match!');}```@@ -168,7 +170,7 @@ https://www.twitter.com/realdonaldtrump/followingExample usage:```-if (Apify.utils.social.TWITTER_REGEX_STRING.test('https://www.twitter.com/apify')) {+if (Apify.utils.social.TWITTER_REGEX.test('https://www.twitter.com/apify')) {console.log('Match!');}```@@ -226,7 +228,7 @@ https://www.facebook.com/apifytech/photosExample usage:```-if (Apify.utils.social.FACEBOOK_REGEX_STRING.test('https://www.facebook.com/apifytech')) {+if (Apify.utils.social.FACEBOOK_REGEX.test('https://www.facebook.com/apifytech')) {console.log('Match!');}```@@ -263,6 +265,43 @@ const matches = text.match(Apify.utils.social.FACEBOOK_REGEX_GLOBAL);if (matches) console.log(`${matches.length} Facebook profiles found!`);```+<a name="social.YOUTUBE_REGEX"></a>++## `social.YOUTUBE_REGEX` : `RegExp`++Regular expression to exactly match a single Youtube video URL. It has the following form: `/^...$/i` and matches URLs such as:++```+https://www.youtube.com/watch?v=kM7YfhfkiEE+https://youtu.be/kM7YfhfkiEE+```++Example usage:++```+if (Apify.utils.social.YOUTUBE_REGEX.test('https://www.youtube.com/watch?v=kM7YfhfkiEE')) {+ console.log('Match!');+}+```++<a name="social.YOUTUBE_REGEX_GLOBAL"></a>++## `social.YOUTUBE_REGEX_GLOBAL` : `RegExp`++Regular expression to find multiple Youtube video URLs in a text or HTML. It has the following form: `/.../ig` and matches URLs such as:++```+https://www.youtube.com/watch?v=kM7YfhfkiEE+https://youtu.be/kM7YfhfkiEE+```++Example usage:++```+const matches = text.match(Apify.utils.social.YOUTUBE_REGEX_GLOBAL);+if (matches) console.log(`${matches.length} Youtube videos found!`);+```+<a name="social.EMAIL_REGEX"></a>## `social.EMAIL_REGEX` : `RegExp`@@ -384,6 +423,7 @@ The result of the function is an object with the following structure:twitters: String[],instagrams: String[],facebooks: String[],+ youtubes: String[],}```diff --git a/src/utils_social.js b/src/utils_social.jsindex 417c2b720..c57e9643a 100644--- a/src/utils_social.js+++ b/src/utils_social.js@@ -200,7 +200,8 @@ const TWITTER_REGEX_STRING = `(?<!\\w)(?:http(?:s)?:\\/\\/)?(?:www.)?(?:twitter.const FACEBOOK_RESERVED_PATHS = 'rsrc\\.php|apps|groups|events|l\\.php|friends|images|photo.php|chat|ajax|dyi|common|policies|login|recover|reg|help|security|messages|marketplace|pages|live|bookmarks|games|fundraisers|saved|gaming|salesgroups|jobs|people|ads|ad_campaign|weather|offers|recommendations|crisisresponse|onthisday|developers|settings|connect|business|plugins|intern|sharer';// eslint-disable-next-line max-len, quotesconst FACEBOOK_REGEX_STRING = `(?<!\\w)(?:http(?:s)?:\\/\\/)?(?:www.)?(?:facebook.com|fb.com)\\/(?!(?:${FACEBOOK_RESERVED_PATHS})(?:[\\'\\"\\?\\.\\/]|$))(profile\\.php\\?id\\=[0-9]{3,20}|(?!profile\\.php)[a-z0-9\\.]{5,51})(?![a-z0-9\\.])(?:/)?`;-+// eslint-disable-next-line max-len, quotes+const YOUTUBE_REGEX_STRING = '(?:https?:\\/\\/)?(?:youtu\\.be\\/|(?:www\\.|m\\.)?youtube\\.com\\/(?:watch|v|embed)(?:\\.php)?(?:\\?.*v=|\\/))([a-zA-Z0-9\\-_]+)';let LINKEDIN_REGEX;let LINKEDIN_REGEX_GLOBAL;@@ -210,6 +211,8 @@ let TWITTER_REGEX;let TWITTER_REGEX_GLOBAL;let FACEBOOK_REGEX;let FACEBOOK_REGEX_GLOBAL;+let YOUTUBE_REGEX;+let YOUTUBE_REGEX_GLOBAL;try {/**@@ -284,7 +287,7 @@ try {** Example usage:* ```- * if (Apify.utils.social.INSTAGRAM_REGEX_STRING.test('https://www.instagram.com/old_prague')) {+ * if (Apify.utils.social.INSTAGRAM_REGEX.test('https://www.instagram.com/old_prague')) {* console.log('Match!');* }* ```@@ -338,7 +341,7 @@ try {** Example usage:* ```- * if (Apify.utils.social.TWITTER_REGEX_STRING.test('https://www.twitter.com/apify')) {+ * if (Apify.utils.social.TWITTER_REGEX.test('https://www.twitter.com/apify')) {* console.log('Match!');* }* ```@@ -393,7 +396,7 @@ try {** Example usage:* ```- * if (Apify.utils.social.FACEBOOK_REGEX_STRING.test('https://www.facebook.com/apifytech')) {+ * if (Apify.utils.social.FACEBOOK_REGEX.test('https://www.facebook.com/apifytech')) {* console.log('Match!');* }* ```@@ -430,6 +433,43 @@ try {* @memberOf social*/FACEBOOK_REGEX_GLOBAL = new RegExp(FACEBOOK_REGEX_STRING, 'ig');++ /**+ * Regular expression to exactly match a single Youtube video URL.+ * It has the following form: `/^...$/i` and matches URLs such as:+ * ```+ * https://www.youtube.com/watch?v=kM7YfhfkiEE+ * https://youtu.be/kM7YfhfkiEE+ * ```+ *+ * Example usage:+ * ```+ * if (Apify.utils.social.YOUTUBE_REGEX.test('https://www.youtube.com/watch?v=kM7YfhfkiEE')) {+ * console.log('Match!');+ * }+ * ```+ * @type {RegExp}+ * @memberOf social+ */+ YOUTUBE_REGEX = new RegExp(`^${YOUTUBE_REGEX_STRING}$`, 'i');++ /**+ * Regular expression to find multiple Youtube video URLs in a text or HTML.+ * It has the following form: `/.../ig` and matches URLs such as:+ * ```+ * https://www.youtube.com/watch?v=kM7YfhfkiEE+ * https://youtu.be/kM7YfhfkiEE+ * ```+ *+ * Example usage:+ * ```+ * const matches = text.match(Apify.utils.social.YOUTUBE_REGEX_GLOBAL);+ * if (matches) console.log(`${matches.length} Youtube videos found!`);+ * ```+ * @type {RegExp}+ * @memberOf social+ */+ YOUTUBE_REGEX_GLOBAL = new RegExp(YOUTUBE_REGEX_STRING, 'ig');} catch (e) {// Older versions of Node don't support negative lookbehind and lookahead expressions.// Show warning instead of failing.@@ -457,6 +497,7 @@ try {* twitters: String[],* instagrams: String[],* facebooks: String[],+ * youtubes: String[],* }* ```*@@ -496,6 +537,7 @@ const parseHandlesFromHtml = (html, data = null) => {twitters: [],instagrams: [],facebooks: [],+ youtubes: [],};// TODO: maybe extract phone numbers from JSON+LD@@ -526,6 +568,7 @@ const parseHandlesFromHtml = (html, data = null) => {result.twitters = html.match(TWITTER_REGEX_GLOBAL) || [];result.instagrams = html.match(INSTAGRAM_REGEX_GLOBAL) || [];result.facebooks = html.match(FACEBOOK_REGEX_GLOBAL) || [];+ result.youtubes = html.match(YOUTUBE_REGEX_GLOBAL) || [];// Sort and deduplicate handles// eslint-disable-next-line guard-for-in, no-restricted-syntax@@ -572,4 +615,7 @@ export const socialUtils = {FACEBOOK_REGEX,FACEBOOK_REGEX_GLOBAL,++ YOUTUBE_REGEX,+ YOUTUBE_REGEX_GLOBAL,};diff --git a/test/utils_social.js b/test/utils_social.jsindex 8cc75135c..305e3c7d8 100644--- a/test/utils_social.js+++ b/test/utils_social.js@@ -318,6 +318,7 @@ describe('utils.social', () => {twitters: [],instagrams: [],facebooks: [],+ youtubes: [],};it('handles invalid arg', () => {@@ -382,6 +383,7 @@ describe('utils.social', () => {https://www.facebook.com/bob.username123/ duplicatehttp://www.facebook.com/alice.username123<a href="https://www.facebook.com/profile.php?id=1155802082&xxx=5">link x</a>+ <a href="https://youtu.be/kM7YfhfkiEE">Youtube</a><a href="fb.com/dada5678?query=1">link</a></body>@@ -419,6 +421,9 @@ describe('utils.social', () => {'https://www.facebook.com/bob.username123/','https://www.facebook.com/profile.php?id=1155802082',],+ youtubes: [+ 'https://youtu.be/kM7YfhfkiEE',+ ],});});@@ -757,4 +762,27 @@ describe('utils.social', () => {expect(''.match(social.FACEBOOK_REGEX_GLOBAL)).to.eql(null);});});+ describe('YOUTUBE_REGEX', () => {+ it('works', () => {+ expect(_.isRegExp(social.YOUTUBE_REGEX)).to.eql(true);+ expect(_.isRegExp(social.YOUTUBE_REGEX_GLOBAL)).to.eql(true);++ expect(social.YOUTUBE_REGEX.flags).to.eql('i');+ expect(social.YOUTUBE_REGEX_GLOBAL.flags).to.eql('gi');++ expect(social.YOUTUBE_REGEX.test('https://www.youtube.com/watch?v=kM7YfhfkiEE')).to.eql(true);+ expect(social.YOUTUBE_REGEX.test('https://youtu.be/kM7YfhfkiEE')).to.eql(true);+ expect(`+ -https://www.youtube.com/someusername/+ youtube.com/jack4567+ https://www.youtube.com/watch?v=kM7YfhfkiEE+ byoutube.com/bob+ ayoutube.com/bob+ _youtube.com/bob+ `.match(social.YOUTUBE_REGEX_GLOBAL))+ .to.eql([+ 'https://www.youtube.com/watch?v=kM7YfhfkiEE',+ ]);+ });+ });});