Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Added regexp for Youtube video URLs to utils.social

drobnikj45a420ddb486
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 08830b7d2..69aa260a9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,7 @@
+NEXT
+====================
+- Added regexps for Youtube videos (`YOUTUBE_REGEX`, `YOUTUBE_REGEX_GLOBAL`) to `utils.social`
+
0.16.1 / 2019-10-31
====================
- Bump Puppeteer to 2.0.0 and use `{ pipe: true }` again because upstream bug has been fixed.
diff --git a/docs/api/social.md b/docs/api/social.md
index 5f576b038..d75953642 100644
--- a/docs/api/social.md
+++ b/docs/api/social.md
@@ -24,6 +24,8 @@ const emails = Apify.utils.social.emailsFromText('[email protected] bob@example.
- [`.TWITTER_REGEX_GLOBAL`](#social.TWITTER_REGEX_GLOBAL) : `RegExp`
- [`.FACEBOOK_REGEX`](#social.FACEBOOK_REGEX) : `RegExp`
- [`.FACEBOOK_REGEX_GLOBAL`](#social.FACEBOOK_REGEX_GLOBAL) : `RegExp`
+ - [`.YOUTUBE_REGEX`](#social.YOUTUBE_REGEX) : `RegExp`
+ - [`.YOUTUBE_REGEX_GLOBAL`](#social.YOUTUBE_REGEX_GLOBAL) : `RegExp`
- [`.EMAIL_REGEX`](#social.EMAIL_REGEX) : `RegExp`
- [`.EMAIL_REGEX_GLOBAL`](#social.EMAIL_REGEX_GLOBAL) : `RegExp`
- [`.emailsFromText(text)`](#social.emailsFromText) ⇒ `Array<String>`
@@ -111,7 +113,7 @@ https://www.instagram.com/cristiano/followers
Example usage:
```
-if (Apify.utils.social.INSTAGRAM_REGEX_STRING.test('https://www.instagram.com/old_prague')) {
+if (Apify.utils.social.INSTAGRAM_REGEX.test('https://www.instagram.com/old_prague')) {
console.log('Match!');
}
```
@@ -168,7 +170,7 @@ https://www.twitter.com/realdonaldtrump/following
Example usage:
```
-if (Apify.utils.social.TWITTER_REGEX_STRING.test('https://www.twitter.com/apify')) {
+if (Apify.utils.social.TWITTER_REGEX.test('https://www.twitter.com/apify')) {
console.log('Match!');
}
```
@@ -226,7 +228,7 @@ https://www.facebook.com/apifytech/photos
Example usage:
```
-if (Apify.utils.social.FACEBOOK_REGEX_STRING.test('https://www.facebook.com/apifytech')) {
+if (Apify.utils.social.FACEBOOK_REGEX.test('https://www.facebook.com/apifytech')) {
console.log('Match!');
}
```
@@ -263,6 +265,43 @@ const matches = text.match(Apify.utils.social.FACEBOOK_REGEX_GLOBAL);
if (matches) console.log(`${matches.length} Facebook profiles found!`);
```
+<a name="social.YOUTUBE_REGEX"></a>
+
+## `social.YOUTUBE_REGEX` : `RegExp`
+
+Regular expression to exactly match a single Youtube video URL. It has the following form: `/^...$/i` and matches URLs such as:
+
+```
+https://www.youtube.com/watch?v=kM7YfhfkiEE
+https://youtu.be/kM7YfhfkiEE
+```
+
+Example usage:
+
+```
+if (Apify.utils.social.YOUTUBE_REGEX.test('https://www.youtube.com/watch?v=kM7YfhfkiEE')) {
+ console.log('Match!');
+}
+```
+
+<a name="social.YOUTUBE_REGEX_GLOBAL"></a>
+
+## `social.YOUTUBE_REGEX_GLOBAL` : `RegExp`
+
+Regular expression to find multiple Youtube video URLs in a text or HTML. It has the following form: `/.../ig` and matches URLs such as:
+
+```
+https://www.youtube.com/watch?v=kM7YfhfkiEE
+https://youtu.be/kM7YfhfkiEE
+```
+
+Example usage:
+
+```
+const matches = text.match(Apify.utils.social.YOUTUBE_REGEX_GLOBAL);
+if (matches) console.log(`${matches.length} Youtube videos found!`);
+```
+
<a name="social.EMAIL_REGEX"></a>
## `social.EMAIL_REGEX` : `RegExp`
@@ -384,6 +423,7 @@ The result of the function is an object with the following structure:
twitters: String[],
instagrams: String[],
facebooks: String[],
+ youtubes: String[],
}
```
diff --git a/src/utils_social.js b/src/utils_social.js
index 417c2b720..c57e9643a 100644
--- a/src/utils_social.js
+++ b/src/utils_social.js
@@ -200,7 +200,8 @@ const TWITTER_REGEX_STRING = `(?<!\\w)(?:http(?:s)?:\\/\\/)?(?:www.)?(?:twitter.
const FACEBOOK_RESERVED_PATHS = 'rsrc\\.php|apps|groups|events|l\\.php|friends|images|photo.php|chat|ajax|dyi|common|policies|login|recover|reg|help|security|messages|marketplace|pages|live|bookmarks|games|fundraisers|saved|gaming|salesgroups|jobs|people|ads|ad_campaign|weather|offers|recommendations|crisisresponse|onthisday|developers|settings|connect|business|plugins|intern|sharer';
// eslint-disable-next-line max-len, quotes
const FACEBOOK_REGEX_STRING = `(?<!\\w)(?:http(?:s)?:\\/\\/)?(?:www.)?(?:facebook.com|fb.com)\\/(?!(?:${FACEBOOK_RESERVED_PATHS})(?:[\\'\\"\\?\\.\\/]|$))(profile\\.php\\?id\\=[0-9]{3,20}|(?!profile\\.php)[a-z0-9\\.]{5,51})(?![a-z0-9\\.])(?:/)?`;
-
+// eslint-disable-next-line max-len, quotes
+const YOUTUBE_REGEX_STRING = '(?:https?:\\/\\/)?(?:youtu\\.be\\/|(?:www\\.|m\\.)?youtube\\.com\\/(?:watch|v|embed)(?:\\.php)?(?:\\?.*v=|\\/))([a-zA-Z0-9\\-_]+)';
let LINKEDIN_REGEX;
let LINKEDIN_REGEX_GLOBAL;
@@ -210,6 +211,8 @@ let TWITTER_REGEX;
let TWITTER_REGEX_GLOBAL;
let FACEBOOK_REGEX;
let FACEBOOK_REGEX_GLOBAL;
+let YOUTUBE_REGEX;
+let YOUTUBE_REGEX_GLOBAL;
try {
/**
@@ -284,7 +287,7 @@ try {
*
* Example usage:
* ```
- * if (Apify.utils.social.INSTAGRAM_REGEX_STRING.test('https://www.instagram.com/old_prague')) {
+ * if (Apify.utils.social.INSTAGRAM_REGEX.test('https://www.instagram.com/old_prague')) {
* console.log('Match!');
* }
* ```
@@ -338,7 +341,7 @@ try {
*
* Example usage:
* ```
- * if (Apify.utils.social.TWITTER_REGEX_STRING.test('https://www.twitter.com/apify')) {
+ * if (Apify.utils.social.TWITTER_REGEX.test('https://www.twitter.com/apify')) {
* console.log('Match!');
* }
* ```
@@ -393,7 +396,7 @@ try {
*
* Example usage:
* ```
- * if (Apify.utils.social.FACEBOOK_REGEX_STRING.test('https://www.facebook.com/apifytech')) {
+ * if (Apify.utils.social.FACEBOOK_REGEX.test('https://www.facebook.com/apifytech')) {
* console.log('Match!');
* }
* ```
@@ -430,6 +433,43 @@ try {
* @memberOf social
*/
FACEBOOK_REGEX_GLOBAL = new RegExp(FACEBOOK_REGEX_STRING, 'ig');
+
+ /**
+ * Regular expression to exactly match a single Youtube video URL.
+ * It has the following form: `/^...$/i` and matches URLs such as:
+ * ```
+ * https://www.youtube.com/watch?v=kM7YfhfkiEE
+ * https://youtu.be/kM7YfhfkiEE
+ * ```
+ *
+ * Example usage:
+ * ```
+ * if (Apify.utils.social.YOUTUBE_REGEX.test('https://www.youtube.com/watch?v=kM7YfhfkiEE')) {
+ * console.log('Match!');
+ * }
+ * ```
+ * @type {RegExp}
+ * @memberOf social
+ */
+ YOUTUBE_REGEX = new RegExp(`^${YOUTUBE_REGEX_STRING}$`, 'i');
+
+ /**
+ * Regular expression to find multiple Youtube video URLs in a text or HTML.
+ * It has the following form: `/.../ig` and matches URLs such as:
+ * ```
+ * https://www.youtube.com/watch?v=kM7YfhfkiEE
+ * https://youtu.be/kM7YfhfkiEE
+ * ```
+ *
+ * Example usage:
+ * ```
+ * const matches = text.match(Apify.utils.social.YOUTUBE_REGEX_GLOBAL);
+ * if (matches) console.log(`${matches.length} Youtube videos found!`);
+ * ```
+ * @type {RegExp}
+ * @memberOf social
+ */
+ YOUTUBE_REGEX_GLOBAL = new RegExp(YOUTUBE_REGEX_STRING, 'ig');
} catch (e) {
// Older versions of Node don't support negative lookbehind and lookahead expressions.
// Show warning instead of failing.
@@ -457,6 +497,7 @@ try {
* twitters: String[],
* instagrams: String[],
* facebooks: String[],
+ * youtubes: String[],
* }
* ```
*
@@ -496,6 +537,7 @@ const parseHandlesFromHtml = (html, data = null) => {
twitters: [],
instagrams: [],
facebooks: [],
+ youtubes: [],
};
// TODO: maybe extract phone numbers from JSON+LD
@@ -526,6 +568,7 @@ const parseHandlesFromHtml = (html, data = null) => {
result.twitters = html.match(TWITTER_REGEX_GLOBAL) || [];
result.instagrams = html.match(INSTAGRAM_REGEX_GLOBAL) || [];
result.facebooks = html.match(FACEBOOK_REGEX_GLOBAL) || [];
+ result.youtubes = html.match(YOUTUBE_REGEX_GLOBAL) || [];
// Sort and deduplicate handles
// eslint-disable-next-line guard-for-in, no-restricted-syntax
@@ -572,4 +615,7 @@ export const socialUtils = {
FACEBOOK_REGEX,
FACEBOOK_REGEX_GLOBAL,
+
+ YOUTUBE_REGEX,
+ YOUTUBE_REGEX_GLOBAL,
};
diff --git a/test/utils_social.js b/test/utils_social.js
index 8cc75135c..305e3c7d8 100644
--- a/test/utils_social.js
+++ b/test/utils_social.js
@@ -318,6 +318,7 @@ describe('utils.social', () => {
twitters: [],
instagrams: [],
facebooks: [],
+ youtubes: [],
};
it('handles invalid arg', () => {
@@ -382,6 +383,7 @@ describe('utils.social', () => {
https://www.facebook.com/bob.username123/ duplicate
http://www.facebook.com/alice.username123
<a href="https://www.facebook.com/profile.php?id=1155802082&xxx=5">link x</a>
+ <a href="https://youtu.be/kM7YfhfkiEE">Youtube</a>
<a href="fb.com/dada5678?query=1">link</a>
</body>
@@ -419,6 +421,9 @@ describe('utils.social', () => {
'https://www.facebook.com/bob.username123/',
'https://www.facebook.com/profile.php?id=1155802082',
],
+ youtubes: [
+ 'https://youtu.be/kM7YfhfkiEE',
+ ],
});
});
@@ -757,4 +762,27 @@ describe('utils.social', () => {
expect(''.match(social.FACEBOOK_REGEX_GLOBAL)).to.eql(null);
});
});
+ describe('YOUTUBE_REGEX', () => {
+ it('works', () => {
+ expect(_.isRegExp(social.YOUTUBE_REGEX)).to.eql(true);
+ expect(_.isRegExp(social.YOUTUBE_REGEX_GLOBAL)).to.eql(true);
+
+ expect(social.YOUTUBE_REGEX.flags).to.eql('i');
+ expect(social.YOUTUBE_REGEX_GLOBAL.flags).to.eql('gi');
+
+ expect(social.YOUTUBE_REGEX.test('https://www.youtube.com/watch?v=kM7YfhfkiEE')).to.eql(true);
+ expect(social.YOUTUBE_REGEX.test('https://youtu.be/kM7YfhfkiEE')).to.eql(true);
+ expect(`
+ -https://www.youtube.com/someusername/
+ youtube.com/jack4567
+ https://www.youtube.com/watch?v=kM7YfhfkiEE
+ byoutube.com/bob
+ ayoutube.com/bob
+ _youtube.com/bob
+ `.match(social.YOUTUBE_REGEX_GLOBAL))
+ .to.eql([
+ 'https://www.youtube.com/watch?v=kM7YfhfkiEE',
+ ]);
+ });
+ });
});