Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Merge branch 'master' into feature/add_yelp_youtube_social

Ondra Urban13a5bf814856
diff --cc CHANGELOG.md
index 69aa260a9,5c3317e29..075ec1e49
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@@ -1,6 -1,13 +1,14 @@@
NEXT
====================
+ - **DEPRECATED**: `Apify.callTask()` `body` and `contentType` options are now deprecated.
+ Use `input` instead. It must be of `content-type: application/json`.
+ - Add the ability to create ad-hoc webhooks via `Apify.call()` and `Apify.callTask()`.
+ - Add an example of form filling with `Puppeteer`.
+ - Fix a race condition in `RequestQueueLocal`.
+ - Fix `RequestList` persistence of downloaded sources in key-value store.
+ - Fix `Apify.utils.puppeteer.blockRequests()` always including default patterns.
+ - Fix parsing bad Content-Type headers for `CheerioCrawler`.
+- Added regexps for Youtube videos (`YOUTUBE_REGEX`, `YOUTUBE_REGEX_GLOBAL`) to `utils.social`
0.16.1 / 2019-10-31
====================
diff --cc test/utils_social.test.js
index 305e3c7d8,9851ff059..a1508935f
--- a/test/utils_social.test.js
+++ b/test/utils_social.test.js
@@@ -318,20 -314,19 +314,20 @@@ describe('utils.social', () =>
twitters: [],
instagrams: [],
facebooks: [],
+ youtubes: [],
};
- it('handles invalid arg', () => {
- expect(social.parseHandlesFromHtml()).to.eql(EMPTY_RESULT);
- expect(social.parseHandlesFromHtml(undefined)).to.eql(EMPTY_RESULT);
- expect(social.parseHandlesFromHtml(null)).to.eql(EMPTY_RESULT);
- expect(social.parseHandlesFromHtml({})).to.eql(EMPTY_RESULT);
- expect(social.parseHandlesFromHtml(1234)).to.eql(EMPTY_RESULT);
+ test('handles invalid arg', () => {
+ expect(social.parseHandlesFromHtml()).toEqual(EMPTY_RESULT);
+ expect(social.parseHandlesFromHtml(undefined)).toEqual(EMPTY_RESULT);
+ expect(social.parseHandlesFromHtml(null)).toEqual(EMPTY_RESULT);
+ expect(social.parseHandlesFromHtml({})).toEqual(EMPTY_RESULT);
+ expect(social.parseHandlesFromHtml(1234)).toEqual(EMPTY_RESULT);
});
- it('works', () => {
- expect(social.parseHandlesFromHtml('')).to.eql(EMPTY_RESULT);
- expect(social.parseHandlesFromHtml(' ')).to.eql(EMPTY_RESULT);
+ test('works', () => {
+ expect(social.parseHandlesFromHtml('')).toEqual(EMPTY_RESULT);
+ expect(social.parseHandlesFromHtml(' ')).toEqual(EMPTY_RESULT);
expect(social.parseHandlesFromHtml(`
<html>
@@@ -753,36 -736,12 +741,35 @@@
xfacebook.com/bob
afacebook.com/bob
_facebook.com/bob
- `.match(social.FACEBOOK_REGEX_GLOBAL))
- .to.eql([
- 'https://www.facebook.com/someusername/',
- 'facebook.com/jack4567',
- 'fb.com/carl123',
- ]);
- expect(''.match(social.FACEBOOK_REGEX_GLOBAL)).to.eql(null);
+ `.match(social.FACEBOOK_REGEX_GLOBAL)).toEqual([
+ 'https://www.facebook.com/someusername/',
+ 'facebook.com/jack4567',
+ 'fb.com/carl123',
+ ]);
+ expect(''.match(social.FACEBOOK_REGEX_GLOBAL)).toBe(null);
});
});
+ describe('YOUTUBE_REGEX', () => {
+ it('works', () => {
+ expect(_.isRegExp(social.YOUTUBE_REGEX)).to.eql(true);
+ expect(_.isRegExp(social.YOUTUBE_REGEX_GLOBAL)).to.eql(true);
+
+ expect(social.YOUTUBE_REGEX.flags).to.eql('i');
+ expect(social.YOUTUBE_REGEX_GLOBAL.flags).to.eql('gi');
+
+ expect(social.YOUTUBE_REGEX.test('https://www.youtube.com/watch?v=kM7YfhfkiEE')).to.eql(true);
+ expect(social.YOUTUBE_REGEX.test('https://youtu.be/kM7YfhfkiEE')).to.eql(true);
+ expect(`
+ -https://www.youtube.com/someusername/
+ youtube.com/jack4567
+ https://www.youtube.com/watch?v=kM7YfhfkiEE
+ byoutube.com/bob
+ ayoutube.com/bob
+ _youtube.com/bob
+ `.match(social.YOUTUBE_REGEX_GLOBAL))
+ .to.eql([
+ 'https://www.youtube.com/watch?v=kM7YfhfkiEE',
+ ]);
+ });
+ });
});