Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Merge branch 'master' into feature/add_yelp_youtube_social
Ondra Urban13a5bf814856
diff --cc CHANGELOG.mdindex 69aa260a9,5c3317e29..075ec1e49--- a/CHANGELOG.md+++ b/CHANGELOG.md@@@ -1,6 -1,13 +1,14 @@@NEXT====================+ - **DEPRECATED**: `Apify.callTask()` `body` and `contentType` options are now deprecated.+ Use `input` instead. It must be of `content-type: application/json`.+ - Add the ability to create ad-hoc webhooks via `Apify.call()` and `Apify.callTask()`.+ - Add an example of form filling with `Puppeteer`.+ - Fix a race condition in `RequestQueueLocal`.+ - Fix `RequestList` persistence of downloaded sources in key-value store.+ - Fix `Apify.utils.puppeteer.blockRequests()` always including default patterns.+ - Fix parsing bad Content-Type headers for `CheerioCrawler`.+- Added regexps for Youtube videos (`YOUTUBE_REGEX`, `YOUTUBE_REGEX_GLOBAL`) to `utils.social`0.16.1 / 2019-10-31====================diff --cc test/utils_social.test.jsindex 305e3c7d8,9851ff059..a1508935f--- a/test/utils_social.test.js+++ b/test/utils_social.test.js@@@ -318,20 -314,19 +314,20 @@@ describe('utils.social', () =>twitters: [],instagrams: [],facebooks: [],+ youtubes: [],};- it('handles invalid arg', () => {- expect(social.parseHandlesFromHtml()).to.eql(EMPTY_RESULT);- expect(social.parseHandlesFromHtml(undefined)).to.eql(EMPTY_RESULT);- expect(social.parseHandlesFromHtml(null)).to.eql(EMPTY_RESULT);- expect(social.parseHandlesFromHtml({})).to.eql(EMPTY_RESULT);- expect(social.parseHandlesFromHtml(1234)).to.eql(EMPTY_RESULT);+ test('handles invalid arg', () => {+ expect(social.parseHandlesFromHtml()).toEqual(EMPTY_RESULT);+ expect(social.parseHandlesFromHtml(undefined)).toEqual(EMPTY_RESULT);+ expect(social.parseHandlesFromHtml(null)).toEqual(EMPTY_RESULT);+ expect(social.parseHandlesFromHtml({})).toEqual(EMPTY_RESULT);+ expect(social.parseHandlesFromHtml(1234)).toEqual(EMPTY_RESULT);});- it('works', () => {- expect(social.parseHandlesFromHtml('')).to.eql(EMPTY_RESULT);- expect(social.parseHandlesFromHtml(' ')).to.eql(EMPTY_RESULT);+ test('works', () => {+ expect(social.parseHandlesFromHtml('')).toEqual(EMPTY_RESULT);+ expect(social.parseHandlesFromHtml(' ')).toEqual(EMPTY_RESULT);expect(social.parseHandlesFromHtml(`<html>@@@ -753,36 -736,12 +741,35 @@@xfacebook.com/bobafacebook.com/bob_facebook.com/bob- `.match(social.FACEBOOK_REGEX_GLOBAL))- .to.eql([- 'https://www.facebook.com/someusername/',- 'facebook.com/jack4567',- 'fb.com/carl123',- ]);- expect(''.match(social.FACEBOOK_REGEX_GLOBAL)).to.eql(null);+ `.match(social.FACEBOOK_REGEX_GLOBAL)).toEqual([+ 'https://www.facebook.com/someusername/',+ 'facebook.com/jack4567',+ 'fb.com/carl123',+ ]);+ expect(''.match(social.FACEBOOK_REGEX_GLOBAL)).toBe(null);});});+ describe('YOUTUBE_REGEX', () => {+ it('works', () => {+ expect(_.isRegExp(social.YOUTUBE_REGEX)).to.eql(true);+ expect(_.isRegExp(social.YOUTUBE_REGEX_GLOBAL)).to.eql(true);++ expect(social.YOUTUBE_REGEX.flags).to.eql('i');+ expect(social.YOUTUBE_REGEX_GLOBAL.flags).to.eql('gi');++ expect(social.YOUTUBE_REGEX.test('https://www.youtube.com/watch?v=kM7YfhfkiEE')).to.eql(true);+ expect(social.YOUTUBE_REGEX.test('https://youtu.be/kM7YfhfkiEE')).to.eql(true);+ expect(`+ -https://www.youtube.com/someusername/+ youtube.com/jack4567+ https://www.youtube.com/watch?v=kM7YfhfkiEE+ byoutube.com/bob+ ayoutube.com/bob+ _youtube.com/bob+ `.match(social.YOUTUBE_REGEX_GLOBAL))+ .to.eql([+ 'https://www.youtube.com/watch?v=kM7YfhfkiEE',+ ]);+ });+ });});