Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Updating docs
Jan Curnad207be4b77f
diff --git a/src/puppeteer_utils.js b/src/puppeteer_utils.jsindex 65c08c32f..a8b18e734 100644--- a/src/puppeteer_utils.js+++ b/src/puppeteer_utils.js@@ -272,11 +272,13 @@ const enqueueLinks = async (page, selector, requestQueue, pseudoUrls) => {* For more details, see Puppeteer's* <a href="https://pptr.dev/#?product=Puppeteer&show=api-requestresourcetype" target="_blank">Request.resourceType() documentation</a>.*- * By default, the function blocks these resource types: `stylesheet`, `font`, `image`, `media`.+ * If the `resourceTypes` parameter is not provided,+ * by default the function blocks these resource types: `stylesheet`, `font`, `image`, `media`.** @param {Page} page* Puppeteer <a href="https://pptr.dev/#?product=Puppeteer&show=api-class-page" target="_blank"><code>Page</code></a> object.- * @param {String[]} resourceTypes Array of resource types to block.+ * @param {String[]} [resourceTypes=['stylesheet', 'font', 'image', 'media']]+ * Array of resource types to block.* @return {Promise}* @memberOf puppeteer*/diff --git a/src/utils_social.js b/src/utils_social.jsindex d56b9da03..db39db95e 100644--- a/src/utils_social.js+++ b/src/utils_social.js@@ -212,12 +212,26 @@ let FACEBOOK_REGEX_GLOBAL;try {/**- * Regular expression to exactly match a single LinkedIn profile URL, without any additional- * subdirectories or query parameters. The regular expression has the following form: `/^...$/i`.+ * Regular expression to exactly match a single LinkedIn profile URL.+ * It has the following form: `/^...$/i` and matches URLs such as:+ * ```+ * https://www.linkedin.com/in/alan-turing+ * en.linkedin.com/in/alan-turing+ * linkedin.com/in/alan-turing+ * ```+ *+ * The regular expression does NOT match URLs with additional+ * subdirectories or query parameters, such as:+ * ```+ * https://www.linkedin.com/in/alan-turing/latest-activity+ * en.linkedin.com/in/alan-turing?param=123+ * ```** Example usage:* ```- * TODO+ * if (Apify.utils.social.LINKEDIN_REGEX.test('https://www.linkedin.com/in/alan-turing')) {+ * console.log('Match!');+ * }* ```* @type {RegExp}* @memberOf social@@ -225,8 +239,26 @@ try {LINKEDIN_REGEX = new RegExp(`^${LINKEDIN_REGEX_STRING}$`, 'i');/**- * Regular expression to find multiple LinkedIn profile URLs in a text.- * It has the following form: `/.../ig`.+ * Regular expression to find multiple LinkedIn profile URLs in a text or HTML.+ * It has the following form: `/.../ig` and matches URLs such as:+ * ```+ * https://www.linkedin.com/in/alan-turing+ * en.linkedin.com/in/alan-turing+ * linkedin.com/in/alan-turing+ * ```+ *+ * The regular expression does NOT match URLs with additional+ * subdirectories or query parameters, such as:+ * ```+ * https://www.linkedin.com/in/alan-turing/latest-activity+ * en.linkedin.com/in/alan-turing?param=123+ * ```+ *+ * Example usage:+ * ```+ * const matches = text.match(Apify.utils.social.LINKEDIN_REGEX_GLOBAL);+ * if (matches) console.log(`${matches.length} matches found!`);+ * ```* @type {RegExp}* @memberOf social*/diff --git a/test/utils_social.js b/test/utils_social.jsindex 5e7710675..fa47120b0 100644--- a/test/utils_social.js+++ b/test/utils_social.js@@ -490,6 +490,10 @@ describe('utils.social', () => {expect(social.LINKEDIN_REGEX.test('www.linkedin.com/in/bobnewman')).to.eql(true);expect(social.LINKEDIN_REGEX.test('linkedin.com/in/bobnewman')).to.eql(true);+ expect(social.LINKEDIN_REGEX.test('https://www.linkedin.com/in/alan-turing')).to.eql(true);+ expect(social.LINKEDIN_REGEX.test('en.linkedin.com/in/alan-turing')).to.eql(true);+ expect(social.LINKEDIN_REGEX.test('linkedin.com/in/alan-turing')).to.eql(true);+// Test there is just on matching group for the usernameexpect('https://www.linkedin.com/in/bobnewman/'.match(social.LINKEDIN_REGEX)[1]).to.eql('bobnewman');expect('http://www.linkedin.com/in/bobnewman'.match(social.LINKEDIN_REGEX)[1]).to.eql('bobnewman');