Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Implement, add tests
Ondra Urban414bca2e0f1f
diff --git a/docs/api/PseudoUrl.md b/docs/api/PseudoUrl.mdindex f3f80938f..eeea316a6 100644--- a/docs/api/PseudoUrl.md+++ b/docs/api/PseudoUrl.md@@ -6,12 +6,14 @@ title: PseudoUrlRepresents a pseudo URL (PURL) - an URL pattern used by web crawlersto specify which URLs should the crawler visit.-This class is used by the [`utils.puppeteer.enqueueLinks()`](puppeteer#puppeteer.enqueueLinks) function.+This class is used by the [`utils.enqueueLinks()`](utils#utils.enqueueLinks) function.A PURL is simply a URL with special directives enclosed in `[]` brackets.Currently, the only supported directive is `[RegExp]`,which defines a JavaScript-style regular expression to match against the URL.+The matching of PURLs against URLs is always case insensitive.+For example, a PURL `http://www.example.com/pages/[(\w|-)*]` will match all of the following URLs:<ul>@@ -57,16 +59,16 @@ if (purl.matches('http://www.example.com/pages/my-awesome-page')) console.log('M</thead><tbody><tr>-<td><code>purl</code></td><td><code>String</code></td>+<td><code>purl</code></td><td><code>String</code> | <code>RegExp</code></td></tr><tr>-<td colspan="3"><p>Pseudo URL.</p>+<td colspan="3"><p>Pseudo URL string or a RegExp instance.</p></td></tr><tr><td><code>requestTemplate</code></td><td><code>Object</code></td></tr><tr><td colspan="3"><p>Options for the new <a href="request"><code>Request</code></a> instances created for matching URLs- by the <a href="puppeteer#puppeteer.enqueueLinks"><code>utils.puppeteer.enqueueLinks()</code></a> function.</p>+ by the <a href="utils#utils.enqueueLinks"><code>utils.enqueueLinks()</code></a> function.</p></td></tr></tbody></table><a name="PseudoUrl+matches"></a>diff --git a/docs/api/utils.md b/docs/api/utils.mdindex a3c91a3a3..37f93eb4b 100644--- a/docs/api/utils.md+++ b/docs/api/utils.md@@ -99,12 +99,12 @@ await Apify.utils.enqueueLinks({<tr><td colspan="3"><p>A CSS selector matching links to be enqueued.</p></td></tr><tr>-<td><code>[options.pseudoUrls]</code></td><td><code><a href="pseudourl">Array<PseudoUrl></a></code> | <code>Array<Object></code> | <code>Array<String></code></td><td></td>+<td><code>[options.pseudoUrls]</code></td><td><code>Array<Object></code> | <code>Array<String></code></td><td></td></tr><tr><td colspan="3"><p>An array of <a href="pseudourl"><code>PseudoUrl</code></a>s matching the URLs to be enqueued,- or an array of strings or objects from which the <a href="pseudourl"><code>PseudoUrl</code></a>s can be constructed.- The objects must include at least the <code>purl</code> property, which holds the pseudo-URL string.+ or an array of strings or RegExps or plain Objects from which the <a href="pseudourl"><code>PseudoUrl</code></a>s can be constructed.</p>+<p> The plain objects must include at least the <code>purl</code> property, which holds the pseudo-URL string or RegExp.All remaining keys will be used as the <code>requestTemplate</code> argument of the <a href="pseudourl"><code>PseudoUrl</code></a> constructor.which lets you specify special properties for the enqueued <a href="request"><code>Request</code></a> objects.</p><p> If <code>pseudoUrls</code> is an empty array, <code>null</code> or <code>undefined</code>, then the functiondiff --git a/src/enqueue_links.js b/src/enqueue_links.jsindex a8e7ea9ae..12c8e3d42 100644--- a/src/enqueue_links.js+++ b/src/enqueue_links.js@@ -17,7 +17,7 @@ export const MAX_ENQUEUE_LINKS_CACHE_SIZE = 1000;/*** Helper factory used in the `enqueueLinks()` function.- * @param {Array} pseudoUrls+ * @param {string[]|Object[]} pseudoUrls* @returns {Array}* @ignore*/@@ -27,10 +27,15 @@ export const constructPseudoUrlInstances = (pseudoUrls) => {let pUrl = enqueueLinksCache.get(item);if (pUrl) return pUrl;// Nothing in cache, make a new instance.- checkParamOrThrow(item, `pseudoUrls[${idx}]`, 'Object|String');+ checkParamOrThrow(item, `pseudoUrls[${idx}]`, 'RegExp|Object|String');++ // If it's already a PseudoURL, just save it.if (item instanceof PseudoUrl) pUrl = item;- else if (typeof item === 'string') pUrl = new PseudoUrl(item);+ // If it's a string or RegExp, construct a PURL from it directly.+ else if (typeof item === 'string' || item instanceof RegExp) pUrl = new PseudoUrl(item);+ // If it's an object, look for a purl property and use it and the rest to construct a PURL with a Request template.else pUrl = new PseudoUrl(item.purl, _.omit(item, 'purl'));+// Manage cacheenqueueLinksCache.set(item, pUrl);if (enqueueLinksCache.size > MAX_ENQUEUE_LINKS_CACHE_SIZE) {@@ -117,10 +122,11 @@ let logDeprecationWarning = true;* A request queue to which the URLs will be enqueued.* @param {String} [options.selector='a']* A CSS selector matching links to be enqueued.- * @param {PseudoUrl[]|Object[]|String[]} [options.pseudoUrls]+ * @param {Object[]|String[]} [options.pseudoUrls]* An array of {@link PseudoUrl}s matching the URLs to be enqueued,- * or an array of strings or objects from which the {@link PseudoUrl}s can be constructed.- * The objects must include at least the `purl` property, which holds the pseudo-URL string.+ * or an array of strings or RegExps or plain Objects from which the {@link PseudoUrl}s can be constructed.+ *+ * The plain objects must include at least the `purl` property, which holds the pseudo-URL string or RegExp.* All remaining keys will be used as the `requestTemplate` argument of the {@link PseudoUrl} constructor.* which lets you specify special properties for the enqueued {@link Request} objects.*diff --git a/src/pseudo_url.js b/src/pseudo_url.jsindex e578d6787..2cd848861 100644--- a/src/pseudo_url.js+++ b/src/pseudo_url.js@@ -52,12 +52,14 @@ const parsePurl = (purl) => {/*** Represents a pseudo URL (PURL) - an URL pattern used by web crawlers* to specify which URLs should the crawler visit.- * This class is used by the [`utils.puppeteer.enqueueLinks()`](puppeteer#puppeteer.enqueueLinks) function.+ * This class is used by the [`utils.enqueueLinks()`](utils#utils.enqueueLinks) function.** A PURL is simply a URL with special directives enclosed in `[]` brackets.* Currently, the only supported directive is `[RegExp]`,* which defines a JavaScript-style regular expression to match against the URL.*+ * The matching of PURLs against URLs is always case insensitive.+ ** For example, a PURL `http://www.example.com/pages/[(\w|-)*]` will match all of the following URLs:** <ul>@@ -86,22 +88,25 @@ const parsePurl = (purl) => {* if (purl.matches('http://www.example.com/pages/my-awesome-page')) console.log('Match!');* ```*- * @param {String} purl- * Pseudo URL.+ * @param {String|RegExp} purl+ * Pseudo URL string or a RegExp instance.* @param {Object} requestTemplate* Options for the new {@link Request} instances created for matching URLs- * by the [`utils.puppeteer.enqueueLinks()`](puppeteer#puppeteer.enqueueLinks) function.+ * by the [`utils.enqueueLinks()`](utils#utils.enqueueLinks) function.*/class PseudoUrl {constructor(purl, requestTemplate = {}) {- checkParamOrThrow(purl, 'purl', 'String');+ checkParamOrThrow(purl, 'purl', 'String|RegExp');checkParamOrThrow(requestTemplate, 'requestTemplate', 'Object');- const regex = parsePurl(purl);-- log.debug('PURL parsed', { purl, regex });+ if (purl instanceof RegExp) {+ this.regex = purl;+ } else {+ const regex = parsePurl(purl);+ log.debug('PURL parsed', { purl, regex });+ this.regex = new RegExp(regex, 'i');+ }- this.regex = new RegExp(regex);this.requestTemplate = requestTemplate;}diff --git a/test/enqueue_links.js b/test/enqueue_links.jsindex 5b5a2fc7d..0a5576264 100644--- a/test/enqueue_links.js+++ b/test/enqueue_links.js@@ -136,6 +136,34 @@ describe('enqueueLinks()', () => {expect(enqueued[2].userData).to.be.eql({});});+ it('works with RegExp pseudoUrls', async () => {+ const enqueued = [];+ const requestQueue = new RequestQueue('xxx');+ requestQueue.addRequest = async (request) => {+ enqueued.push(request);+ };+ const pseudoUrls = [+ /https:\/\/example\.com\/(\w|-|\/)*/,+ /(http|https):\/\/cool\.com\//,+ ];++ await enqueueLinks({ page, selector: '.click', requestQueue, pseudoUrls });++ expect(enqueued).to.have.lengthOf(3);++ expect(enqueued[0].url).to.be.eql('https://example.com/a/b/first');+ expect(enqueued[0].method).to.be.eql('GET');+ expect(enqueued[0].userData).to.be.eql({});++ expect(enqueued[1].url).to.be.eql('https://example.com/a/b/third');+ expect(enqueued[1].method).to.be.eql('GET');+ expect(enqueued[1].userData).to.be.eql({});++ expect(enqueued[2].url).to.be.eql('http://cool.com/');+ expect(enqueued[2].method).to.be.eql('GET');+ expect(enqueued[2].userData).to.be.eql({});+ });+it('works with undefined pseudoUrls[]', async () => {const enqueued = [];const requestQueue = new RequestQueue('xxx');@@ -446,6 +474,34 @@ describe('enqueueLinks()', () => {expect(enqueued[2].userData).to.be.eql({});});+ it('works with RegExp pseudoUrls', async () => {+ const enqueued = [];+ const requestQueue = new RequestQueue('xxx');+ requestQueue.addRequest = async (request) => {+ enqueued.push(request);+ };+ const pseudoUrls = [+ /https:\/\/example\.com\/(\w|-|\/)*/,+ /(http|https):\/\/cool\.com\//,+ ];++ await enqueueLinks({ $, selector: '.click', requestQueue, pseudoUrls });++ expect(enqueued).to.have.lengthOf(3);++ expect(enqueued[0].url).to.be.eql('https://example.com/a/b/first');+ expect(enqueued[0].method).to.be.eql('GET');+ expect(enqueued[0].userData).to.be.eql({});++ expect(enqueued[1].url).to.be.eql('https://example.com/a/b/third');+ expect(enqueued[1].method).to.be.eql('GET');+ expect(enqueued[1].userData).to.be.eql({});++ expect(enqueued[2].url).to.be.eql('http://cool.com/');+ expect(enqueued[2].method).to.be.eql('GET');+ expect(enqueued[2].userData).to.be.eql({});+ });+it('works with undefined pseudoUrls[]', async () => {const enqueued = [];const requestQueue = new RequestQueue('xxx');diff --git a/test/pseudo_url.js b/test/pseudo_url.jsindex d166a0a9f..1a828ebe2 100644--- a/test/pseudo_url.js+++ b/test/pseudo_url.js@@ -3,11 +3,17 @@ import Apify from '../build/index';describe('Apify.PseudoUrl', () => {it('matches() should work', () => {- const purl = new Apify.PseudoUrl('http://www.example.com/pages/[(\\w|-)*]'); // eslint-disable-line+ let purl = new Apify.PseudoUrl('http://www.example.com/PAGES/[(\\w|-)*]'); // eslint-disable-line- expect(purl.matches('http://www.example.com/pages/')).to.be.eql(true);+ expect(purl.matches('http://www.example.com/PAGES/')).to.be.eql(true);expect(purl.matches('http://www.example.com/pages/my-awesome-page')).to.be.eql(true);- expect(purl.matches('http://www.example.com/pages/not@working')).to.be.eql(false);+ expect(purl.matches('http://www.example.com/PAGES/not@working')).to.be.eql(false);++ purl = new Apify.PseudoUrl(/example\.com\/pages/);++ expect(purl.matches('http://www.example.com/PAGES/')).to.be.eql(false);+ expect(purl.matches('http://www.example.com/pages/my-awesome-page')).to.be.eql(true);+ expect(purl.matches('http://www.example.com/pages/not@working')).to.be.eql(true);});it('createRequest() should work', () => {