Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Adding enqueueClickables function and requestTemplate to Pseudo-URL (#110)

Marek Trunkát0183133abcbf
diff --git a/src/pseudo_url.js b/src/pseudo_url.js
index 6f66778e3..a6ee39189 100644
--- a/src/pseudo_url.js
+++ b/src/pseudo_url.js
@@ -1,5 +1,52 @@
import _ from 'underscore';
import log from 'apify-shared/log';
+import { checkParamOrThrow } from 'apify-client/build/utils';
+import Request from './request';
+
+/**
+ * Parses PURL into Regex string.
+ */
+const parsePurl = (purl) => {
+ const trimmedPurl = purl.trim();
+ if (trimmedPurl.length === 0) throw new Error(`Cannot parse PURL '${trimmedPurl}': it must be an non-empty string`);
+
+ let regex = '^';
+
+ try {
+ let openBrackets = 0;
+ for (let i = 0; i < trimmedPurl.length; i++) {
+ const ch = trimmedPurl.charAt(i);
+
+ if (ch === '[' && ++openBrackets === 1) {
+ // Beginning of '[regex]' section
+ // Enclose regex in () brackets to enforce operator priority
+ regex += '(';
+ } else if (ch === ']' && openBrackets > 0 && --openBrackets === 0) {
+ // End of '[regex]' section
+ regex += ')';
+ } else if (openBrackets > 0) {
+ // Inside '[regex]' section
+ regex += ch;
+ } else {
+ // Outside '[regex]' section, parsing the URL part
+ const code = ch.charCodeAt(0);
+ if ((code >= 48 && code <= 57) || (code >= 65 && code <= 90) || (code >= 97 && code <= 122)) {
+ // Alphanumeric character => copy it.
+ regex += ch;
+ } else {
+ // Special character => escape it
+ const hex = code < 16 ? `0${code.toString(16)}` : code.toString(16);
+ regex += `\\x${hex}`;
+ }
+ }
+ }
+ regex += '$';
+ } catch (err) {
+ throw new Error(`Cannot parse PURL '${purl}': ${err}`);
+ }
+
+ return regex;
+};
/**
* Represents a pseudo URL (PURL), which is simply a URL with special directives enclosed in [] brackets.
@@ -22,51 +69,19 @@ import log from 'apify-shared/log';
* ```
*
* @param {String} purl Pseudo url.
+ * @param {Object} requestTemplate Request options for created requests.
*/
export default class PseudoUrl {
- constructor(purl) {
- purl = _.isString(purl) ? purl.trim() : '';
- if (purl.length === 0) throw new Error(`Cannot parse PURL '${purl}': it must be an non-empty string`);
+ constructor(purl, requestTemplate = {}) {
+ checkParamOrThrow(purl, 'purl', 'String');
+ checkParamOrThrow(requestTemplate, 'requestTemplate', 'Object');
- // Generate a regular expression from the pseudo-URL
- // TODO: if input URL contains '[' or ']', they should be matched their URL-escaped counterparts !!!
- try {
- let regex = '^';
- let openBrackets = 0;
- for (let i = 0; i < purl.length; i++) {
- const ch = purl.charAt(i);
+ const regex = parsePurl(purl);
- if (ch === '[' && ++openBrackets === 1) {
- // Beginning of '[regex]' section
- // Enclose regex in () brackets to enforce operator priority
- regex += '(';
- } else if (ch === ']' && openBrackets > 0 && --openBrackets === 0) {
- // End of '[regex]' section
- regex += ')';
- } else if (openBrackets > 0) {
- // Inside '[regex]' section
- regex += ch;
- } else {
- // Outside '[regex]' section, parsing the URL part
- const code = ch.charCodeAt(0);
- if ((code >= 48 && code <= 57) || (code >= 65 && code <= 90) || (code >= 97 && code <= 122)) {
- // Alphanumeric character => copy it.
- regex += ch;
- } else {
- // Special character => escape it
- const hex = code < 16 ? `0${code.toString(16)}` : code.toString(16);
- regex += `\\x${hex}`;
- }
- }
- }
- regex += '$';
- this.regExpString = regex; // useful for debugging, prepared config is printed out including this filed
- this.regExp = new RegExp(regex);
+ log.debug('PURL parsed', { purl, regex });
- log.debug('PURL parsed', { purl, regex });
- } catch (e) {
- throw new Error(`Cannot parse PURL '${purl}': ${e}`);
- }
+ this.regex = new RegExp(regex);
+ this.requestTemplate = requestTemplate;
}
/**
@@ -76,6 +91,16 @@ export default class PseudoUrl {
* @return {Boolean} Returns `true` if given URL matches pseudo URL.
*/
matches(url) {
- return _.isString(url) && url.match(this.regExp) !== null;
+ return _.isString(url) && url.match(this.regex) !== null;
+ }
+
+ /**
+ * Creates a Request object from requestTemplate and given URL.
+ *
+ * @param {String} url
+ * @return {Request}
+ */
+ createRequest(url) {
+ return new Request(Object.assign({ url }, this.requestTemplate));
}
}
diff --git a/src/puppeteer_utils.js b/src/puppeteer_utils.js
index 3d0c0464e..8ebc22539 100644
--- a/src/puppeteer_utils.js
+++ b/src/puppeteer_utils.js
@@ -1,6 +1,7 @@
import fs from 'fs';
import Promise from 'bluebird';
import _ from 'underscore';
+import log from 'apify-shared/log';
import { checkParamOrThrow } from 'apify-client/build/utils';
import { checkParamPrototypeOrThrow } from 'apify-shared/utilities';
import { RequestQueue, RequestQueueLocal } from './request_queue';
@@ -104,8 +105,31 @@ const injectUnderscore = (page) => {
};
/**
- * Finds HTML elements matching a CSS selector, clicks the elements and if a redirect is triggered
- * and destination URL matches one of the provided pseudo-URLs, then the function enqueues that URL to a given request queue.
+ * DEPRECATED!
+ * TODO: Remove after v1.0.0 gets released.
+ * @ignore
+ */
+const enqueueRequestsFromClickableElements = async (page, selector, purls, requestQueue, requestOpts = {}) => {
+ log.warning('Function enqueueRequestsFromClickableElements is deprecated!!! Use `enqueueLinks` instead!');
+
+ checkParamOrThrow(page, 'page', 'Object');
+ checkParamOrThrow(purls, 'purls', 'Array');
+ checkParamPrototypeOrThrow(requestQueue, 'requestQueue', [RequestQueue, RequestQueueLocal], 'Apify.RequestQueue');
+ checkParamOrThrow(requestOpts, 'requestOpts', 'Object');
+
+ /* istanbul ignore next */
+ const getHrefs = linkEls => linkEls.map(link => link.href).filter(href => !!href);
+ const matchesPseudoUrl = url => _.some(purls, purl => purl.matches(url));
+ const urls = await page.$$eval(selector, getHrefs);
+ const requests = urls.filter(matchesPseudoUrl).map(url => new Request(Object.assign({ url }, requestOpts)));
+
+ return Promise.mapSeries(requests, request => requestQueue.addRequest(request));
+};
+
+/**
+ * Finds HTML elements matching a CSS selector, clicks the elements and if a redirect is triggered and destination URL matches
+ * one of the provided pseudo-URLs, then the function enqueues that URL to a given request queue.
+ * To create a Request object function uses `requestTemplate` from a matching Pseudo-URL.
*
* *WARNING*: This is work in progress. Currently the function doesn't click elements and only takes their `href` attribute and so
* is working only for link (`a`) elements and not for buttons or javascript links.
@@ -114,21 +138,25 @@ const injectUnderscore = (page) => {
* @param {String} selector CSS selector matching elements to be clicked.
* @param {Array} pseudoUrls An array of `Apify.PseudoUrl` objects matching URL to be enqueued.
* @param {RequestQueue} requestQueue `Apify.RequestQueue` object where URLs will be enqueued.
- * @param {Object} requestOpts Optional `Apify.Request` options such as `userData` or `method` for the enqueued `Request` objects.
* @return {Promise}
* @memberof utils.puppeteer
*/
-const enqueueRequestsFromClickableElements = async (page, selector, purls, requestQueue, requestOpts = {}) => {
+const enqueueLinks = async (page, selector, purls, requestQueue) => {
checkParamOrThrow(page, 'page', 'Object');
+ checkParamOrThrow(selector, 'selector', 'String');
checkParamOrThrow(purls, 'purls', 'Array');
checkParamPrototypeOrThrow(requestQueue, 'requestQueue', [RequestQueue, RequestQueueLocal], 'Apify.RequestQueue');
- checkParamOrThrow(requestOpts, 'requestOpts', 'Object');
/* istanbul ignore next */
const getHrefs = linkEls => linkEls.map(link => link.href).filter(href => !!href);
- const matchesPseudoUrl = url => _.some(purls, purl => purl.matches(url));
const urls = await page.$$eval(selector, getHrefs);
- const requests = urls.filter(matchesPseudoUrl).map(url => new Request(Object.assign({ url }, requestOpts)));
+ const requests = [];
+
+ urls.forEach((url) => {
+ purls
+ .filter(purl => purl.matches(url))
+ .forEach(purl => requests.push(purl.createRequest(url)));
+ });
return Promise.mapSeries(requests, request => requestQueue.addRequest(request));
};
@@ -157,4 +185,5 @@ export const puppeteerUtils = {
injectJQuery,
injectUnderscore,
enqueueRequestsFromClickableElements,
+ enqueueLinks,
};
diff --git a/src/request.js b/src/request.js
index d8a4f99fb..35f32d706 100644
--- a/src/request.js
+++ b/src/request.js
@@ -101,4 +101,3 @@ export default class Request {
this.errorMessages.push(message);
}
}
-
diff --git a/test/pseudo_url.js b/test/pseudo_url.js
index 33cbee71b..d166a0a9f 100644
--- a/test/pseudo_url.js
+++ b/test/pseudo_url.js
@@ -2,11 +2,21 @@ import { expect } from 'chai';
import Apify from '../build/index';
describe('Apify.PseudoUrl', () => {
- it('should work', () => {
+ it('matches() should work', () => {
const purl = new Apify.PseudoUrl('http://www.example.com/pages/[(\\w|-)*]'); // eslint-disable-line
expect(purl.matches('http://www.example.com/pages/')).to.be.eql(true);
expect(purl.matches('http://www.example.com/pages/my-awesome-page')).to.be.eql(true);
expect(purl.matches('http://www.example.com/pages/not@working')).to.be.eql(false);
});
+
+ it('createRequest() should work', () => {
+ const purl = new Apify.PseudoUrl('something', { method: 'POST', userData: { foo: 'bar' } });
+ const request = purl.createRequest('http://example.com');
+
+ expect(request).to.be.an.instanceof(Apify.Request);
+ expect(request.url).to.be.eql('http://example.com');
+ expect(request.method).to.be.eql('POST');
+ expect(request.userData).to.be.eql({ foo: 'bar' });
+ });
});
diff --git a/test/puppeteer_utils.js b/test/puppeteer_utils.js
index 4075eb680..56909f5ed 100644
--- a/test/puppeteer_utils.js
+++ b/test/puppeteer_utils.js
@@ -139,4 +139,60 @@ describe('Apify.utils.puppeteer', () => {
browser.close();
}
});
+
+ it('enqueueLinks()', async () => {
+ const browser = await Apify.launchPuppeteer({ headless: true, dumpio: true });
+
+ try {
+ const page = await browser.newPage();
+ await page.setContent(`<html>
+ <head>
+ <title>Example</title>
+ </head>
+ <body>
+ <p>
+ The ships hung in the sky, much the <a class="click" href="https://example.com/a/b/first">way that</a> bricks don't.
+ </p>
+ <ul>
+ <li>These aren't the Droids you're looking for</li>
+ <li><a href="https://example.com/a/second">I'm sorry, Dave. I'm afraid I can't do that.</a></li>
+ <li><a class="click" href="https://example.com/a/b/third">I'm sorry, Dave. I'm afraid I can't do that.</a></li>
+ </ul>
+ <a class="click" href="https://another.com/a/fifth">The Greatest Science Fiction Quotes Of All Time</a>
+ <p>
+ Don't know, I don't know such stuff. I just do eyes, ju-, ju-, just eyes... just genetic design,
+ just eyes. You Nexus, huh? I design your <a class="click" href="http://cool.com/">eyes</a>.
+ </p>
+ </body>
+ </html>`);
+
+
+ const enqueued = [];
+ const queue = new RequestQueue('xxx');
+ queue.addRequest = (request) => {
+ enqueued.push(request);
+ return Promise.resolve();
+ };
+ const purls = [
+ new Apify.PseudoUrl('https://example.com/[(\\w|-|/)*]', { method: 'POST' }),
+ new Apify.PseudoUrl('[http|https]://cool.com/', { userData: { foo: 'bar' } }),
+ ];
+
+ await Apify.utils.puppeteer.enqueueLinks(page, '.click', purls, queue);
+
+ expect(enqueued[0].url).to.be.eql('https://example.com/a/b/first');
+ expect(enqueued[0].method).to.be.eql('POST');
+ expect(enqueued[0].userData).to.be.eql({});
+
+ expect(enqueued[1].url).to.be.eql('https://example.com/a/b/third');
+ expect(enqueued[1].method).to.be.eql('POST');
+ expect(enqueued[1].userData).to.be.eql({});
+
+ expect(enqueued[2].url).to.be.eql('http://cool.com/');
+ expect(enqueued[2].method).to.be.eql('GET');
+ expect(enqueued[2].userData.foo).to.be.eql('bar');
+ } finally {
+ browser.close();
+ }
+ });
});