Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Adding util function to click elements (#91)

Marek Trunkát7df7facb5ace
diff --git a/src/index.js b/src/index.js
index e48808c95..f0620a7cc 100644
--- a/src/index.js
+++ b/src/index.js
@@ -16,6 +16,7 @@ import SettingsRotator from './settings_rotator';
import { apifyClient, getMemoryInfo, isProduction, isAtHome, publicUtils } from './utils';
import { browse, launchWebDriver } from './webdriver';
import { puppeteerUtils } from './puppeteer_utils';
+import PseudoUrl from './pseudo_url';
/* globals module */
@@ -73,6 +74,9 @@ module.exports = {
PuppeteerPool,
PuppeteerCrawler,
+ // PseudoUrl
+ PseudoUrl,
+
// Requests
Request,
RequestList,
diff --git a/src/pseudo_url.js b/src/pseudo_url.js
new file mode 100644
index 000000000..6f66778e3
--- /dev/null
+++ b/src/pseudo_url.js
@@ -0,0 +1,81 @@
+import _ from 'underscore';
+import log from 'apify-shared/log';
+
+/**
+ * Represents a pseudo URL (PURL), which is simply a URL with special directives enclosed in [] brackets.
+ * Currently, the only supported directive is [regexp], which defines a JavaScript-style regular expression to match against the URL.
+ *
+ * For example, a PURL `http://www.example.com/pages/[(\w|-)*]` will match all of the following URLs:
+ *
+ * <ul>
+ * <li>`http://www.example.com/pages/`</li>
+ * <li>`http://www.example.com/pages/my-awesome-page`</li>
+ * <li>`http://www.example.com/pages/something`</li>
+ * </ul>
+ *
+ * Example use:
+ *
+ * ```javascript
+ * const purl = new Apify.PseudoUrl('http://www.example.com/pages/[(\w|-)*]');
+ *
+ * if (purl.matches('http://www.example.com/pages/my-awesome-page')) console.log('Match!');
+ * ```
+ *
+ * @param {String} purl Pseudo url.
+ */
+export default class PseudoUrl {
+ constructor(purl) {
+ purl = _.isString(purl) ? purl.trim() : '';
+ if (purl.length === 0) throw new Error(`Cannot parse PURL '${purl}': it must be an non-empty string`);
+
+ // Generate a regular expression from the pseudo-URL
+ // TODO: if input URL contains '[' or ']', they should be matched their URL-escaped counterparts !!!
+ try {
+ let regex = '^';
+ let openBrackets = 0;
+ for (let i = 0; i < purl.length; i++) {
+ const ch = purl.charAt(i);
+
+ if (ch === '[' && ++openBrackets === 1) {
+ // Beginning of '[regex]' section
+ // Enclose regex in () brackets to enforce operator priority
+ regex += '(';
+ } else if (ch === ']' && openBrackets > 0 && --openBrackets === 0) {
+ // End of '[regex]' section
+ regex += ')';
+ } else if (openBrackets > 0) {
+ // Inside '[regex]' section
+ regex += ch;
+ } else {
+ // Outside '[regex]' section, parsing the URL part
+ const code = ch.charCodeAt(0);
+ if ((code >= 48 && code <= 57) || (code >= 65 && code <= 90) || (code >= 97 && code <= 122)) {
+ // Alphanumeric character => copy it.
+ regex += ch;
+ } else {
+ // Special character => escape it
+ const hex = code < 16 ? `0${code.toString(16)}` : code.toString(16);
+ regex += `\\x${hex}`;
+ }
+ }
+ }
+ regex += '$';
+ this.regExpString = regex; // useful for debugging, prepared config is printed out including this filed
+ this.regExp = new RegExp(regex);
+
+ log.debug('PURL parsed', { purl, regex });
+ } catch (e) {
+ throw new Error(`Cannot parse PURL '${purl}': ${e}`);
+ }
+ }
+
+ /**
+ * Determines whether a URL matches this pseudo-URL pattern.
+ *
+ * @param {String} url URL to be matched.
+ * @return {Boolean} Returns `true` if given URL matches pseudo URL.
+ */
+ matches(url) {
+ return _.isString(url) && url.match(this.regExp) !== null;
+ }
+}
diff --git a/src/puppeteer_utils.js b/src/puppeteer_utils.js
index f0770598f..0e46ea975 100644
--- a/src/puppeteer_utils.js
+++ b/src/puppeteer_utils.js
@@ -1,6 +1,11 @@
import fs from 'fs';
import path from 'path';
import Promise from 'bluebird';
+import _ from 'underscore';
+import { checkParamOrThrow } from 'apify-client/build/utils';
+import { checkParamPrototypeOrThrow } from 'apify-shared/utilities';
+import { RequestQueue, RequestQueueLocal } from './request_queue';
+import Request from './request';
const readFilePromised = Promise.promisify(fs.readFile);
@@ -8,11 +13,13 @@ const readFilePromised = Promise.promisify(fs.readFile);
* Hides certain Puppeteer fingerprints from the page, in order to help avoid detection of the crawler.
* The function should be called on a newly-created page object before navigating to the target crawled page.
*
- * @param page Puppeteer [Page](https://github.com/GoogleChrome/puppeteer/blob/master/docs/api.md#class-page) object.
+ * @param {Page} page Puppeteer [Page](https://github.com/GoogleChrome/puppeteer/blob/master/docs/api.md#class-page) object.
* @return {Promise}
* @memberof utils.puppeteer
*/
const hideWebDriver = async (page) => {
+ checkParamOrThrow(page, 'page', 'Object');
+
await page.evaluateOnNewDocument(() => {
var modifiedNavigator; // eslint-disable-line no-var
try {
@@ -42,23 +49,24 @@ const hideWebDriver = async (page) => {
});
};
-
/**
* Injects a JavaScript file into a Puppeteer page.
* Unlike Puppeteer's `addScriptTag` function, this function works on pages
* with arbitrary Cross-Origin Resource Sharing (CORS) policies.
*
- * @param page Puppeteer [Page](https://github.com/GoogleChrome/puppeteer/blob/master/docs/api.md#class-page) object.
- * @param filePath File path
+ * @param {Page} page Puppeteer [Page](https://github.com/GoogleChrome/puppeteer/blob/master/docs/api.md#class-page) object.
+ * @param {String} filePath File path
* @return {Promise}
*/
const injectFile = async (page, filePath) => {
+ checkParamOrThrow(page, 'page', 'Object');
+ checkParamOrThrow(filePath, 'filePath', 'String');
+
const contents = await readFilePromised(filePath, 'utf8');
return page.evaluate(contents);
};
-
/**
* Injects [jQuery](https://jquery.com/) library into a Puppeteer page.
* jQuery is often useful for various web scraping and crawling tasks,
@@ -67,30 +75,64 @@ const injectFile = async (page, filePath) => {
* Beware that the injected jQuery object will be set to the `window.$` variable and thus it might cause conflicts with
* libraries included by the page that use the same variable (e.g. another version of jQuery).
*
- * @param page Puppeteer [Page](https://github.com/GoogleChrome/puppeteer/blob/master/docs/api.md#class-page) object.
+ * @param {Page} page Puppeteer [Page](https://github.com/GoogleChrome/puppeteer/blob/master/docs/api.md#class-page) object.
* @return {Promise}
* @memberof utils.puppeteer
*/
const injectJQuery = (page) => {
+ checkParamOrThrow(page, 'page', 'Object');
+
const scriptPath = path.resolve(path.join(__dirname, '../node_modules/jquery/dist/jquery.min.js'));
+
return injectFile(page, scriptPath);
};
-
/**
* Injects [Underscore.js](https://underscorejs.org/) library into a Puppeteer page.
* Beware that the injected Underscore object will be set to the `window._` variable and thus it might cause conflicts with
* libraries included by the page that use the same variable.
*
- * @param page Puppeteer [Page](https://github.com/GoogleChrome/puppeteer/blob/master/docs/api.md#class-page) object.
+ * @param {Page} page Puppeteer [Page](https://github.com/GoogleChrome/puppeteer/blob/master/docs/api.md#class-page) object.
* @return {Promise}
* @memberof utils.puppeteer
*/
const injectUnderscore = (page) => {
+ checkParamOrThrow(page, 'page', 'Object');
+
const scriptPath = path.resolve(path.join(__dirname, '../node_modules/underscore/underscore-min.js'));
+
return injectFile(page, scriptPath);
};
+/**
+ * Finds HTML elements matching a CSS selector, clicks the elements and if a redirect is triggered
+ * and destination URL matches one of the provided pseudo-URLs, then the function enqueues that URL to a given request queue.
+ *
+ * *WARNING*: This is work in progress. Currently the function doesn't click elements and only takes their `href` attribute and so
+ * is working only for link (`a`) elements and not for buttons or javascript links.
+ *
+ * @param {Page} page Puppeteer [Page](https://github.com/GoogleChrome/puppeteer/blob/master/docs/api.md#class-page) object.
+ * @param {String} selector CSS selector matching elements to be clicked.
+ * @param {Array} pseudoUrls An array of `Apify.PseudoUrl` objects matching URL to be enqueued.
+ * @param {RequestQueue} requestQueue `Apify.RequestQueue` object where URLs will be enqueued.
+ * @param {Object} requestOpts Optional `Apify.Request` options such as `userData` or `method` for the enqueued `Request` objects.
+ * @return {Promise}
+ * @memberof utils.puppeteer
+ */
+const enqueueRequestsFromClickableElements = async (page, selector, purls, requestQueue, requestOpts = {}) => {
+ checkParamOrThrow(page, 'page', 'Object');
+ checkParamOrThrow(purls, 'purls', 'Array');
+ checkParamPrototypeOrThrow(requestQueue, 'requestQueue', [RequestQueue, RequestQueueLocal], 'Apify.RequestQueue');
+ checkParamOrThrow(requestOpts, 'requestOpts', 'Object');
+
+ /* istanbul ignore next */
+ const getHrefs = linkEls => linkEls.map(link => link.href).filter(href => !!href);
+ const matchesPseudoUrl = url => _.some(purls, purl => purl.matches(url));
+ const urls = await page.$$eval(selector, getHrefs);
+ const requests = urls.filter(matchesPseudoUrl).map(url => new Request(Object.assign({ url }, requestOpts)));
+
+ return Promise.mapSeries(requests, request => requestQueue.addRequest(request));
+};
/**
* A namespace that contains various Puppeteer utilities.
@@ -115,4 +157,5 @@ export const puppeteerUtils = {
injectFile,
injectJQuery,
injectUnderscore,
+ enqueueRequestsFromClickableElements,
};
diff --git a/test/pseudo_url.js b/test/pseudo_url.js
new file mode 100644
index 000000000..33cbee71b
--- /dev/null
+++ b/test/pseudo_url.js
@@ -0,0 +1,12 @@
+import { expect } from 'chai';
+import Apify from '../build/index';
+
+describe('Apify.PseudoUrl', () => {
+ it('should work', () => {
+ const purl = new Apify.PseudoUrl('http://www.example.com/pages/[(\\w|-)*]'); // eslint-disable-line
+
+ expect(purl.matches('http://www.example.com/pages/')).to.be.eql(true);
+ expect(purl.matches('http://www.example.com/pages/my-awesome-page')).to.be.eql(true);
+ expect(purl.matches('http://www.example.com/pages/not@working')).to.be.eql(false);
+ });
+});
diff --git a/test/puppeteer_utils.js b/test/puppeteer_utils.js
index 11203013a..b5e258667 100644
--- a/test/puppeteer_utils.js
+++ b/test/puppeteer_utils.js
@@ -1,6 +1,7 @@
import chai, { expect } from 'chai';
import chaiAsPromised from 'chai-as-promised';
import Apify from '../build/index';
+import { RequestQueue } from '../build/request_queue';
chai.use(chaiAsPromised);
@@ -86,4 +87,55 @@ describe('Apify.utils.puppeteer', () => {
browser.close();
}
});
+
+ it('enqueueRequestsFromClickableElements()', async () => {
+ const browser = await Apify.launchPuppeteer({ headless: true, dumpio: true });
+
+ try {
+ const page = await browser.newPage();
+ await page.setContent(`<html>
+ <head>
+ <title>Example</title>
+ </head>
+ <body>
+ <p>
+ The ships hung in the sky, much the <a class="click" href="https://example.com/a/b/first">way that</a> bricks don't.
+ </p>
+ <ul>
+ <li>These aren't the Droids you're looking for</li>
+ <li><a href="https://example.com/a/second">I'm sorry, Dave. I'm afraid I can't do that.</a></li>
+ <li><a class="click" href="https://example.com/a/b/third">I'm sorry, Dave. I'm afraid I can't do that.</a></li>
+ </ul>
+ <a class="click" href="https://another.com/a/fifth">The Greatest Science Fiction Quotes Of All Time</a>
+ <p>
+ Don't know, I don't know such stuff. I just do eyes, ju-, ju-, just eyes... just genetic design,
+ just eyes. You Nexus, huh? I design your <a class="click" href="http://cool.com/">eyes</a>.
+ </p>
+ </body>
+ </html>`);
+
+
+ const enqueuedUrls = [];
+ const queue = new RequestQueue('xxx');
+ queue.addRequest = (request) => {
+ expect(request.method).to.be.eql('POST');
+ enqueuedUrls.push(request.url);
+
+ return Promise.resolve();
+ };
+ const purls = [
+ new Apify.PseudoUrl('https://example.com/[(\\w|-|/)*]'),
+ new Apify.PseudoUrl('[http|https]://cool.com/'),
+ ];
+ await Apify.utils.puppeteer.enqueueRequestsFromClickableElements(page, '.click', purls, queue, { method: 'POST' });
+
+ expect(enqueuedUrls).to.be.eql([
+ 'https://example.com/a/b/first',
+ 'https://example.com/a/b/third',
+ 'http://cool.com/',
+ ]);
+ } finally {
+ browser.close();
+ }
+ });
});