Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Add element clicking, refactor enqueue links, add tests
Ondra Urban8336ea4ca6c2
diff --git a/pptr-clicking/.gitignore b/pptr-clicking/.gitignoredeleted file mode 100644index e74bccb3a..000000000--- a/pptr-clicking/.gitignore+++ /dev/null@@ -1,3 +0,0 @@-node_modules--apify_storage\ No newline at end of filediff --git a/pptr-clicking/Dockerfile b/pptr-clicking/Dockerfiledeleted file mode 100644index b8fc0e642..000000000--- a/pptr-clicking/Dockerfile+++ /dev/null@@ -1,18 +0,0 @@-FROM apify/actor-node-chrome--# Copy source code-COPY . ./--# Install default dependencies, print versions of everything-RUN npm --quiet set progress=false \- && npm install --only=prod --no-optional \- && echo "Installed NPM packages:" \- && npm list \- && echo "Node.js version:" \- && node --version \- && echo "NPM version:" \- && npm --version--# By default, the apify/actor-node-chrome image uses "npm start" to run the code.-# You can override this behavior using the CMD instruction here:-# CMD [ "npm", "start" ]diff --git a/pptr-clicking/apify.json b/pptr-clicking/apify.jsondeleted file mode 100644index 56968b21a..000000000--- a/pptr-clicking/apify.json+++ /dev/null@@ -1,7 +0,0 @@-{- "name": "pptr-clicking",- "version": "0.0",- "buildTag": "latest",- "env": null,- "template": "puppeteer_crawler"-}diff --git a/pptr-clicking/main.js b/pptr-clicking/main.jsdeleted file mode 100644index 95a6567c2..000000000--- a/pptr-clicking/main.js+++ /dev/null@@ -1,96 +0,0 @@-const Apify = require('apify');--/**- * Clicks all elements of given page matching given selector.- * Catches and intercepts all the initiated requests and opened pages.- * Returns a list of all target URLs.- *- * @param {Page} page- * @param {String} selector- * @return {Promise<String>}- */-const clickElements = async (page, selector) => {- const urlSet = new Set();- const browser = page.browser();-- const interceptionHandler = (req) => {- if (req.isNavigationRequest() && req.frame() === page.mainFrame() && req.url() !== page.url()) {- // TODO: We need to store also HTTP method, payload and headers here.- urlSet.add(req.url());- req.respond(req.redirectChain().length- ? { body: '' } // Prevents 301/302 redirect- : { status: 204 } // Prevents navigation by js- )- } else {- req.continue();- }- };-- const targetCreatedHandler = async (target) => {- if (target.type() === 'page') {- // TODO: We need to store also HTTP method, payload and headers here.- const page = await target.page();- urlSet.add(page.url());- page.close();- }- };-- // Add request interception handler and on new-target handler.- await Apify.utils.puppeteer.addInterceptRequestHandler(page, interceptionHandler);- browser.on('targetcreated', targetCreatedHandler);-- // Click all the elements.- const elementHandles = await page.$$(selector);- let remainingCount = elementHandles.length;- let zIndex = 10000;- for (let elementHandle of elementHandles) {- console.log(`Remaining elements: ${remainingCount--}`);- try {- // Ensure that element is visible ...- await page.evaluate((el, zIndex) => {- el.style.visiblity = 'visible';- el.style.display = 'block';- el.style.position = 'absolute';- el.style.zindex = zIndex;- }, elementHandle, zIndex++);-- // ... and then click it.- await elementHandle.click();- } catch (err) {- console.log(`Click failed: ${err.message}`);- }- }-- // Removes all listeners.- browser.removeListener('targetcreated', targetCreatedHandler);- await Apify.utils.puppeteer.removeInterceptRequestHandler(page, interceptionHandler);-- return urls;-};--Apify.main(async () => {- const requestQueue = await Apify.openRequestQueue();- await requestQueue.addRequest({- // url: 'https://www.retailmenot.com/',- url: 'https://www.imdb.com/',- });-- const crawler = new Apify.PuppeteerCrawler({- requestQueue,- handlePageTimeoutSecs: 300,- launchPuppeteerOptions: {- headless: true,- },- handlePageFunction: async ({ request, page }) => {- const urls = await clickElements(page, '*');-- console.log(urls);- },- handleFailedRequestFunction: async ({ request }) => {- console.log(`Request ${request.url} failed too many times`);- },- });-- await crawler.run();- console.log('Crawler finished.');-});diff --git a/pptr-clicking/package.json b/pptr-clicking/package.jsondeleted file mode 100644index eeb275dcc..000000000--- a/pptr-clicking/package.json+++ /dev/null@@ -1,21 +0,0 @@-{- "name": "pptr-clicking",- "version": "1.0.0",- "description": "Crawl a number of web pages using headless Chrome and Puppeteer",- "main": "main.js",- "dependencies": {- "apify": "^0.14.5"- },- "devDependencies": {},- "scripts": {- "start": "node main.js",- "test": "echo \"Error: no test specified\" && exit 1"- },- "repository": {- "type": "git",- "url": ""- },- "author": "",- "license": "ISC",- "homepage": ""-}diff --git a/src/enqueue_links/click_elements.js b/src/enqueue_links/click_elements.jsnew file mode 100644index 000000000..4201592a5--- /dev/null+++ b/src/enqueue_links/click_elements.js@@ -0,0 +1,235 @@+import { checkParamOrThrow } from 'apify-client/build/utils';+import { checkParamPrototypeOrThrow } from 'apify-shared/utilities';+import log from 'apify-shared/log';+import { RequestQueue, RequestQueueLocal } from '../request_queue';+import { addInterceptRequestHandler, removeInterceptRequestHandler } from '../puppeteer_request_interception';+import { constructPseudoUrlInstances, createRequests, addRequestsToQueueInBatches } from './shared';++const STARTING_Z_INDEX = 10000;++/**+ * The function finds elements matching a specific CSS selector in a Puppeteer page,+ * clicks all those elements using a mouse move and a left mouse button click and intercepts+ * all the navigation requests that are subsequently produced by the page. The intercepted+ * requests, including their methods, headers and payloads are then enqueued to a provided+ * {@link RequestQueue}. This is useful to crawl JavaScript heavy pages where links are not available+ * in `href` elements, but rather navigations are triggered in click handlers.+ * If you're looking to find URLs in `href` attributes of the page, see [`enqueueLinks()`](utils#utils.enqueueLinks).+ *+ * Optionally, the function allows you to filter the target links' URLs using an array of {@link PseudoUrl} objects+ * and override settings of the enqueued {@link Request} objects.+ *+ * *IMPORTANT*: To be able to do this, this function uses various mutations on the page,+ * such as changing the Z-index of elements being clicked and their visibility. Therefore,+ * it is recommended to only use this function as the last operation in the page.+ *+ * *PERFORMANCE*: Clicking elements with a mouse and intercepting requests is not a low level operation+ * that takes nanoseconds. It's not very CPU intensive, but it takes time. We strongly recommend limiting+ * the scope of the clicking as much as possible by using a specific selector that targets only the elements+ * that you assume or know will produce a navigation. You can certainly click everything by using+ * the `*` selector, but be prepared to wait minutes to get results on a large and complex page.+ *+ * **Example usage**+ *+ * ```javascript+ * const Apify = require('apify');+ *+ * const browser = await Apify.launchPuppeteer();+ * const page = await browser.goto('https://www.example.com');+ * const requestQueue = await Apify.openRequestQueue();+ *+ * await Apify.utils.enqueueLinksByClickingElements({+ * page,+ * requestQueue,+ * selector: 'a.product-detail',+ * pseudoUrls: [+ * 'https://www.example.com/handbags/[.*]'+ * 'https://www.example.com/purses/[.*]'+ * ],+ * });+ * ```+ *+ * @param {Object} options+ * All `enqueueLinksByClickingElements()` parameters are passed+ * via an options object with the following keys:+ * @param {Page} options.page+ * Puppeteer <a href="https://pptr.dev/#?product=Puppeteer&show=api-class-page" target="_blank"><code>Page</code></a> object.+ * @param {RequestQueue} options.requestQueue+ * A request queue to which the URLs will be enqueued.+ * @param {String} options.selector+ * A CSS selector matching elements to be clicked on. Unlike in [`enqueueLinks()`](utils#utils.enqueueLinks), there is no default+ * value. This is to prevent suboptimal use of this function by using it too broadly.+ * @param {Object[]|String[]} [options.pseudoUrls]+ * An array of {@link PseudoUrl}s matching the URLs to be enqueued,+ * or an array of strings or RegExps or plain Objects from which the {@link PseudoUrl}s can be constructed.+ *+ * The plain objects must include at least the `purl` property, which holds the pseudo-URL string or RegExp.+ * All remaining keys will be used as the `requestTemplate` argument of the {@link PseudoUrl} constructor,+ * which lets you specify special properties for the enqueued {@link Request} objects.+ *+ * If `pseudoUrls` is an empty array, `null` or `undefined`, then the function+ * enqueues all links found on the page.+ * @param {Object} [options.userData]+ * An object that will be merged with the new {@link Request}'s `userData`, overriding any values that+ * were set via templating from `pseudoUrls`. This is useful when you need to override generic+ * `userData` set by the {@link PseudoUrl} template in specific use cases.+ *+ * **Example:**+ * ```+ * // pseudoUrl.userData+ * {+ * name: 'John',+ * surname: 'Doe',+ * }+ * ```+ * ```+ * // userData+ * {+ * name: 'Albert',+ * age: 31+ * }+ * ```+ * ```+ * // Enqueued request.userData+ * {+ * name: 'Albert',+ * surname: 'Doe',+ * age: 31,+ * }+ * ```+ * @return {Promise<QueueOperationInfo[]>}+ * Promise that resolves to an array of {@link QueueOperationInfo} objects.+ * @memberOf puppeteer+ * @name enqueueLinksByClickingElements+ */+export async function enqueueLinksByClickingElements(options = {}) {+ const {+ page,+ requestQueue,+ selector,+ pseudoUrls,+ userData = {},+ } = options;++ checkParamOrThrow(page, 'page', 'Object');+ checkParamOrThrow(selector, 'selector', 'String');+ checkParamPrototypeOrThrow(requestQueue, 'requestQueue', [RequestQueue, RequestQueueLocal], 'Apify.RequestQueue');+ checkParamOrThrow(pseudoUrls, 'pseudoUrls', 'Maybe Array');+ checkParamOrThrow(userData, 'userData', 'Object');++ const pseudoUrlInstances = constructPseudoUrlInstances(pseudoUrls || []);+ const interceptedRequests = await clickElementsAndInterceptNavigationRequests(page, selector);+ const requests = createRequests(interceptedRequests, pseudoUrlInstances, userData);+ return addRequestsToQueueInBatches(requests, requestQueue);+}++/**+ * Clicks all elements of given page matching given selector.+ * Catches and intercepts all initiated navigation requests and opened pages.+ * Returns a list of all target URLs.+ *+ * @param {Page} page+ * @param {string} selector+ * @return {Promise<Object[]>}+ * @ignore+ */+async function clickElementsAndInterceptNavigationRequests(page, selector) {+ const uniqueRequests = new Map();+ const browser = page.browser();++ const onInterceptedRequest = createInterceptRequestHandler(page, uniqueRequests);+ const onTargetCreated = createTargetCreatedHandler(page, uniqueRequests);++ await addInterceptRequestHandler(page, onInterceptedRequest);+ browser.on('targetcreated', onTargetCreated);++ await clickElements(page, selector);++ browser.removeListener('targetcreated', onTargetCreated);+ await removeInterceptRequestHandler(page, onInterceptedRequest);++ return Array.from(uniqueRequests.values());+}++/**+ * @param {Page} page+ * @param {Map} requests+ * @return {Function}+ */+function createInterceptRequestHandler(page, requests) {+ return function onInterceptedRequest(req) {+ if (!isTopFrameNavigationRequest(page, req)) return req.continue();+ const url = req.url();+ requests.set(url, {+ url,+ headers: req.headers(),+ method: req.method(),+ payload: req.postData(),+ });+ req.respond(req.redirectChain().length+ ? { body: '' } // Prevents 301/302 redirect+ : { status: 204 }); // Prevents navigation by js+ };+}++/**+ * @param {Page} page+ * @param {Request} req+ * @return {boolean}+ */+function isTopFrameNavigationRequest(page, req) {+ return req.isNavigationRequest()+ && req.frame() === page.mainFrame()+ && req.url() !== page.url();+}++/**+ * @param {Page} page+ * @param {Map} requests+ * @return {Function}+ */+function createTargetCreatedHandler(page, requests) {+ return async function onTargetCreated(target) {+ if (target.type() !== 'page') return;+ if (page.target() !== target.opener()) return;+ const newPage = await target.page();+ const url = newPage.url();+ requests.set(url, { url });+ page.close().catch((err) => {+ log.debug('enqueueLinksByClickingElements: Could not close spawned page.', { stack: err.stack });+ });+ };+}++/**+ * Click all elements matching the given selector. To be able to do this using+ * Puppeteer's `.click()` we need to make sure the elements are reachable by mouse,+ * so we first move them to the top of the page's stacking context and then click.+ * We do all in series to prevent elements from hiding one another. Therefore,+ * for large element sets, this will take considerable amount of time.+ *+ * @param {Page} page+ * @param {string} selector+ * @return {Promise}+ */+async function clickElements(page, selector) {+ const elementHandles = await page.$$(selector);+ log.debug(`enqueueLinksByClickingElements: There are ${elementHandles.length} elements to click.`);+ let clickedElementsCount = 0;+ let zIndex = STARTING_Z_INDEX;+ for (const handle of elementHandles) {+ try {+ await page.evaluate((el, zIndex) => { // eslint-disable-line no-shadow+ el.style.visiblity = 'visible';+ el.style.display = 'block';+ el.style.position = 'absolute';+ el.style.zindex = zIndex;+ }, handle, zIndex++);+ await handle.click();+ clickedElementsCount++;+ } catch (err) {+ log.debug('enqueueLinksByClickingElements: Click failed.', { stack: err.stack });+ }+ }+ log.debug(`enqueueLinksByClickingElements: Successfully clicked ${clickedElementsCount} elements out of ${elementHandles.length}`);+}diff --git a/src/enqueue_links.js b/src/enqueue_links/enqueue_links.jssimilarity index 65%rename from src/enqueue_links.jsrename to src/enqueue_links/enqueue_links.jsindex fe586b42a..c37770ba4 100644--- a/src/enqueue_links.js+++ b/src/enqueue_links/enqueue_links.js@@ -1,110 +1,21 @@import { URL } from 'url';-import _ from 'underscore';import log from 'apify-shared/log';import { checkParamOrThrow } from 'apify-client/build/utils';import { checkParamPrototypeOrThrow } from 'apify-shared/utilities';-import { RequestQueue, RequestQueueLocal } from './request_queue';-import Request from './request';-import PseudoUrl from './pseudo_url';--/**- * To enable direct use of the Actor UI `pseudoUrls` output while keeping high performance,- * all the pseudoUrls from the output are only constructed once and kept in a cache- * by the `enqueueLinks()` function.- * @ignore- */-const enqueueLinksCache = new Map();-export const MAX_ENQUEUE_LINKS_CACHE_SIZE = 1000;--/**- * Helper factory used in the `enqueueLinks()` function.- * @param {string[]|Object[]} pseudoUrls- * @returns {Array}- * @ignore- */-export const constructPseudoUrlInstances = (pseudoUrls) => {- return pseudoUrls.map((item, idx) => {- // Get pseudoUrl instance from cache.- let pUrl = enqueueLinksCache.get(item);- if (pUrl) return pUrl;- // Nothing in cache, make a new instance.- checkParamOrThrow(item, `pseudoUrls[${idx}]`, 'RegExp|Object|String');-- // If it's already a PseudoURL, just save it.- if (item instanceof PseudoUrl) pUrl = item;- // If it's a string or RegExp, construct a PURL from it directly.- else if (typeof item === 'string' || item instanceof RegExp) pUrl = new PseudoUrl(item);- // If it's an object, look for a purl property and use it and the rest to construct a PURL with a Request template.- else pUrl = new PseudoUrl(item.purl, _.omit(item, 'purl'));-- // Manage cache- enqueueLinksCache.set(item, pUrl);- if (enqueueLinksCache.size > MAX_ENQUEUE_LINKS_CACHE_SIZE) {- const key = enqueueLinksCache.keys().next().value;- enqueueLinksCache.delete(key);- }- return pUrl;- });-};--/**- * Extracts URLs from a given Puppeteer Page.- *- * @param {Page} page- * @param {string} selector- * @returns {string[]}- * @ignore- */-export const extractUrlsFromPage = async (page, selector) => {- /* istanbul ignore next */- return page.$$eval(selector, linkEls => linkEls.map(link => link.href).filter(href => !!href));-};--/**- * Extracts URLs from a given Cheerio object.- *- * @param {Function} $- * @param {string} selector- * @param {string} baseUrl- * @returns {string[]}- * @ignore- */-export const extractUrlsFromCheerio = ($, selector, baseUrl) => {- return $(selector)- .map((i, el) => $(el).attr('href'))- .get()- .filter(href => !!href)- .map((href) => {- // Throw a meaningful error when only a relative URL would be extracted instead of waiting for the Request to fail later.- const isHrefAbsolute = /^[a-z][a-z0-9+.-]*:/.test(href); // Grabbed this in 'is-absolute-url' package.- if (!isHrefAbsolute && !baseUrl) {- throw new Error(`An extracted URL: ${href} is relative and options.baseUrl is not set. `- + 'Use options.baseUrl in utils.enqueueLinks() to automatically resolve relative URLs.');- }- return baseUrl- ? (new URL(href, baseUrl)).href- : href;- });-};--/**- * Remove with 1.0.0- * @ignore- * @todo deprecate- */-let logDeprecationWarning = true;+import { RequestQueue, RequestQueueLocal } from '../request_queue';+import { constructPseudoUrlInstances, createRequests, addRequestsToQueueInBatches } from './shared';/*** The function finds elements matching a specific CSS selector (HTML anchor (`<a>`) by default)* either in a Puppeteer page, or in a Cheerio object (parsed HTML),- * and enqueues the corresponding links to the provided {@link RequestQueue}.+ * and enqueues the URLs in their `href` attributes to the provided {@link RequestQueue}.+ * If you're looking to find URLs in JavaScript heavy pages where links are not available+ * in `href` elements, but rather navigations are triggered in click handlers+ * see [`enqueueLinksByClickingElements()`](puppeteer#puppeteer.enqueueLinksByClickingElements).+ ** Optionally, the function allows you to filter the target links' URLs using an array of {@link PseudoUrl} objects* and override settings of the enqueued {@link Request} objects.*- * *IMPORTANT*: This is a work in progress. Currently the function only supports elements with- * `href` attribute pointing to a URL. However, in the future the function will also support- * JavaScript links, buttons and form submissions when used with a Puppeteer Page.- ** **Example usage**** ```javascript@@ -146,7 +57,7 @@ let logDeprecationWarning = true;* or an array of strings or RegExps or plain Objects from which the {@link PseudoUrl}s can be constructed.** The plain objects must include at least the `purl` property, which holds the pseudo-URL string or RegExp.- * All remaining keys will be used as the `requestTemplate` argument of the {@link PseudoUrl} constructor.+ * All remaining keys will be used as the `requestTemplate` argument of the {@link PseudoUrl} constructor,* which lets you specify special properties for the enqueued {@link Request} objects.** If `pseudoUrls` is an empty array, `null` or `undefined`, then the function@@ -184,7 +95,7 @@ let logDeprecationWarning = true;* @memberOf utils* @name enqueueLinks*/-export const enqueueLinks = async (...args) => {+export async function enqueueLinks(...args) {// TODO: Remove after v1.0.0 gets released.// Refactor enqueueLinks to use an options object and keep backwards compatibilitylet page, $, selector, requestQueue, baseUrl, pseudoUrls, userData; // eslint-disable-line@@ -192,11 +103,8 @@ export const enqueueLinks = async (...args) => {[{ page, $, selector = 'a', requestQueue, baseUrl, pseudoUrls, userData = {} }] = args;} else {[page, selector = 'a', requestQueue, pseudoUrls, userData = {}] = args;- if (logDeprecationWarning) {- log.warning('Passing individual arguments to enqueueLinks() is deprecated. '+ log.deprecated('Passing individual arguments to enqueueLinks() is deprecated. '+ 'Use an options object: enqueueLinks({ page, selector, requestQueue, pseudoUrls, userData }) instead.');- logDeprecationWarning = false;- }}// Check for pseudoUrls as a third parameter.@@ -219,29 +127,52 @@ export const enqueueLinks = async (...args) => {checkParamOrThrow(baseUrl, 'baseUrl', 'Maybe String');if (baseUrl && page) log.warning('The parameter options.baseUrl can only be used when parsing a Cheerio object. It will be ignored.');checkParamOrThrow(pseudoUrls, 'pseudoUrls', 'Maybe Array');- checkParamOrThrow(userData, 'userData', 'Maybe Object');+ checkParamOrThrow(userData, 'userData', 'Object');// Construct pseudoUrls from input where necessary.const pseudoUrlInstances = constructPseudoUrlInstances(pseudoUrls || []);const urls = page ? await extractUrlsFromPage(page, selector) : extractUrlsFromCheerio($, selector, baseUrl);- let requests = [];+ const requests = createRequests(urls, pseudoUrlInstances, userData);+ return addRequestsToQueueInBatches(requests, requestQueue);+}- if (pseudoUrlInstances.length) {- urls.forEach((url) => {- pseudoUrlInstances- .filter(purl => purl.matches(url))- .forEach(purl => requests.push(purl.createRequest(url)));- });- } else {- requests = urls.map(url => new Request({ url }));- }+/**+ * Extracts URLs from a given Puppeteer Page.+ *+ * @param {Page} page+ * @param {string} selector+ * @return {string[]}+ * @ignore+ */+export async function extractUrlsFromPage(page, selector) {+ /* istanbul ignore next */+ return page.$$eval(selector, linkEls => linkEls.map(link => link.href).filter(href => !!href));+}- const queueOperationInfos = [];- for (const request of requests) {- // Inject custom userData- Object.assign(request.userData, userData);- queueOperationInfos.push(await requestQueue.addRequest(request));- }- return queueOperationInfos;-};+/**+ * Extracts URLs from a given Cheerio object.+ *+ * @param {Function} $+ * @param {string} selector+ * @param {string} baseUrl+ * @return {string[]}+ * @ignore+ */+export function extractUrlsFromCheerio($, selector, baseUrl) {+ return $(selector)+ .map((i, el) => $(el).attr('href'))+ .get()+ .filter(href => !!href)+ .map((href) => {+ // Throw a meaningful error when only a relative URL would be extracted instead of waiting for the Request to fail later.+ const isHrefAbsolute = /^[a-z][a-z0-9+.-]*:/.test(href); // Grabbed this in 'is-absolute-url' package.+ if (!isHrefAbsolute && !baseUrl) {+ throw new Error(`An extracted URL: ${href} is relative and options.baseUrl is not set. `+ + 'Use options.baseUrl in utils.enqueueLinks() to automatically resolve relative URLs.');+ }+ return baseUrl+ ? (new URL(href, baseUrl)).href+ : href;+ });+}diff --git a/src/enqueue_links/shared.js b/src/enqueue_links/shared.jsnew file mode 100644index 000000000..987e82c79--- /dev/null+++ b/src/enqueue_links/shared.js@@ -0,0 +1,90 @@+import _ from 'underscore';+import { checkParamOrThrow } from 'apify-client/build/utils';+import PseudoUrl from '../pseudo_url';+import Request from '../request';++const MAX_ENQUEUE_LINKS_CACHE_SIZE = 1000;++/**+ * To enable direct use of the Actor UI `pseudoUrls` output while keeping high performance,+ * all the pseudoUrls from the output are only constructed once and kept in a cache+ * by the `enqueueLinks()` function.+ * @ignore+ */+const enqueueLinksPseudoUrlCache = new Map();++/**+ * Helper factory used in the `enqueueLinks()` and enqueueLinksByClickingElements() function.+ * @param {string[]|Object[]} pseudoUrls+ * @return {PseudoUrl[]}+ * @ignore+ */+export function constructPseudoUrlInstances(pseudoUrls) {+ return pseudoUrls.map((item, idx) => {+ // Get pseudoUrl instance from cache.+ let pUrl = enqueueLinksPseudoUrlCache.get(item);+ if (pUrl) return pUrl;+ // Nothing in cache, make a new instance.+ checkParamOrThrow(item, `pseudoUrls[${idx}]`, 'RegExp|Object|String');++ // If it's already a PseudoURL, just save it.+ if (item instanceof PseudoUrl) pUrl = item;+ // If it's a string or RegExp, construct a PURL from it directly.+ else if (typeof item === 'string' || item instanceof RegExp) pUrl = new PseudoUrl(item);+ // If it's an object, look for a purl property and use it and the rest to construct a PURL with a Request template.+ else pUrl = new PseudoUrl(item.purl, _.omit(item, 'purl'));++ // Manage cache+ enqueueLinksPseudoUrlCache.set(item, pUrl);+ if (enqueueLinksPseudoUrlCache.size > MAX_ENQUEUE_LINKS_CACHE_SIZE) {+ const key = enqueueLinksPseudoUrlCache.keys().next().value;+ enqueueLinksPseudoUrlCache.delete(key);+ }+ return pUrl;+ });+}+/**+ * @param {string[]|Object[]} sources+ * @param {PseudoUrl[]} pseudoUrls+ * @param {Object} userData+ * @return {Request[]}+ * @ignore+ */+export function createRequests(sources, pseudoUrls, userData) {+ const normalizedSources = sources.map((src) => {+ return typeof src === 'string'+ ? { url: src, userData }+ : { ...src, userData };+ });++ if (!(pseudoUrls && pseudoUrls.length)) {+ return normalizedSources.map(src => new Request(src));+ }++ const requests = [];+ normalizedSources.forEach((src) => {+ pseudoUrls+ .filter(purl => purl.matches(src.url))+ .forEach((purl) => {+ const request = purl.createRequest(src);+ requests.push(request);+ });+ });+ return requests;+}++/**+ * @param {Request[]} requests+ * @param {RequestQueue} requestQueue+ * @param {number} batchSize+ * @return {Promise<QueueOperationInfo[]>}+ * @ignore+ */+export async function addRequestsToQueueInBatches(requests, requestQueue, batchSize = 5) {+ const queueOperationInfos = [];+ for (const request of requests) {+ queueOperationInfos.push(requestQueue.addRequest(request));+ if (queueOperationInfos.length % batchSize === 0) await Promise.all(queueOperationInfos);+ }+ return Promise.all(queueOperationInfos);+}diff --git a/src/index.js b/src/index.jsindex cf4727dac..07b22d26d 100644--- a/src/index.js+++ b/src/index.js@@ -19,7 +19,7 @@ import { apifyClient, getMemoryInfo, isAtHome, publicUtils, logSystemInfo } fromimport { browse, launchWebDriver } from './webdriver';import { puppeteerUtils } from './puppeteer_utils';import { socialUtils } from './utils_social';-import { enqueueLinks } from './enqueue_links';+import { enqueueLinks } from './enqueue_links/enqueue_links';import PseudoUrl from './pseudo_url';import LiveViewServer from './live_view/live_view_server';diff --git a/src/pseudo_url.js b/src/pseudo_url.jsindex c1a314e5c..b945978bc 100644--- a/src/pseudo_url.js+++ b/src/pseudo_url.js@@ -134,13 +134,20 @@ class PseudoUrl {}/**- * Creates a Request object from a provided `requestTemplate` and given URL.+ * Creates a Request object from a provided `requestTemplate` and a given URL+ * or an object that specifies ${@link Request} properties. In case of a collision+ * the properties will override the template, except for `userData`, which will+ * be merged together, with the `userData` property having preference over the template.+ * This enables dynamic overriding of the template.*- * @param {String} url+ * @param {string|Object} urlOrProps* @return {Request}*/- createRequest(url) {- return new Request(Object.assign({ url }, this.requestTemplate));+ createRequest(urlOrProps) {+ const props = typeof urlOrProps === 'string' ? { url: urlOrProps } : urlOrProps;+ props.userData = { ...this.requestTemplate.userData, ...props.userData };+ const options = { ...this.requestTemplate, ...props }; // props.userData will override template with merged data.+ return new Request(options);}}diff --git a/src/puppeteer_utils.js b/src/puppeteer_utils.jsindex 4de3a0a89..46af3ded1 100644--- a/src/puppeteer_utils.js+++ b/src/puppeteer_utils.js@@ -8,7 +8,8 @@ import { checkParamPrototypeOrThrow } from 'apify-shared/utilities';import LruCache from 'apify-shared/lru_cache';import { RequestQueue, RequestQueueLocal } from './request_queue';import Request from './request';-import { enqueueLinks } from './enqueue_links';+import { enqueueLinks } from './enqueue_links/enqueue_links';+import { enqueueLinksByClickingElements } from './enqueue_links/click_elements';import { addInterceptRequestHandler, removeInterceptRequestHandler } from './puppeteer_request_interception';const jqueryPath = require.resolve('jquery/dist/jquery.min');@@ -467,6 +468,7 @@ export const puppeteerUtils = {return enqueueLinks(...args);}},+ enqueueLinksByClickingElements,blockRequests,blockResources,cacheResponses,diff --git a/test/enqueue_links/click_elements.js b/test/enqueue_links/click_elements.jsnew file mode 100644index 000000000..e69de29bbdiff --git a/test/enqueue_links.js b/test/enqueue_links/enqueue_links.jssimilarity index 98%rename from test/enqueue_links.jsrename to test/enqueue_links/enqueue_links.jsindex 3753a02fb..41e5cfeb7 100644--- a/test/enqueue_links.js+++ b/test/enqueue_links/enqueue_links.js@@ -1,8 +1,8 @@import cheerio from 'cheerio';import { expect } from 'chai';-import Apify from '../build/index';-import { enqueueLinks } from '../build/enqueue_links';-import { RequestQueue } from '../build/request_queue';+import Apify from '../../build';+import { enqueueLinks } from '../../build/enqueue_links/enqueue_links';+import { RequestQueue } from '../../build/request_queue';const { utils: { log } } = Apify;@@ -266,9 +266,9 @@ describe('enqueueLinks()', () => {new Apify.PseudoUrl('[http|https]://cool.com/', { userData: { foo: 'bar' } }),];- const originalLogWarning = log.warning;+ const originalLogDeprecated = log.deprecated;const logOutput = [];- log.warning = (item) => { logOutput.push(item); };+ log.deprecated = (item) => { logOutput.push(item); };try {await enqueueLinks(page, '.click', purls, queue, { hello: 'world' });@@ -287,7 +287,7 @@ describe('enqueueLinks()', () => {expect(enqueued[2].method).to.be.eql('GET');expect(enqueued[2].userData).to.be.eql({ hello: 'world', foo: 'bar' });} finally {- log.warning = originalLogWarning;+ log.deprecated = originalLogDeprecated;}expect(logOutput.length).to.be.eql(1);diff --git a/test/enqueue_links/shared.js b/test/enqueue_links/shared.jsnew file mode 100644index 000000000..ca31947c1--- /dev/null+++ b/test/enqueue_links/shared.js@@ -0,0 +1,80 @@+import { expect } from 'chai';+import PseudoUrl from '../../build/pseudo_url';+import * as shared from '../../build/enqueue_links/shared';++describe('Enqueue links shared functions', () => {+ describe('constructPseudoUrlInstances()', () => {+ it('should work', () => {+ const pseudoUrlSources = [+ new PseudoUrl(/^https?:\/\/example\.com/, { userData: { foo: 'bar' } }),+ /^https?:\/\/example\.com/,+ 'http[s?]://example.com/[.*]',+ { purl: 'http[s?]://example.com[.*]', userData: { foo: 'bar' } },+ ];+ const pseudoUrls = shared.constructPseudoUrlInstances(pseudoUrlSources);+ expect(pseudoUrls).to.have.lengthOf(4);+ pseudoUrls.forEach((purl) => {+ expect(purl.matches('https://example.com/foo')).to.be.eql(true);+ });+ let request = pseudoUrls[0].createRequest('https://example.com/foo');+ expect(request.userData).to.be.eql({ foo: 'bar' });+ request = pseudoUrls[3].createRequest('https://example.com/bar');+ expect(request.userData).to.be.eql({ foo: 'bar' });+ });++ it('should cache items', () => {+ const pseudoUrls = shared.constructPseudoUrlInstances(['http[s?]://example.com/[.*]']);+ const pseudoUrls2 = shared.constructPseudoUrlInstances(['http[s?]://example.com/[.*]']);+ expect(pseudoUrls[0] === pseudoUrls2[0]).to.be.eql(true);+ });+ });++ describe('createRequests()', () => {+ it('should work', () => {+ const sources = [+ 'http://example.com/foo',+ { url: 'https://example.com/bar', method: 'POST' },+ 'https://apify.com',+ ];+ const pseudoUrls = [+ new PseudoUrl('http[s?]://example.com/[.*]', { userData: { one: 1 } }),+ ];++ const userData = { bar: 'foo' };+ const requests = shared.createRequests(sources, pseudoUrls, userData);++ expect(requests).to.have.lengthOf(2);+ requests.forEach((r) => {+ expect(r.url).to.match(/^https?:\/\/example\.com\//);+ expect(r.userData).to.include({ bar: 'foo', one: 1 });+ });+ expect(requests[1].method).to.be.eql('POST');+ });+ });++ describe('addRequestsToQueueInBatches()', () => {+ it('should work', async () => {+ const fakeRequestQueue = {+ requests: [],+ async addRequest(request) {+ this.requests.push(request);+ },+ };++ const requests = Array(5).fill(null).map((_, i) => i);++ const finished = shared.addRequestsToQueueInBatches(requests, fakeRequestQueue, 2);++ // With batch size 2, two requests will be dispatched synchronously before the async function+ // returns and thus the following push should place 1000 on the third place in the array.+ fakeRequestQueue.requests.push(1000);+++ await finished;+ const results = fakeRequestQueue.requests;+ expect(results).to.have.lengthOf(6);+ expect(results[2]).to.be.eql(1000);+ expect(results.reduce((sum, num) => sum + num)).to.be.eql(1010);+ });+ });+});diff --git a/test/pseudo_url.js b/test/pseudo_url.jsindex 1a828ebe2..b75f9843f 100644--- a/test/pseudo_url.js+++ b/test/pseudo_url.js@@ -16,7 +16,7 @@ describe('Apify.PseudoUrl', () => {expect(purl.matches('http://www.example.com/pages/not@working')).to.be.eql(true);});- it('createRequest() should work', () => {+ it('createRequest() should work with a string', () => {const purl = new Apify.PseudoUrl('something', { method: 'POST', userData: { foo: 'bar' } });const request = purl.createRequest('http://example.com');@@ -25,4 +25,19 @@ describe('Apify.PseudoUrl', () => {expect(request.method).to.be.eql('POST');expect(request.userData).to.be.eql({ foo: 'bar' });});++ it('createRequest() should work with an object', () => {+ const purl = new Apify.PseudoUrl('something', { method: 'POST', userData: { foo: 'bar' } });+ const request = purl.createRequest({+ url: 'http://example.com',+ userData: {+ bar: 'foo',+ },+ });++ expect(request).to.be.an.instanceof(Apify.Request);+ expect(request.url).to.be.eql('http://example.com');+ expect(request.method).to.be.eql('POST');+ expect(request.userData).to.be.eql({ foo: 'bar', bar: 'foo' });+ });});