Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Add element clicking tests, resolve various edge cases
Ondra Urbanb847a2b4eb9f
diff --git a/src/enqueue_links/click_elements.js b/src/enqueue_links/click_elements.jsindex 4201592a5..b6410322d 100644--- a/src/enqueue_links/click_elements.js+++ b/src/enqueue_links/click_elements.js@@ -1,3 +1,4 @@+import { URL } from 'url';import { checkParamOrThrow } from 'apify-client/build/utils';import { checkParamPrototypeOrThrow } from 'apify-shared/utilities';import log from 'apify-shared/log';@@ -6,6 +7,8 @@ import { addInterceptRequestHandler, removeInterceptRequestHandler } from '../puimport { constructPseudoUrlInstances, createRequests, addRequestsToQueueInBatches } from './shared';const STARTING_Z_INDEX = 10000;+const DEFAULT_NETWORK_IDLE_INTERVAL_MILLIS = 25;+const DEFAULT_NETWORK_IDLE_TIMEOUT_MILLIS = 250;/*** The function finds elements matching a specific CSS selector in a Puppeteer page,@@ -133,19 +136,27 @@ export async function enqueueLinksByClickingElements(options = {}) {* @return {Promise<Object[]>}* @ignore*/-async function clickElementsAndInterceptNavigationRequests(page, selector) {+export async function clickElementsAndInterceptNavigationRequests(page, selector) {const uniqueRequests = new Map();const browser = page.browser();const onInterceptedRequest = createInterceptRequestHandler(page, uniqueRequests);const onTargetCreated = createTargetCreatedHandler(page, uniqueRequests);+ const onFrameNavigated = createFrameNavigatedHandler(page, uniqueRequests);await addInterceptRequestHandler(page, onInterceptedRequest);browser.on('targetcreated', onTargetCreated);+ page.on('framenavigated', onFrameNavigated);++ await preventHistoryNavigation(page);await clickElements(page, selector);+ await waitForNetworkIdle(page);++ await restoreHistoryNavigationAndSaveCapturedUrls(page, uniqueRequests);browser.removeListener('targetcreated', onTargetCreated);+ page.removeListener('framenavigated', onFrameNavigated);await removeInterceptRequestHandler(page, onInterceptedRequest);return Array.from(uniqueRequests.values());@@ -155,6 +166,7 @@ async function clickElementsAndInterceptNavigationRequests(page, selector) {* @param {Page} page* @param {Map} requests* @return {Function}+ * @ignore*/function createInterceptRequestHandler(page, requests) {return function onInterceptedRequest(req) {@@ -176,31 +188,74 @@ function createInterceptRequestHandler(page, requests) {* @param {Page} page* @param {Request} req* @return {boolean}+ * @ignore*/function isTopFrameNavigationRequest(page, req) {return req.isNavigationRequest()- && req.frame() === page.mainFrame()- && req.url() !== page.url();+ && req.frame() === page.mainFrame();}/*** @param {Page} page* @param {Map} requests* @return {Function}+ * @ignore*/function createTargetCreatedHandler(page, requests) {return async function onTargetCreated(target) {if (target.type() !== 'page') return;if (page.target() !== target.opener()) return;- const newPage = await target.page();- const url = newPage.url();+ const createdPage = await target.page();+ const url = createdPage.url();requests.set(url, { url });page.close().catch((err) => {- log.debug('enqueueLinksByClickingElements: Could not close spawned page.', { stack: err.stack });+ log.debug('enqueueLinksByClickingElements: Could not close spawned page.', { error: err.stack });});};}+/**+ * @param {Page} page+ * @param {Map} requests+ * @return {Function}+ * @ignore+ */+function createFrameNavigatedHandler(page, requests) {+ return function onFrameNavigated(frame) {+ if (frame !== page.mainFrame()) return;+ const url = frame.url();+ requests.set(url, { url });+ };+}++/**+ * @param {Page} page+ * @return {Promise}+ * @ignore+ */+async function preventHistoryNavigation(page) {+ /* istanbul ignore next */+ return page.evaluate(() => {+ /* global window */+ window.__originalHistory__ = window.history; // eslint-disable-line no-underscore-dangle+ delete window.history; // Simple override does not work.+ window.history = {+ stateHistory: [],+ length: 0,+ state: {},+ go() {},+ back() {},+ forward() {},+ pushState(...args) {+ this.stateHistory.push(args);+ },+ replaceState(...args) {+ this.stateHistory.push(args);+ },+ };+ });+}+/*** Click all elements matching the given selector. To be able to do this using* Puppeteer's `.click()` we need to make sure the elements are reachable by mouse,@@ -211,20 +266,16 @@ function createTargetCreatedHandler(page, requests) {* @param {Page} page* @param {string} selector* @return {Promise}+ * @ignore*/-async function clickElements(page, selector) {+export async function clickElements(page, selector) {const elementHandles = await page.$$(selector);log.debug(`enqueueLinksByClickingElements: There are ${elementHandles.length} elements to click.`);let clickedElementsCount = 0;let zIndex = STARTING_Z_INDEX;for (const handle of elementHandles) {try {- await page.evaluate((el, zIndex) => { // eslint-disable-line no-shadow- el.style.visiblity = 'visible';- el.style.display = 'block';- el.style.position = 'absolute';- el.style.zindex = zIndex;- }, handle, zIndex++);+ await page.evaluate(updateElementCssToEnableMouseClick, handle, zIndex++);await handle.click();clickedElementsCount++;} catch (err) {@@ -233,3 +284,96 @@ async function clickElements(page, selector) {}log.debug(`enqueueLinksByClickingElements: Successfully clicked ${clickedElementsCount} elements out of ${elementHandles.length}`);}++/* istanbul ignore next */+/**+ * This is an in browser function!+ * @param {Element} el+ * @param {number} zIndex+ */+function updateElementCssToEnableMouseClick(el, zIndex) {+ el.style.visibility = 'visible';+ el.style.display = 'block';+ el.style.position = 'absolute';+ el.style.zIndex = zIndex;+ const boundingRect = el.getBoundingClientRect();+ if (!boundingRect.height) el.style.height = '10px';+ if (!boundingRect.width) el.style.width = '10px';+}++/**+ * This function tracks whether any request events or frame navigations were emitted+ * in the past idleIntervalMillis and whenever the interval registers no activity,+ * the function returns.+ *+ * It will also return when a final timeout, represented by the timeoutMillis parameter+ * is reached, to prevent blocking on pages with constant network activity.+ *+ * We need this to make sure we don't finish too soon when intercepting requests triggered+ * by clicking in the page. They often get registered by the Node.js process only some+ * milliseconds after clicking and we would lose those requests. This is especially prevalent+ * when there's only a single element to click.+ *+ * @param {Page} page+ * @param {number} idleIntervalMillis+ * @param {number} timeoutMillis+ * @return {Promise}+ * @ignore+ */+async function waitForNetworkIdle(+ page,+ idleIntervalMillis = DEFAULT_NETWORK_IDLE_INTERVAL_MILLIS,+ timeoutMillis = DEFAULT_NETWORK_IDLE_TIMEOUT_MILLIS,+) {+ return new Promise((resolve) => {+ let timeout;+ let maxTimeout;++ function activityHandler() {+ clearTimeout(timeout);+ timeout = setTimeout(() => {+ clearTimeout(maxTimeout);+ resolve();+ page.removeListener('request', activityHandler);+ page.removeListener('framenavigated', activityHandler);+ }, idleIntervalMillis);+ }++ function maxTimeoutHandler() {+ log.debug(`enqueueLinksByClickingElements: Network still showed activity after ${timeoutMillis}ms. `+ + 'This is probably due to the page itself dispatching requests, but some links may also have been missed.');+ resolve();+ }++ maxTimeout = setTimeout(maxTimeoutHandler, timeoutMillis);+ timeout = activityHandler(); // We call this once manually in case there would be no requests at all.+ page.on('request', activityHandler);+ page.on('framenavigated', activityHandler);+ });+}++/**+ * @param {Page} page+ * @param {Map} requests+ * @return {Promise}+ * @ignore+ */+async function restoreHistoryNavigationAndSaveCapturedUrls(page, requests) {+ /* eslint-disable no-shadow */+ /* istanbul ignore next */+ const stateHistory = await page.evaluate(() => {+ /* global window */+ const { stateHistory } = window.history;+ window.history = window.__originalHistory__; // eslint-disable-line no-underscore-dangle+ return stateHistory;+ });+ stateHistory.forEach((args) => {+ try {+ const stateUrl = args[args.length - 1];+ const url = new URL(stateUrl, page.url()).href;+ requests.set(url, { url });+ } catch (err) {+ log.debug('enqueueLinksByClickingElements: Failed to ', { error: err.stack });+ }+ });+}diff --git a/src/enqueue_links/shared.js b/src/enqueue_links/shared.jsindex 987e82c79..8338b32d2 100644--- a/src/enqueue_links/shared.js+++ b/src/enqueue_links/shared.js@@ -1,3 +1,4 @@+import { URL } from 'url';import _ from 'underscore';import { checkParamOrThrow } from 'apify-client/build/utils';import PseudoUrl from '../pseudo_url';@@ -51,11 +52,19 @@ export function constructPseudoUrlInstances(pseudoUrls) {* @ignore*/export function createRequests(sources, pseudoUrls, userData) {- const normalizedSources = sources.map((src) => {- return typeof src === 'string'- ? { url: src, userData }- : { ...src, userData };- });+ const normalizedSources = sources+ .map((src) => {+ return typeof src === 'string'+ ? { url: src, userData }+ : { ...src, userData };+ })+ .filter(({ url }) => {+ try {+ return new URL(url).href;+ } catch (err) {+ return false;+ }+ });if (!(pseudoUrls && pseudoUrls.length)) {return normalizedSources.map(src => new Request(src));diff --git a/test/enqueue_links/click_elements.js b/test/enqueue_links/click_elements.jsindex e69de29bb..85239b36e 100644--- a/test/enqueue_links/click_elements.js+++ b/test/enqueue_links/click_elements.js@@ -0,0 +1,242 @@+import { expect } from 'chai';+import Apify from '../../build';+import { clickElements, clickElementsAndInterceptNavigationRequests } from '../../build/enqueue_links/click_elements';++/* global window */++describe('enqueueLinksByClickingElements()', () => {+ let browser;+ let page;++ before(async () => {+ browser = await Apify.launchPuppeteer({ headless: true });+ });++ after(async () => {+ await browser.close();+ });++ beforeEach(async () => {+ page = await browser.newPage();+ });++ afterEach(async () => {+ await page.close();+ });++ describe('clickElements()', () => {+ it('should click an element', async () => {+ const html = `+<html>+ <body>+ <div onclick="window.clicked = true;">div</div>+ </body>+</html>+ `;+ await page.setContent(html);+ await clickElements(page, 'div');+ const wasClicked = await page.evaluate(() => window.clicked);+ expect(wasClicked).to.be.eql(true);+ });++ it('should click an empty element', async () => {+ const html = `+<html>+ <body>+ <div onclick="window.clicked = true;"></div>+ </body>+</html>+ `;+ await page.setContent(html);+ await clickElements(page, 'div');+ const wasClicked = await page.evaluate(() => window.clicked);+ expect(wasClicked).to.be.eql(true);+ });+ it('should click multiple elements', async () => {+ const html = `+<html>+ <script>+ window.clickedElements = [];+ window.handleClick = (evt) => window.clickedElements.push(evt.target.nodeName);+ </script>+ <body>+ <header onclick="return window.handleClick(event)"></header>+ <div onclick="return window.handleClick(event)"></div>+ <p onclick="return window.handleClick(event)"></p>+ <footer onclick="return window.handleClick(event)"></footer>+ </body>+</html>+ `;+ await page.setContent(html);+ await clickElements(page, 'header, div, p, footer');+ const clickedElements = await page.evaluate(() => window.clickedElements);+ expect(clickedElements).to.be.eql(['HEADER', 'DIV', 'P', 'FOOTER']);+ });++ it('should click hidden elements', async () => {+ const html = `+<html>+ <body>+ <div onclick="window.clicked = true;" style="visibility: hidden; display: none"></div>+ </body>+</html>+ `;+ await page.setContent(html);+ await clickElements(page, 'div');+ const wasClicked = await page.evaluate(() => window.clicked);+ expect(wasClicked).to.be.eql(true);+ });++ it('should click elements stacked on top of each other', async () => {+ const html = `+<html>+ <script>+ window.clickedElements = [];+ window.handleClick = (evt) => window.clickedElements.push(evt.target.nodeName);+ </script>+ <body>+ <header onclick="return window.handleClick(event)" style="position: absolute; z-index: auto">header</header>+ <div onclick="return window.handleClick(event)" style="position: absolute; z-index: 1">div</div>+ <main onclick="return window.handleClick(event)" style="position: absolute; z-index: 2">main</main>+ <footer onclick="return window.handleClick(event)" style="position: absolute; z-index: 3">footer</footer>+ </body>+</html>+ `;+ await page.setContent(html);+ await clickElements(page, 'header, div, main, footer');+ const clickedElements = await page.evaluate(() => window.clickedElements);+ expect(clickedElements).to.be.eql(['HEADER', 'DIV', 'MAIN', 'FOOTER']);+ });+ });++ describe('clickElementsAndInterceptNavigationRequests()', () => {+ it('should intercept navigation by clicking a link', async () => {+ const html = `+<html>+ <body>+ <a href="https://example.com">link</div>+ </body>+</html>+ `;+ await page.setContent(html);+ console.time('x');+ const interceptedRequests = await clickElementsAndInterceptNavigationRequests(page, 'a');+ console.timeEnd('x');+ expect(interceptedRequests).to.have.lengthOf(1);+ expect(interceptedRequests[0].url).to.match(/https:\/\/example\.com\/?$/);+ expect(page.url()).to.be.eql('about:blank');++ });++ it('should intercept navigation with window.location', async () => {+ const html = `+<html>+ <body>+ <div onclick="return window.location = 'https://example.com'">div</div>+ </body>+</html>+ `;+ await page.goto('https://example.com');+ await page.setContent(html);+ const interceptedRequests = await clickElementsAndInterceptNavigationRequests(page, 'div');+ expect(interceptedRequests).to.have.lengthOf(1);+ expect(interceptedRequests[0].url).to.match(/https:\/\/example\.com\/?$/);+ const pageContent = await page.content();+ expect(pageContent).to.include('onclick="return window.location = ');+ });++ it('should save the hash when changing it with window.location', async () => {+ const html = `+<html>+ <body>+ <div onclick="return window.location = '#foo'">div</div>+ </body>+</html>+ `;+ await page.goto('https://example.com');+ await page.setContent(html);+ const interceptedRequests = await clickElementsAndInterceptNavigationRequests(page, 'div');+ expect(interceptedRequests).to.have.lengthOf(1);+ expect(interceptedRequests[0].url).be.eql('https://example.com/#foo');+ const pageContent = await page.content();+ expect(pageContent).to.include('onclick="return window.location = ');+ });++ it('should prevent reload from cache with window.reload()', async () => {+ const html = `+<html>+ <body>+ <div onclick="return window.location.reload()">div</div>+ </body>+</html>+ `;+ await page.goto('https://example.com');+ await page.setContent(html);+ const interceptedRequests = await clickElementsAndInterceptNavigationRequests(page, 'div');+ expect(interceptedRequests).to.have.lengthOf(1);+ expect(interceptedRequests[0].url).to.match(/https:\/\/example\.com\/?$/);+ const pageContent = await page.content();+ expect(pageContent).to.include('onclick="return window.location.reload()');+ });++ it('should prevent forced reload with window.reload(true)', async () => {+ const html = `+<html>+ <body>+ <div onclick="return window.location.reload(true)">div</div>+ </body>+</html>+ `;+ await page.goto('https://example.com');+ await page.setContent(html);+ const interceptedRequests = await clickElementsAndInterceptNavigationRequests(page, 'div');+ expect(interceptedRequests).to.have.lengthOf(1);+ expect(interceptedRequests[0].url).to.match(/https:\/\/example\.com\/?$/);+ const pageContent = await page.content();+ expect(pageContent).to.include('onclick="return window.location.reload(true)');+ });++ it('should prevent manipulation with window.history', async () => {+ const html = `+<html>+ <script>+ window.handleClick = () => {+ window.history.go(1)+ window.history.forward();+ window.history.go()+ window.history.back();+ window.history.go(-1)+ if (window.history.length !== 0) throw new Error('history.length is not 0')+ }+ </script>+ <body>+ <div onclick="return handleClick();">div</div>+ </body>+</html>+ `;+ await page.goto('https://example.com');+ await page.setContent(html);+ const interceptedRequests = await clickElementsAndInterceptNavigationRequests(page, 'div');+ expect(interceptedRequests).to.have.lengthOf(0);+ const pageContent = await page.content();+ expect(pageContent).to.include('onclick="return handleClick();');+ });++ it('should save urls pushed to window.history', async () => {+ const html = `+<html>+ <body>+ <div onclick="return window.history.pushState({}, '', 'foo');">div</div>+ </body>+</html>+ `;+ await page.goto('https://example.com/bar/');+ await page.setContent(html);+ const interceptedRequests = await clickElementsAndInterceptNavigationRequests(page, 'div');+ expect(interceptedRequests).to.have.lengthOf(1);+ expect(interceptedRequests[0].url).to.be.eql('https://example.com/bar/foo');+ const pageContent = await page.content();+ expect(pageContent).to.include('onclick="return window.history.pushState');+ });+ });+});