Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Implement and test clicking that opens new tabs
Ondra Urbanb857fc01f86c
diff --git a/src/enqueue_links/click_elements.js b/src/enqueue_links/click_elements.jsindex b6410322d..4ff76b3c4 100644--- a/src/enqueue_links/click_elements.js+++ b/src/enqueue_links/click_elements.js@@ -7,8 +7,6 @@ import { addInterceptRequestHandler, removeInterceptRequestHandler } from '../puimport { constructPseudoUrlInstances, createRequests, addRequestsToQueueInBatches } from './shared';const STARTING_Z_INDEX = 10000;-const DEFAULT_NETWORK_IDLE_INTERVAL_MILLIS = 25;-const DEFAULT_NETWORK_IDLE_TIMEOUT_MILLIS = 250;/*** The function finds elements matching a specific CSS selector in a Puppeteer page,@@ -100,6 +98,23 @@ const DEFAULT_NETWORK_IDLE_TIMEOUT_MILLIS = 250;* age: 31,* }* ```+ * @param {Function} [options.waitForPageIdleSecs=1]+ * Clicking in the page triggers various asynchronous operations that lead to new URLs being shown+ * by the browser. It could be a simple JavaScript redirect or opening of a new tab in the browser.+ * These events often happen only some time after the actual click. Requests typically take milliseconds+ * while new tabs open in hundreds of milliseconds.+ *+ * To be able to capture all those events, the `enqueueLinksByClickingElements()` function repeatedly waits+ * for the `waitForPageIdleSecs`. By repeatedly we mean that whenever a relevant event is triggered, the timer+ * is restarted. As long as new events keep coming, the function will not return, unless+ * the below `maxWaitForPageIdleSecs` timeout is reached.+ *+ * You may want to reduce this for example when you're sure that your clicks do not open new tabs,+ * or increase when you're not getting all the expected URLs.+ * @param {Function} [options.maxWaitForPageIdleSecs=5]+ * This is the maximum period for which the function will keep tracking events, even if more events keep coming.+ * Its purpose is to prevent a deadlock in the page by periodic events, often unrelated to the clicking itself.+ * See `waitForPageIdleSecs` above for an explanation.* @return {Promise<QueueOperationInfo[]>}* Promise that resolves to an array of {@link QueueOperationInfo} objects.* @memberOf puppeteer@@ -112,6 +127,8 @@ export async function enqueueLinksByClickingElements(options = {}) {selector,pseudoUrls,userData = {},+ waitForPageIdleSecs = 1,+ maxWaitForPageIdleSecs = 5,} = options;checkParamOrThrow(page, 'page', 'Object');@@ -119,9 +136,18 @@ export async function enqueueLinksByClickingElements(options = {}) {checkParamPrototypeOrThrow(requestQueue, 'requestQueue', [RequestQueue, RequestQueueLocal], 'Apify.RequestQueue');checkParamOrThrow(pseudoUrls, 'pseudoUrls', 'Maybe Array');checkParamOrThrow(userData, 'userData', 'Object');+ checkParamOrThrow(userData, 'clickingFunction', 'Function');++ const waitForPageIdleMillis = waitForPageIdleSecs * 1000;+ const maxWaitForPageIdleMillis = maxWaitForPageIdleSecs * 1000;const pseudoUrlInstances = constructPseudoUrlInstances(pseudoUrls || []);- const interceptedRequests = await clickElementsAndInterceptNavigationRequests(page, selector);+ const interceptedRequests = await clickElementsAndInterceptNavigationRequests({+ page,+ selector,+ waitForPageIdleMillis,+ maxWaitForPageIdleMillis,+ });const requests = createRequests(interceptedRequests, pseudoUrlInstances, userData);return addRequestsToQueueInBatches(requests, requestQueue);}@@ -131,12 +157,20 @@ export async function enqueueLinksByClickingElements(options = {}) {* Catches and intercepts all initiated navigation requests and opened pages.* Returns a list of all target URLs.*- * @param {Page} page- * @param {string} selector+ * @param {Object} options+ * @param {Page} options.page+ * @param {string} options.selector* @return {Promise<Object[]>}* @ignore*/-export async function clickElementsAndInterceptNavigationRequests(page, selector) {+export async function clickElementsAndInterceptNavigationRequests(options) {+ const {+ page,+ selector,+ waitForPageIdleMillis,+ maxWaitForPageIdleMillis,+ } = options;+const uniqueRequests = new Map();const browser = page.browser();@@ -151,7 +185,7 @@ export async function clickElementsAndInterceptNavigationRequests(page, selectorawait preventHistoryNavigation(page);await clickElements(page, selector);- await waitForNetworkIdle(page);+ await waitForPageIdle({ page, waitForPageIdleMillis, maxWaitForPageIdleMillis });await restoreHistoryNavigationAndSaveCapturedUrls(page, uniqueRequests);@@ -203,17 +237,33 @@ function isTopFrameNavigationRequest(page, req) {*/function createTargetCreatedHandler(page, requests) {return async function onTargetCreated(target) {- if (target.type() !== 'page') return;- if (page.target() !== target.opener()) return;- const createdPage = await target.page();- const url = createdPage.url();+ if (!isTargetRelevant(page, target)) return;+ const url = target.url();requests.set(url, { url });- page.close().catch((err) => {++ // We want to close the page but don't care about+ // possible errors like target closed.+ try {+ const createdPage = await target.page();+ await createdPage.close();+ } catch (err) {log.debug('enqueueLinksByClickingElements: Could not close spawned page.', { error: err.stack });- });+ }};}+/**+ * We're only interested in pages created by the page we're currently clicking in.+ * There will generally be a lot of other targets being created in the browser.+ * @param {Page} page+ * @param {Target} target+ * @return {boolean}+ */+export function isTargetRelevant(page, target) {+ return target.type() === 'page'+ && page.target() === target.opener();+}+/*** @param {Page} page* @param {Map} requests@@ -302,7 +352,7 @@ function updateElementCssToEnableMouseClick(el, zIndex) {}/**- * This function tracks whether any request events or frame navigations were emitted+ * This function tracks whether any requests, frame navigations or targets were emitted* in the past idleIntervalMillis and whenever the interval registers no activity,* the function returns.*@@ -314,41 +364,48 @@ function updateElementCssToEnableMouseClick(el, zIndex) {* milliseconds after clicking and we would lose those requests. This is especially prevalent* when there's only a single element to click.*- * @param {Page} page- * @param {number} idleIntervalMillis- * @param {number} timeoutMillis+ * @param {Object} options+ * @param {Page} options.page+ * @param {number} options.waitForPageIdleMillis+ * @param {number} options.maxWaitForPageIdleMillis* @return {Promise}* @ignore*/-async function waitForNetworkIdle(- page,- idleIntervalMillis = DEFAULT_NETWORK_IDLE_INTERVAL_MILLIS,- timeoutMillis = DEFAULT_NETWORK_IDLE_TIMEOUT_MILLIS,-) {+async function waitForPageIdle({ page, waitForPageIdleMillis, maxWaitForPageIdleMillis }) {return new Promise((resolve) => {let timeout;let maxTimeout;+ function newTabTracker(target) {+ if (isTargetRelevant(page, target)) activityHandler();+ }+function activityHandler() {clearTimeout(timeout);timeout = setTimeout(() => {clearTimeout(maxTimeout);- resolve();- page.removeListener('request', activityHandler);- page.removeListener('framenavigated', activityHandler);- }, idleIntervalMillis);+ finish();+ }, waitForPageIdleMillis);}function maxTimeoutHandler() {- log.debug(`enqueueLinksByClickingElements: Network still showed activity after ${timeoutMillis}ms. `- + 'This is probably due to the page itself dispatching requests, but some links may also have been missed.');+ log.debug(`enqueueLinksByClickingElements: Page still showed activity after ${timeoutMillis}ms. `+ + 'This is probably due to the website itself dispatching requests, but some links may also have been missed.');+ finish();+ }++ function finish() {+ page.removeListener('request', activityHandler);+ page.removeListener('framenavigated', activityHandler);+ page.removeListener('targetcreated', newTabTracker);resolve();}- maxTimeout = setTimeout(maxTimeoutHandler, timeoutMillis);+ maxTimeout = setTimeout(maxTimeoutHandler, maxWaitForPageIdleMillis);timeout = activityHandler(); // We call this once manually in case there would be no requests at all.page.on('request', activityHandler);page.on('framenavigated', activityHandler);+ page.on('targetcreated', newTabTracker);});}diff --git a/test/enqueue_links/click_elements.js b/test/enqueue_links/click_elements.jsindex 1188ec344..8909359da 100644--- a/test/enqueue_links/click_elements.js+++ b/test/enqueue_links/click_elements.js@@ -1,6 +1,6 @@import { expect } from 'chai';import Apify from '../../build';-import { clickElements, clickElementsAndInterceptNavigationRequests } from '../../build/enqueue_links/click_elements';+import { clickElements, clickElementsAndInterceptNavigationRequests, isTargetRelevant } from '../../build/enqueue_links/click_elements';/* global window */@@ -110,6 +110,16 @@ describe('enqueueLinksByClickingElements()', () => {});describe('clickElementsAndInterceptNavigationRequests()', () => {+ function getOpts(overrides = {}) {+ return {+ page,+ selector: 'div',+ waitForPageIdleMillis: 25,+ maxWaitForPageIdleMillis: 250,+ ...overrides,+ };+ }+it('should intercept navigation by clicking a link', async () => {const html = `<html>@@ -120,7 +130,9 @@ describe('enqueueLinksByClickingElements()', () => {`;await page.setContent(html);console.time('x');- const interceptedRequests = await clickElementsAndInterceptNavigationRequests(page, 'a');+ const interceptedRequests = await clickElementsAndInterceptNavigationRequests(getOpts({+ selector: 'a',+ }));console.timeEnd('x');expect(interceptedRequests).to.have.lengthOf(1);expect(interceptedRequests[0].url).to.match(/https:\/\/example\.com\/?$/);@@ -137,7 +149,7 @@ describe('enqueueLinksByClickingElements()', () => {`;await page.goto('https://example.com');await page.setContent(html);- const interceptedRequests = await clickElementsAndInterceptNavigationRequests(page, 'div');+ const interceptedRequests = await clickElementsAndInterceptNavigationRequests(getOpts());expect(interceptedRequests).to.have.lengthOf(1);expect(interceptedRequests[0].url).to.match(/https:\/\/example\.com\/?$/);const pageContent = await page.content();@@ -154,7 +166,7 @@ describe('enqueueLinksByClickingElements()', () => {`;await page.goto('https://example.com');await page.setContent(html);- const interceptedRequests = await clickElementsAndInterceptNavigationRequests(page, 'div');+ const interceptedRequests = await clickElementsAndInterceptNavigationRequests(getOpts());expect(interceptedRequests).to.have.lengthOf(1);expect(interceptedRequests[0].url).be.eql('https://example.com/#foo');const pageContent = await page.content();@@ -171,7 +183,7 @@ describe('enqueueLinksByClickingElements()', () => {`;await page.goto('https://example.com');await page.setContent(html);- const interceptedRequests = await clickElementsAndInterceptNavigationRequests(page, 'div');+ const interceptedRequests = await clickElementsAndInterceptNavigationRequests(getOpts());expect(interceptedRequests).to.have.lengthOf(1);expect(interceptedRequests[0].url).to.match(/https:\/\/example\.com\/?$/);const pageContent = await page.content();@@ -188,7 +200,7 @@ describe('enqueueLinksByClickingElements()', () => {`;await page.goto('https://example.com');await page.setContent(html);- const interceptedRequests = await clickElementsAndInterceptNavigationRequests(page, 'div');+ const interceptedRequests = await clickElementsAndInterceptNavigationRequests(getOpts());expect(interceptedRequests).to.have.lengthOf(1);expect(interceptedRequests[0].url).to.match(/https:\/\/example\.com\/?$/);const pageContent = await page.content();@@ -215,7 +227,7 @@ describe('enqueueLinksByClickingElements()', () => {`;await page.goto('https://example.com');await page.setContent(html);- const interceptedRequests = await clickElementsAndInterceptNavigationRequests(page, 'div');+ const interceptedRequests = await clickElementsAndInterceptNavigationRequests(getOpts());expect(interceptedRequests).to.have.lengthOf(0);const pageContent = await page.content();expect(pageContent).to.include('onclick="return handleClick();');@@ -231,11 +243,66 @@ describe('enqueueLinksByClickingElements()', () => {`;await page.goto('https://example.com/bar/');await page.setContent(html);- const interceptedRequests = await clickElementsAndInterceptNavigationRequests(page, 'div');+ const interceptedRequests = await clickElementsAndInterceptNavigationRequests(getOpts());expect(interceptedRequests).to.have.lengthOf(1);expect(interceptedRequests[0].url).to.be.eql('https://example.com/bar/foo');const pageContent = await page.content();expect(pageContent).to.include('onclick="return window.history.pushState');});++ it('should save urls from newly opened tabs', async () => {+ const html = `+<html>+ <body>+ <div onclick="return window.open('https://example.com');">div</div>+ </body>+</html>+ `;+ await page.setContent(html);+ const interceptedRequests = await clickElementsAndInterceptNavigationRequests(getOpts({+ waitForPageIdleMillis: 1000,+ maxWaitForPageIdleMillis: 5000,+ }));+ await new Promise(r => setTimeout(r, 1000));+ expect(interceptedRequests).to.have.lengthOf(1);+ expect(interceptedRequests[0].url).to.be.eql('https://example.com/');+ const pageContent = await page.content();+ expect(pageContent).to.include('onclick="return window.open(');+ });++ it('should close newly opened tabs', async () => {+ const html = `+<html>+ <body>+ <div onclick="return window.open('https://example.com');">div</div>+ </body>+</html>+ `;+ await page.setContent(html);+ const callCounts = await new Promise(async (resolve) => {+ let spawnedTarget;+ const counts = {+ create: 0,+ destroy: 0,+ };+ browser.on('targetcreated', (target) => {+ counts.create++;+ if (isTargetRelevant(page, target)) spawnedTarget = target;+ });+ browser.on('targetdestroyed', (target) => {+ counts.destroy++;+ if (spawnedTarget === target) resolve(counts);+ });+ await clickElementsAndInterceptNavigationRequests(getOpts({+ waitForPageIdleMillis: 1000,+ maxWaitForPageIdleMillis: 5000,+ }));+ });++ expect(callCounts.create).to.be.eql(1);+ expect(callCounts.destroy).to.be.eql(1);+ const pageContent = await page.content();+ expect(pageContent).to.include('onclick="return window.open(');+ });});});