Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

feat: add multiple click (#1295)

audi Bookning09eeb28b6b05
diff --git a/src/enqueue_links/click_elements.js b/src/enqueue_links/click_elements.js
index decf85e7e..ab65964a2 100644
--- a/src/enqueue_links/click_elements.js
+++ b/src/enqueue_links/click_elements.js
@@ -74,6 +74,12 @@ const STARTING_Z_INDEX = 2147400000;
*
* If `pseudoUrls` is an empty array, `null` or `undefined`, then the function
* enqueues all links found on the page.
+ * @param {object} [options.clickOptions]
+ * click options for use in Puppeteer's click handler
+ * @param {number} [options.clickOptions.clickCount]
+ * Number of clicks to be executed. Defaults to 1
+ * @param {number} [options.clickOptions.delay]
+ * Time to wait between mousedown and mouseup in milliseconds. Defaults to 0
* @param {RequestTransform} [options.transformRequestFunction]
* Just before a new {@link Request} is constructed and enqueued to the {@link RequestQueue}, this function can be used
* to remove it or modify its contents such as `userData`, `payload` or, most importantly `uniqueKey`. This is useful
@@ -121,6 +127,7 @@ export async function enqueueLinksByClickingElements(options) {
ow(options, ow.object.exactShape({
page: ow.object.hasKeys('goto', 'evaluate'),
requestQueue: ow.object.hasKeys('fetchNextRequest', 'addRequest'),
+ clickOptions: ow.optional.object.hasKeys('clickCount', 'delay'),
selector: ow.string,
pseudoUrls: ow.optional.array.ofType(ow.any(ow.string, ow.regExp, ow.object.hasKeys('purl'))),
transformRequestFunction: ow.optional.function,
@@ -136,6 +143,7 @@ export async function enqueueLinksByClickingElements(options) {
transformRequestFunction,
waitForPageIdleSecs = 1,
maxWaitForPageIdleSecs = 5,
+ clickOptions,
} = options;
const waitForPageIdleMillis = waitForPageIdleSecs * 1000;
@@ -147,6 +155,7 @@ export async function enqueueLinksByClickingElements(options) {
selector,
waitForPageIdleMillis,
maxWaitForPageIdleMillis,
+ clickOptions,
});
let requestOptions = createRequestOptions(interceptedRequests);
if (transformRequestFunction) {
@@ -166,6 +175,9 @@ export async function enqueueLinksByClickingElements(options) {
* @param {string} options.selector
* @param {number} [options.waitForPageIdleMillis]
* @param {number} [options.maxWaitForPageIdleMillis]
+ * @param {object} [clickOptions]
+ * @param {number} [clickOptions.clickCount]
+ * @param {number} [clickOptions.delay]
* @return {Promise<Array<*>>}
* @ignore
*/
@@ -175,6 +187,7 @@ export async function clickElementsAndInterceptNavigationRequests(options) {
selector,
waitForPageIdleMillis,
maxWaitForPageIdleMillis,
+ clickOptions,
} = options;
const uniqueRequests = new Set();
@@ -190,7 +203,7 @@ export async function clickElementsAndInterceptNavigationRequests(options) {
await preventHistoryNavigation(page);
- await clickElements(page, selector);
+ await clickElements(page, selector, clickOptions);
await waitForPageIdle({ page, waitForPageIdleMillis, maxWaitForPageIdleMillis });
await restoreHistoryNavigationAndSaveCapturedUrls(page, uniqueRequests);
@@ -324,10 +337,13 @@ async function preventHistoryNavigation(page) {
*
* @param {Page} page
* @param {string} selector
+ * @param {object} [clickOptions]
+ * @param {number} [clickOptions.clickCount]
+ * @param {number} [clickOptions.delay]
* @return {Promise<void>}
* @ignore
*/
-export async function clickElements(page, selector) {
+export async function clickElements(page, selector, clickOptions) {
const elementHandles = await page.$$(selector);
log.debug(`enqueueLinksByClickingElements: There are ${elementHandles.length} elements to click.`);
let clickedElementsCount = 0;
@@ -336,7 +352,7 @@ export async function clickElements(page, selector) {
for (const handle of elementHandles) {
try {
await page.evaluate(updateElementCssToEnableMouseClick, handle, zIndex++);
- await handle.click();
+ await handle.click(clickOptions);
clickedElementsCount++;
} catch (err) {
if (shouldLogWarning && err.stack.includes('is detached from document')) {
diff --git a/test/enqueue_links/click_elements.test.js b/test/enqueue_links/click_elements.test.js
index fdf33e7c8..76ee652bf 100644
--- a/test/enqueue_links/click_elements.test.js
+++ b/test/enqueue_links/click_elements.test.js
@@ -137,6 +137,117 @@ describe('enqueueLinksByClickingElements()', () => {
});
});
+ describe('double clickElements()', () => {
+ test('should double click an element', async () => {
+ const html = `
+<html>
+ <body>
+ <div ondblclick="window.clicked = true;">div</div>
+ </body>
+</html>
+ `;
+ await page.setContent(html);
+ await clickElements(page, 'div', { clickCount: 2, delay: 100 });
+ const wasClicked = await page.evaluate(() => window.clicked);
+ expect(wasClicked).toBe(true);
+ });
+
+ test('should double click an empty element', async () => {
+ const html = `
+<html>
+ <body>
+ <div ondblclick="window.clicked = true;"></div>
+ </body>
+</html>
+ `;
+ await page.setContent(html);
+ await clickElements(page, 'div', { clickCount: 2, delay: 100 });
+ const wasClicked = await page.evaluate(() => window.clicked);
+ expect(wasClicked).toBe(true);
+ });
+ test('should double click multiple elements', async () => {
+ const html = `
+<html>
+ <script>
+ window.clickedElements = [];
+ window.handleClick = (evt) => window.clickedElements.push(evt.target.nodeName);
+ </script>
+ <body>
+ <header ondblclick="return window.handleClick(event)"></header>
+ <div ondblclick="return window.handleClick(event)"></div>
+ <p ondblclick="return window.handleClick(event)"></p>
+ <footer ondblclick="return window.handleClick(event)"></footer>
+ </body>
+</html>
+ `;
+ await page.setContent(html);
+ await clickElements(page, 'header, div, p, footer', { clickCount: 2, delay: 100 });
+ const clickedElements = await page.evaluate(() => window.clickedElements);
+ expect(clickedElements).toEqual(['HEADER', 'DIV', 'P', 'FOOTER']);
+ });
+
+ test('should double click hidden elements', async () => {
+ const html = `
+<html>
+ <body>
+ <div ondblclick="window.clicked = true;" style="visibility: hidden; display: none"></div>
+ </body>
+</html>
+ `;
+ await page.setContent(html);
+ await clickElements(page, 'div', { clickCount: 2, delay: 100 });
+ const wasClicked = await page.evaluate(() => window.clicked);
+ expect(wasClicked).toBe(true);
+ });
+
+ test('should click elements stacked on top of each other', async () => {
+ const html = `
+<html>
+ <script>
+ window.clickedElements = [];
+ window.handleClick = (evt) => window.clickedElements.push(evt.target.nodeName);
+ </script>
+ <body>
+ <header ondblclick="return window.handleClick(event)" style="position: absolute; z-index: auto">header</header>
+ <div ondblclick="return window.handleClick(event)" style="position: absolute; z-index: 1">div</div>
+ <main ondblclick="return window.handleClick(event)" style="position: absolute; z-index: 2">main</main>
+ <footer ondblclick="return window.handleClick(event)" style="position: absolute; z-index: 3">footer</footer>
+ </body>
+</html>
+ `;
+ await page.setContent(html);
+ await clickElements(page, 'header, div, main, footer', { clickCount: 2, delay: 100 });
+ const clickedElements = await page.evaluate(() => window.clickedElements);
+ expect(clickedElements).toEqual(['HEADER', 'DIV', 'MAIN', 'FOOTER']);
+ });
+ });
+
+ describe('select line with triple clickElements()', () => {
+ test('should select the text by triple clicking', async () => {
+ const html = `
+<html>
+ <body>
+ <textarea></textarea>
+ </body>
+</html>
+ `;
+ await page.setContent(html);
+ await page.focus('textarea');
+ const text = "This is the text that we are going to try to select. Let's see how it goes.";
+ await page.keyboard.type(text);
+ await clickElements(page, 'textarea', { clickCount: 3, delay: 100 });
+ expect(
+ await page.evaluate(() => {
+ const textarea = document.querySelector('textarea');
+ return textarea.value.substring(
+ textarea.selectionStart,
+ textarea.selectionEnd,
+ );
+ }),
+ ).toBe(text);
+ });
+ });
+
describe('clickElementsAndInterceptNavigationRequests()', () => {
function getOpts(overrides = {}) {
return {