Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

feat: add a breaking callback to `infiniteScroll` (#1140)

Ondra Urban6bd84119e560
diff --git a/src/puppeteer_utils.js b/src/puppeteer_utils.js
index 38038cb58..5b42a7faf 100644
--- a/src/puppeteer_utils.js
+++ b/src/puppeteer_utils.js
@@ -394,6 +394,8 @@ export const gotoExtended = async (page, request, gotoOptions = {}) => {
* If true, it will scroll up a bit after each scroll down. This is required on some websites for the scroll to work.
* @param {string} [options.buttonSelector]
* Optionally checks and clicks a button if it appears while scrolling. This is required on some websites for the scroll to work.
+ * @param {function} [options.stopScrollCallback]
+ * This function is called after every scroll and stops the scrolling process if it returns `true`. The function can be `async`.
* @returns {Promise<void>}
* @memberOf puppeteer
* @name infiniteScroll
@@ -405,9 +407,10 @@ export const infiniteScroll = async (page, options = {}) => {
waitForSecs: ow.optional.number,
scrollDownAndUp: ow.optional.boolean,
buttonSelector: ow.optional.string,
+ stopScrollCallback: ow.optional.function,
}));
- const { timeoutSecs = 0, waitForSecs = 4, scrollDownAndUp = false, buttonSelector } = options;
+ const { timeoutSecs = 0, waitForSecs = 4, scrollDownAndUp = false, buttonSelector, stopScrollCallback } = options;
let finished;
const startTime = Date.now();
@@ -472,6 +475,12 @@ export const infiniteScroll = async (page, options = {}) => {
if (buttonSelector) {
await maybeClickButton();
}
+ if (stopScrollCallback) {
+ if (await stopScrollCallback()) {
+ clearInterval(checkFinished);
+ break;
+ }
+ }
}
};
diff --git a/test/puppeteer_utils.test.js b/test/puppeteer_utils.test.js
index 91bd189d2..f6dc1de1e 100644
--- a/test/puppeteer_utils.test.js
+++ b/test/puppeteer_utils.test.js
@@ -347,20 +347,34 @@ describe('Apify.utils.puppeteer', () => {
}
});
- test('infiniteScroll() works', async () => {
+ describe('infiniteScroll()', () => {
function isAtBottom() {
return (window.innerHeight + window.pageYOffset) >= document.body.offsetHeight;
}
- const browser = await Apify.launchPuppeteer({ launchOptions: { headless: true } });
- try {
- const page = await browser.newPage();
+
+ let browser;
+ beforeAll(async () => {
+ browser = await Apify.launchPuppeteer({ launchOptions: { headless: true } });
+ });
+ afterAll(async () => {
+ await browser.close();
+ });
+
+ let page;
+ beforeEach(async () => {
+ page = await browser.newPage();
let count = 0;
const content = Array(1000).fill(null).map(() => {
return `<div style="border: 1px solid black">Div number: ${count++}</div>`;
});
const contentHTML = `<html><body>${content}</body></html>`;
await page.setContent(contentHTML);
+ });
+ afterEach(async () => {
+ await page.close();
+ });
+ test('works', async () => {
const before = await page.evaluate(isAtBottom);
expect(before).toBe(false);
@@ -368,9 +382,23 @@ describe('Apify.utils.puppeteer', () => {
const after = await page.evaluate(isAtBottom);
expect(after).toBe(true);
- } finally {
- await browser.close();
- }
+ });
+
+ test('stopScrollCallback works', async () => {
+ const before = await page.evaluate(isAtBottom);
+ expect(before).toBe(false);
+
+ await Apify.utils.puppeteer.infiniteScroll(page, {
+ waitForSecs: Infinity,
+ stopScrollCallback: async () => true,
+ });
+
+ const after = await page.evaluate(isAtBottom);
+ // It scrolls to the bottom in the first scroll so this is correct.
+ // The test passes because the Infinite waitForSecs is broken by the callback.
+ // If it didn't, the test would time out.
+ expect(after).toBe(true);
+ });
});
it('saveSnapshot() works', async () => {