Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

feat: infiniteScroll has maxScrollHeight limit (#1945)

Jindřich Bär44997bba5bbf
diff --git a/packages/playwright-crawler/src/internals/utils/playwright-utils.ts b/packages/playwright-crawler/src/internals/utils/playwright-utils.ts
index 6ce18fda2..22e5f87e9 100644
--- a/packages/playwright-crawler/src/internals/utils/playwright-utils.ts
+++ b/packages/playwright-crawler/src/internals/utils/playwright-utils.ts
@@ -336,10 +336,16 @@ export function compileScript(scriptString: string, context: Dictionary = Object
export interface InfiniteScrollOptions {
/**
* How many seconds to scroll for. If 0, will scroll until bottom of page.
- * @default 1
+ * @default 0
*/
timeoutSecs?: number;
+ /**
+ * How many pixels to scroll down. If 0, will scroll until bottom of page.
+ * @default 0
+ */
+ maxScrollHeight?: number;
+
/**
* How many seconds to wait for no new content to load before exit.
* @default 4
@@ -373,18 +379,20 @@ export async function infiniteScroll(page: Page, options: InfiniteScrollOptions
ow(page, ow.object.validate(validators.browserPage));
ow(options, ow.object.exactShape({
timeoutSecs: ow.optional.number,
+ maxScrollHeight: ow.optional.number,
waitForSecs: ow.optional.number,
scrollDownAndUp: ow.optional.boolean,
buttonSelector: ow.optional.string,
stopScrollCallback: ow.optional.function,
}));
- const { timeoutSecs = 0, waitForSecs = 4, scrollDownAndUp = false, buttonSelector, stopScrollCallback } = options;
+ const { timeoutSecs = 0, maxScrollHeight = 0, waitForSecs = 4, scrollDownAndUp = false, buttonSelector, stopScrollCallback } = options;
let finished;
const startTime = Date.now();
const CHECK_INTERVAL_MILLIS = 1000;
const SCROLL_HEIGHT_IF_ZERO = 10000;
+ let scrolledDistance = 0;
const maybeResourceTypesInfiniteScroll = ['xhr', 'fetch', 'websocket', 'other'];
const resourcesStats = {
newRequested: 0,
@@ -415,6 +423,12 @@ export async function infiniteScroll(page: Page, options: InfiniteScrollOptions
clearInterval(checkFinished);
finished = true;
}
+
+ // check if max scroll height has been reached
+ if (maxScrollHeight > 0 && scrolledDistance >= maxScrollHeight) {
+ clearInterval(checkFinished);
+ finished = true;
+ }
}, CHECK_INTERVAL_MILLIS);
const doScroll = async () => {
@@ -422,6 +436,7 @@ export async function infiniteScroll(page: Page, options: InfiniteScrollOptions
const delta = bodyScrollHeight === 0 ? SCROLL_HEIGHT_IF_ZERO : bodyScrollHeight;
await page.mouse.wheel(0, delta);
+ scrolledDistance += delta;
};
const maybeClickButton = async () => {
diff --git a/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts b/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts
index 5f2122603..f10a9a5e0 100644
--- a/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts
+++ b/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts
@@ -446,10 +446,16 @@ export async function gotoExtended(page: Page, request: Request, gotoOptions: Di
export interface InfiniteScrollOptions {
/**
* How many seconds to scroll for. If 0, will scroll until bottom of page.
- * @default 1
+ * @default 0
*/
timeoutSecs?: number;
+ /**
+ * How many pixels to scroll down. If 0, will scroll until bottom of page.
+ * @default 0
+ */
+ maxScrollHeight?: number;
+
/**
* How many seconds to wait for no new content to load before exit.
* @default 4
@@ -483,18 +489,20 @@ export async function infiniteScroll(page: Page, options: InfiniteScrollOptions
ow(page, ow.object.validate(validators.browserPage));
ow(options, ow.object.exactShape({
timeoutSecs: ow.optional.number,
+ maxScrollHeight: ow.optional.number,
waitForSecs: ow.optional.number,
scrollDownAndUp: ow.optional.boolean,
buttonSelector: ow.optional.string,
stopScrollCallback: ow.optional.function,
}));
- const { timeoutSecs = 0, waitForSecs = 4, scrollDownAndUp = false, buttonSelector, stopScrollCallback } = options;
+ const { timeoutSecs = 0, maxScrollHeight = 0, waitForSecs = 4, scrollDownAndUp = false, buttonSelector, stopScrollCallback } = options;
let finished;
const startTime = Date.now();
const CHECK_INTERVAL_MILLIS = 1000;
const SCROLL_HEIGHT_IF_ZERO = 10000;
+ let scrolledDistance = 0;
const maybeResourceTypesInfiniteScroll = ['xhr', 'fetch', 'websocket', 'other'];
const resourcesStats = {
newRequested: 0,
@@ -545,6 +553,12 @@ export async function infiniteScroll(page: Page, options: InfiniteScrollOptions
clearInterval(checkFinished);
finished = true;
}
+
+ // check if max scroll height has been reached
+ if (maxScrollHeight > 0 && scrolledDistance > maxScrollHeight) {
+ clearInterval(checkFinished);
+ finished = true;
+ }
}, CHECK_INTERVAL_MILLIS);
const doScroll = async () => {
@@ -554,6 +568,7 @@ export async function infiniteScroll(page: Page, options: InfiniteScrollOptions
const delta = bodyScrollHeight === 0 ? SCROLL_HEIGHT_IF_ZERO : bodyScrollHeight;
await page.mouse.wheel({ deltaY: delta });
+ scrolledDistance += delta;
};
const maybeClickButton = async () => {
diff --git a/test/core/playwright_utils.test.ts b/test/core/playwright_utils.test.ts
index 2343a9c41..e5300097f 100644
--- a/test/core/playwright_utils.test.ts
+++ b/test/core/playwright_utils.test.ts
@@ -266,21 +266,35 @@ describe('playwrightUtils', () => {
expect(after).toBe(true);
});
- test('stopScrollCallback works', async () => {
+ test('maxScrollHeight works', async () => {
const before = await page.evaluate(isAtBottom);
expect(before).toBe(false);
await playwrightUtils.infiniteScroll(page, {
waitForSecs: Infinity,
+ maxScrollHeight: 1000,
stopScrollCallback: async () => true,
});
const after = await page.evaluate(isAtBottom);
// It scrolls to the bottom in the first scroll so this is correct.
- // The test passes because the Infinite waitForSecs is broken by the callback.
+ // The test passes because the Infinite waitForSecs is broken by the height requirement.
// If it didn't, the test would time out.
expect(after).toBe(true);
});
+
+ test('stopScrollCallback works', async () => {
+ const before = await page.evaluate(isAtBottom);
+ expect(before).toBe(false);
+
+ await playwrightUtils.infiniteScroll(page, {
+ waitForSecs: Infinity,
+ stopScrollCallback: async () => true,
+ });
+
+ const after = await page.evaluate(isAtBottom);
+ expect(after).toBe(true);
+ });
});
test('saveSnapshot() works', async () => {
diff --git a/test/core/puppeteer_utils.test.ts b/test/core/puppeteer_utils.test.ts
index 7be72a355..ad7c99532 100644
--- a/test/core/puppeteer_utils.test.ts
+++ b/test/core/puppeteer_utils.test.ts
@@ -392,21 +392,35 @@ describe('puppeteerUtils', () => {
expect(after).toBe(true);
});
- test('stopScrollCallback works', async () => {
+ test('maxScrollHeight works', async () => {
const before = await page.evaluate(isAtBottom);
expect(before).toBe(false);
await puppeteerUtils.infiniteScroll(page, {
waitForSecs: Infinity,
+ maxScrollHeight: 1000,
stopScrollCallback: async () => true,
});
const after = await page.evaluate(isAtBottom);
// It scrolls to the bottom in the first scroll so this is correct.
- // The test passes because the Infinite waitForSecs is broken by the callback.
+ // The test passes because the Infinite waitForSecs is broken by the height requirement.
// If it didn't, the test would time out.
expect(after).toBe(true);
});
+
+ test('stopScrollCallback works', async () => {
+ const before = await page.evaluate(isAtBottom);
+ expect(before).toBe(false);
+
+ await puppeteerUtils.infiniteScroll(page, {
+ waitForSecs: Infinity,
+ stopScrollCallback: async () => true,
+ });
+
+ const after = await page.evaluate(isAtBottom);
+ expect(after).toBe(true);
+ });
});
it('saveSnapshot() works', async () => {