Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat: infiniteScroll has maxScrollHeight limit (#1945)
Jindřich Bär44997bba5bbf
diff --git a/packages/playwright-crawler/src/internals/utils/playwright-utils.ts b/packages/playwright-crawler/src/internals/utils/playwright-utils.tsindex 6ce18fda2..22e5f87e9 100644--- a/packages/playwright-crawler/src/internals/utils/playwright-utils.ts+++ b/packages/playwright-crawler/src/internals/utils/playwright-utils.ts@@ -336,10 +336,16 @@ export function compileScript(scriptString: string, context: Dictionary = Objectexport interface InfiniteScrollOptions {/*** How many seconds to scroll for. If 0, will scroll until bottom of page.- * @default 1+ * @default 0*/timeoutSecs?: number;+ /**+ * How many pixels to scroll down. If 0, will scroll until bottom of page.+ * @default 0+ */+ maxScrollHeight?: number;+/*** How many seconds to wait for no new content to load before exit.* @default 4@@ -373,18 +379,20 @@ export async function infiniteScroll(page: Page, options: InfiniteScrollOptionsow(page, ow.object.validate(validators.browserPage));ow(options, ow.object.exactShape({timeoutSecs: ow.optional.number,+ maxScrollHeight: ow.optional.number,waitForSecs: ow.optional.number,scrollDownAndUp: ow.optional.boolean,buttonSelector: ow.optional.string,stopScrollCallback: ow.optional.function,}));- const { timeoutSecs = 0, waitForSecs = 4, scrollDownAndUp = false, buttonSelector, stopScrollCallback } = options;+ const { timeoutSecs = 0, maxScrollHeight = 0, waitForSecs = 4, scrollDownAndUp = false, buttonSelector, stopScrollCallback } = options;let finished;const startTime = Date.now();const CHECK_INTERVAL_MILLIS = 1000;const SCROLL_HEIGHT_IF_ZERO = 10000;+ let scrolledDistance = 0;const maybeResourceTypesInfiniteScroll = ['xhr', 'fetch', 'websocket', 'other'];const resourcesStats = {newRequested: 0,@@ -415,6 +423,12 @@ export async function infiniteScroll(page: Page, options: InfiniteScrollOptionsclearInterval(checkFinished);finished = true;}++ // check if max scroll height has been reached+ if (maxScrollHeight > 0 && scrolledDistance >= maxScrollHeight) {+ clearInterval(checkFinished);+ finished = true;+ }}, CHECK_INTERVAL_MILLIS);const doScroll = async () => {@@ -422,6 +436,7 @@ export async function infiniteScroll(page: Page, options: InfiniteScrollOptionsconst delta = bodyScrollHeight === 0 ? SCROLL_HEIGHT_IF_ZERO : bodyScrollHeight;await page.mouse.wheel(0, delta);+ scrolledDistance += delta;};const maybeClickButton = async () => {diff --git a/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts b/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.tsindex 5f2122603..f10a9a5e0 100644--- a/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts+++ b/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts@@ -446,10 +446,16 @@ export async function gotoExtended(page: Page, request: Request, gotoOptions: Diexport interface InfiniteScrollOptions {/*** How many seconds to scroll for. If 0, will scroll until bottom of page.- * @default 1+ * @default 0*/timeoutSecs?: number;+ /**+ * How many pixels to scroll down. If 0, will scroll until bottom of page.+ * @default 0+ */+ maxScrollHeight?: number;+/*** How many seconds to wait for no new content to load before exit.* @default 4@@ -483,18 +489,20 @@ export async function infiniteScroll(page: Page, options: InfiniteScrollOptionsow(page, ow.object.validate(validators.browserPage));ow(options, ow.object.exactShape({timeoutSecs: ow.optional.number,+ maxScrollHeight: ow.optional.number,waitForSecs: ow.optional.number,scrollDownAndUp: ow.optional.boolean,buttonSelector: ow.optional.string,stopScrollCallback: ow.optional.function,}));- const { timeoutSecs = 0, waitForSecs = 4, scrollDownAndUp = false, buttonSelector, stopScrollCallback } = options;+ const { timeoutSecs = 0, maxScrollHeight = 0, waitForSecs = 4, scrollDownAndUp = false, buttonSelector, stopScrollCallback } = options;let finished;const startTime = Date.now();const CHECK_INTERVAL_MILLIS = 1000;const SCROLL_HEIGHT_IF_ZERO = 10000;+ let scrolledDistance = 0;const maybeResourceTypesInfiniteScroll = ['xhr', 'fetch', 'websocket', 'other'];const resourcesStats = {newRequested: 0,@@ -545,6 +553,12 @@ export async function infiniteScroll(page: Page, options: InfiniteScrollOptionsclearInterval(checkFinished);finished = true;}++ // check if max scroll height has been reached+ if (maxScrollHeight > 0 && scrolledDistance > maxScrollHeight) {+ clearInterval(checkFinished);+ finished = true;+ }}, CHECK_INTERVAL_MILLIS);const doScroll = async () => {@@ -554,6 +568,7 @@ export async function infiniteScroll(page: Page, options: InfiniteScrollOptionsconst delta = bodyScrollHeight === 0 ? SCROLL_HEIGHT_IF_ZERO : bodyScrollHeight;await page.mouse.wheel({ deltaY: delta });+ scrolledDistance += delta;};const maybeClickButton = async () => {diff --git a/test/core/playwright_utils.test.ts b/test/core/playwright_utils.test.tsindex 2343a9c41..e5300097f 100644--- a/test/core/playwright_utils.test.ts+++ b/test/core/playwright_utils.test.ts@@ -266,21 +266,35 @@ describe('playwrightUtils', () => {expect(after).toBe(true);});- test('stopScrollCallback works', async () => {+ test('maxScrollHeight works', async () => {const before = await page.evaluate(isAtBottom);expect(before).toBe(false);await playwrightUtils.infiniteScroll(page, {waitForSecs: Infinity,+ maxScrollHeight: 1000,stopScrollCallback: async () => true,});const after = await page.evaluate(isAtBottom);// It scrolls to the bottom in the first scroll so this is correct.- // The test passes because the Infinite waitForSecs is broken by the callback.+ // The test passes because the Infinite waitForSecs is broken by the height requirement.// If it didn't, the test would time out.expect(after).toBe(true);});++ test('stopScrollCallback works', async () => {+ const before = await page.evaluate(isAtBottom);+ expect(before).toBe(false);++ await playwrightUtils.infiniteScroll(page, {+ waitForSecs: Infinity,+ stopScrollCallback: async () => true,+ });++ const after = await page.evaluate(isAtBottom);+ expect(after).toBe(true);+ });});test('saveSnapshot() works', async () => {diff --git a/test/core/puppeteer_utils.test.ts b/test/core/puppeteer_utils.test.tsindex 7be72a355..ad7c99532 100644--- a/test/core/puppeteer_utils.test.ts+++ b/test/core/puppeteer_utils.test.ts@@ -392,21 +392,35 @@ describe('puppeteerUtils', () => {expect(after).toBe(true);});- test('stopScrollCallback works', async () => {+ test('maxScrollHeight works', async () => {const before = await page.evaluate(isAtBottom);expect(before).toBe(false);await puppeteerUtils.infiniteScroll(page, {waitForSecs: Infinity,+ maxScrollHeight: 1000,stopScrollCallback: async () => true,});const after = await page.evaluate(isAtBottom);// It scrolls to the bottom in the first scroll so this is correct.- // The test passes because the Infinite waitForSecs is broken by the callback.+ // The test passes because the Infinite waitForSecs is broken by the height requirement.// If it didn't, the test would time out.expect(after).toBe(true);});++ test('stopScrollCallback works', async () => {+ const before = await page.evaluate(isAtBottom);+ expect(before).toBe(false);++ await puppeteerUtils.infiniteScroll(page, {+ waitForSecs: Infinity,+ stopScrollCallback: async () => true,+ });++ const after = await page.evaluate(isAtBottom);+ expect(after).toBe(true);+ });});it('saveSnapshot() works', async () => {