Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: infiniteScroll() not working in Firefox (#1826)
Jindřich Bär4286c5d29b94
diff --git a/packages/playwright-crawler/src/internals/utils/playwright-utils.ts b/packages/playwright-crawler/src/internals/utils/playwright-utils.tsindex 6d75930a8..1551d63cf 100644--- a/packages/playwright-crawler/src/internals/utils/playwright-utils.ts+++ b/packages/playwright-crawler/src/internals/utils/playwright-utils.ts@@ -398,25 +398,6 @@ export async function infiniteScroll(page: Page, options: InfiniteScrollOptions}});- // Move mouse to the center of the page, so we can scroll up-down- let body = await page.$('body');-- for (let retry = 0; retry < 10; retry++) {- if (body) break;- await page.waitForTimeout(100);- body = await page.$('body');- }-- if (!body) {- return;- }-- const boundingBox = await body.boundingBox();- await page.mouse.move(- boundingBox!.x + boundingBox!.width / 2,- boundingBox!.y + boundingBox!.height / 2,- );-const checkFinished = setInterval(() => {if (resourcesStats.oldRequested === resourcesStats.newRequested) {resourcesStats.matchNumber++;@@ -455,7 +436,7 @@ export async function infiniteScroll(page: Page, options: InfiniteScrollOptionsawait doScroll();await page.waitForTimeout(250);if (scrollDownAndUp) {- await page.mouse.wheel(0, -1000);+ await page.mouse.wheel(0, -100);}if (buttonSelector) {await maybeClickButton();