Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

added infiniteScroll util

vdrmotad24e7cf22c7d
diff --git a/src/utils.js b/src/utils.js
index 7d68b97c6..9ebb8a86d 100644
--- a/src/utils.js
+++ b/src/utils.js
@@ -616,6 +616,69 @@ export const snakeCaseToCamelCase = (snakeCaseStr) => {
.join('');
};
+/**
+ * Scrolls to the bottom of a page, or until it times out.
+ * @param {Object} page
+ * @param {Number} [maxTimeout=0] How many seconds to scroll for. If 0, will scroll until bottom of page.
+ * @param {Number} [waitForDynamicContent=4] How many seconds to wait for no new content to load before exit.
+ * @returns {Promise}
+ * @memberOf utils
+ */
+const infiniteScroll = async function (page, maxTimeout = 0, waitForDynamicContent = 4) {
+
+ var finished;
+ const MAX_TIMEOUT = maxTimeout;
+ const WAIT_FOR_DYNAMIC_CONTENT = waitForDynamicContent;
+ const startTime = Date.now()
+ const maybeResourceTypesInfiniteScroll = ['xhr', 'fetch', 'websocket', 'other'];
+ const resourcesStats = {
+ newRequested: 0,
+ oldRequested: 0,
+ matchNumber: 0
+ };
+
+ new Promise((resolve, reject) => {
+ try {
+ page.on('request', (msg) => {
+ if (maybeResourceTypesInfiniteScroll.includes(msg.resourceType())) {
+ resourcesStats.newRequested++;
+ }
+ });
+ const scrollDown = setInterval(() => {
+ //console.log(resourcesStats)
+ if (resourcesStats.oldRequested === resourcesStats.newRequested) {
+ resourcesStats.matchNumber++;
+ if (resourcesStats.matchNumber >= WAIT_FOR_DYNAMIC_CONTENT) {
+ clearInterval(scrollDown)
+ resolve(finished = true);
+ }
+ } else {
+ resourcesStats.matchNumber = 0;
+ resourcesStats.oldRequested = resourcesStats.newRequested;
+ }
+ // check if timeout has been reached
+ if (MAX_TIMEOUT != 0 && (Date.now() - startTime) / 1000 > MAX_TIMEOUT) {
+ //console.log("Timeout limit reached, exiting infinite scroll.")
+ clearInterval(scrollDown)
+ resolve(finished = true);
+ }
+ }, 1000)
+ } catch (error) {
+ reject(error)
+ }
+ })
+
+ while (true) {
+ await page.evaluate(async () => {
+ let delta = document.body.scrollHeight === 0 ? 10000 : document.body.scrollHeight // in case scrollHeight fixed to 0
+ window.scrollBy(0, delta);
+ });
+ if (finished) {
+ break;
+ }
+ }
+}
+
/**
* A namespace that contains various utilities.
*
@@ -641,4 +704,5 @@ export const publicUtils = {
URL_NO_COMMAS_REGEX,
URL_WITH_COMMAS_REGEX,
createRequestDebugInfo,
+ infiniteScroll
};
diff --git a/test/utils.js b/test/utils.js
index 21653e457..1a0ca6a2d 100644
--- a/test/utils.js
+++ b/test/utils.js
@@ -10,6 +10,7 @@ import LruCache from 'apify-shared/lru_cache';
import { ENV_VARS, LOCAL_ENV_VARS } from 'apify-shared/consts';
import * as utils from '../build/utils';
import Apify from '../build/index';
+const puppeteer = require('puppeteer');
/* global process, describe, it */
@@ -878,3 +879,48 @@ describe('utils.addTimeoutToPromise()', () => {
}
});
});
+
+describe('utils.infiniteScroll()', () => {
+
+ it('exits after no more to scroll', () => {
+ (async () => {
+ const browser = await puppeteer.launch({
+ headless: true
+ });
+ const page = await browser.newPage();
+ const contentHTML = "<div>nothing</div>";
+ await page.setContent(contentHTML);
+ await infiniteScroll(page);
+ await browser.close();
+ })();
+ })
+
+ it('exits after reaches the bottom', () => {
+ (async () => {
+ const browser = await puppeteer.launch({
+ headless: true
+ });
+ const page = await browser.newPage();
+ await page.goto('https://twitter.com/search?src=typd&q=%23fingervein&lang=sv', {
+ waitUntil: "networkidle2"
+ });
+ await infiniteScroll(page);
+ await browser.close();
+ })();
+ })
+
+ it('times out if limit is set', () => {
+ (async () => {
+ const browser = await puppeteer.launch({
+ headless: true
+ });
+ const page = await browser.newPage();
+ await page.goto('https://medium.com/search?q=biometrics', {
+ waitUntil: "networkidle2"
+ });
+ const TIMEOUT_AFTER = 10; // seconds
+ await infiniteScroll(page, TIMEOUT_AFTER);
+ await browser.close();
+ })();
+ })
+})