Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fixed tests and eslint
vdrmota48b5b1e45664
diff --git a/src/utils.js b/src/utils.jsindex 9ebb8a86d..ee839cd5e 100644--- a/src/utils.js+++ b/src/utils.js@@ -625,19 +625,18 @@ export const snakeCaseToCamelCase = (snakeCaseStr) => {* @memberOf utils*/const infiniteScroll = async function (page, maxTimeout = 0, waitForDynamicContent = 4) {-- var finished;- const MAX_TIMEOUT = maxTimeout;+ let finished;+ const MAX_TIMEOUT = maxTimeout;const WAIT_FOR_DYNAMIC_CONTENT = waitForDynamicContent;- const startTime = Date.now()+ const startTime = Date.now();const maybeResourceTypesInfiniteScroll = ['xhr', 'fetch', 'websocket', 'other'];const resourcesStats = {newRequested: 0,oldRequested: 0,- matchNumber: 0+ matchNumber: 0,};- new Promise((resolve, reject) => {+ const scroller = new Promise((resolve, reject) => {try {page.on('request', (msg) => {if (maybeResourceTypesInfiniteScroll.includes(msg.resourceType())) {@@ -645,11 +644,11 @@ const infiniteScroll = async function (page, maxTimeout = 0, waitForDynamicConte}});const scrollDown = setInterval(() => {- //console.log(resourcesStats)+ // console.log(resourcesStats)if (resourcesStats.oldRequested === resourcesStats.newRequested) {resourcesStats.matchNumber++;if (resourcesStats.matchNumber >= WAIT_FOR_DYNAMIC_CONTENT) {- clearInterval(scrollDown)+ clearInterval(scrollDown);resolve(finished = true);}} else {@@ -657,27 +656,33 @@ const infiniteScroll = async function (page, maxTimeout = 0, waitForDynamicConteresourcesStats.oldRequested = resourcesStats.newRequested;}// check if timeout has been reached- if (MAX_TIMEOUT != 0 && (Date.now() - startTime) / 1000 > MAX_TIMEOUT) {- //console.log("Timeout limit reached, exiting infinite scroll.")- clearInterval(scrollDown)+ if (MAX_TIMEOUT !== 0 && (Date.now() - startTime) / 1000 > MAX_TIMEOUT) {+ // console.log("Timeout limit reached, exiting infinite scroll.")+ clearInterval(scrollDown);resolve(finished = true);}- }, 1000)+ }, 1000);} catch (error) {- reject(error)+ reject(error);}- })+ });++ scroller();++ /* global window document */++ const doScroll = await page.evaluate(async () => {+ const delta = document.body.scrollHeight === 0 ? 10000 : document.body.scrollHeight; // in case scrollHeight fixed to 0+ window.scrollBy(0, delta);+ });while (true) {- await page.evaluate(async () => {- let delta = document.body.scrollHeight === 0 ? 10000 : document.body.scrollHeight // in case scrollHeight fixed to 0- window.scrollBy(0, delta);- });+ doScroll();if (finished) {break;}}-}+};/*** A namespace that contains various utilities.@@ -704,5 +709,5 @@ export const publicUtils = {URL_NO_COMMAS_REGEX,URL_WITH_COMMAS_REGEX,createRequestDebugInfo,- infiniteScroll+ infiniteScroll,};diff --git a/test/utils.js b/test/utils.jsindex 1a0ca6a2d..8764cda18 100644--- a/test/utils.js+++ b/test/utils.js@@ -10,6 +10,7 @@ import LruCache from 'apify-shared/lru_cache';import { ENV_VARS, LOCAL_ENV_VARS } from 'apify-shared/consts';import * as utils from '../build/utils';import Apify from '../build/index';+const puppeteer = require('puppeteer');/* global process, describe, it */@@ -880,47 +881,46 @@ describe('utils.addTimeoutToPromise()', () => {});});-describe('utils.infiniteScroll()', () => {-+describe.only('utils.infiniteScroll()', () => {it('exits after no more to scroll', () => {- (async () => {+ (async () => {const browser = await puppeteer.launch({- headless: true+ headless: true,});const page = await browser.newPage();- const contentHTML = "<div>nothing</div>";+ const contentHTML = '<div>nothing</div>';await page.setContent(contentHTML);- await infiniteScroll(page);+ await utils.infiniteScroll(page);await browser.close();})();- })+ });it('exits after reaches the bottom', () => {(async () => {const browser = await puppeteer.launch({- headless: true+ headless: true,});const page = await browser.newPage();await page.goto('https://twitter.com/search?src=typd&q=%23fingervein&lang=sv', {- waitUntil: "networkidle2"+ waitUntil: 'networkidle2',});- await infiniteScroll(page);+ await utils.infiniteScroll(page);await browser.close();})();- })+ });it('times out if limit is set', () => {(async () => {const browser = await puppeteer.launch({- headless: true+ headless: true,});const page = await browser.newPage();await page.goto('https://medium.com/search?q=biometrics', {- waitUntil: "networkidle2"+ waitUntil: 'networkidle2',});const TIMEOUT_AFTER = 10; // seconds- await infiniteScroll(page, TIMEOUT_AFTER);+ await utils.infiniteScroll(page, TIMEOUT_AFTER);await browser.close();})();- })-})+ });+});