Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
separated integration tests and added constants
vdrmota605fddca903c
diff --git a/package.json b/package.jsonindex e9a4d122d..d0efed72c 100644--- a/package.json+++ b/package.json@@ -42,6 +42,7 @@"build-docs": "npm run clean && npm run build && node ./website/tools/build_docs.js","build-readme": "node ./tools/build_readme.js","test": "npm run build && nyc --reporter=html --reporter=text mocha --timeout 60000 --require @babel/register --recursive --exit",+ "test:integration": "npm run build && nyc --reporter=html --reporter=text mocha 'test/integration_tests' --timeout 60000 --require @babel/register --recursive --exit","prepare": "npm run build","prepublishOnly": "(test $RUNNING_FROM_SCRIPT || (echo \"You must use publish.sh instead of 'npm publish' directly!\"; exit 1)) && npm test && npm run lint","clean": "rm -rf build",diff --git a/src/utils.js b/src/utils.jsindex 3a2abb4ea..e74896767 100644--- a/src/utils.js+++ b/src/utils.js@@ -632,9 +632,9 @@ export const infiniteScroll = async function ({ page, timeoutSecs = 0, waitForSecheckParamOrThrow(waitForSecs, 'waitForSecs', 'Number');let finished;- const TIMEOUT_SECS = timeoutSecs;- const WAIT_FOR_SECS = waitForSecs;const startTime = Date.now();+ const CHECK_INTERVAL_MILLIS = 1000;+ const SCROLL_HEIGHT_IF_ZERO = 10000;const maybeResourceTypesInfiniteScroll = ['xhr', 'fetch', 'websocket', 'other'];const resourcesStats = {newRequested: 0,@@ -652,7 +652,7 @@ export const infiniteScroll = async function ({ page, timeoutSecs = 0, waitForSe// console.log(resourcesStats)if (resourcesStats.oldRequested === resourcesStats.newRequested) {resourcesStats.matchNumber++;- if (resourcesStats.matchNumber >= WAIT_FOR_SECS) {+ if (resourcesStats.matchNumber >= waitForSecs) {clearInterval(checkFinished);finished = true;return;@@ -662,12 +662,12 @@ export const infiniteScroll = async function ({ page, timeoutSecs = 0, waitForSeresourcesStats.oldRequested = resourcesStats.newRequested;}// check if timeout has been reached- if (TIMEOUT_SECS !== 0 && (Date.now() - startTime) / 1000 > TIMEOUT_SECS) {+ if (timeoutSecs !== 0 && (Date.now() - startTime) / 1000 > timeoutSecs) {// console.log("Timeout limit reached, exiting infinite scroll.")clearInterval(checkFinished);finished = true;}- }, 1000);+ }, CHECK_INTERVAL_MILLIS);/* global window document */@@ -675,7 +675,7 @@ export const infiniteScroll = async function ({ page, timeoutSecs = 0, waitForSeconst doScroll = async () => {/* istanbul ignore next */await page.evaluate(async () => {- const delta = document.body.scrollHeight === 0 ? 10000 : document.body.scrollHeight; // in case scrollHeight fixed to 0+ const delta = document.body.scrollHeight === 0 ? SCROLL_HEIGHT_IF_ZERO : document.body.scrollHeight; // in case scrollHeight fixed to 0window.scrollBy(0, delta);});};diff --git a/test/integration_tests/utils.js b/test/integration_tests/utils.jsnew file mode 100644index 000000000..18d1894db--- /dev/null+++ b/test/integration_tests/utils.js@@ -0,0 +1,49 @@+import * as utils from '../../build/utils';++const puppeteer = require('puppeteer');++describe('utils.infiniteScroll()', () => {+ it('exits after no more to scroll', () => {+ (async () => {+ const browser = await puppeteer.launch({+ headless: true,+ });+ const page = await browser.newPage();+ const contentHTML = '<div>nothing</div>';+ await page.setContent(contentHTML);+ await utils.infiniteScroll({ page });+ await browser.close();+ })();+ });++ it('exits after reaches the bottom', () => {+ (async () => {+ const browser = await puppeteer.launch({+ headless: true,+ });+ const page = await browser.newPage();+ // Note: external website+ await page.goto('https://twitter.com/search?src=typd&q=%23fingervein&lang=sv', {+ waitUntil: 'networkidle2',+ });+ await utils.infiniteScroll({ page });+ await browser.close();+ })();+ });++ it('times out if limit is set', () => {+ (async () => {+ const browser = await puppeteer.launch({+ headless: true,+ });+ const page = await browser.newPage();+ // Note: external website+ await page.goto('https://medium.com/search?q=biometrics', {+ waitUntil: 'networkidle2',+ });+ const timeoutSecs = 10; // seconds+ await utils.infiniteScroll({ page, timeoutSecs });+ await browser.close();+ })();+ });+});diff --git a/test/utils.js b/test/utils.jsindex ff43274ba..21653e457 100644--- a/test/utils.js+++ b/test/utils.js@@ -11,8 +11,6 @@ import { ENV_VARS, LOCAL_ENV_VARS } from 'apify-shared/consts';import * as utils from '../build/utils';import Apify from '../build/index';-const puppeteer = require('puppeteer');-/* global process, describe, it */describe('utils.newClient()', () => {@@ -880,49 +878,3 @@ describe('utils.addTimeoutToPromise()', () => {}});});--describe('utils.infiniteScroll()', () => {- it('exits after no more to scroll', () => {- (async () => {- const browser = await puppeteer.launch({- headless: true,- });- const page = await browser.newPage();- const contentHTML = '<div>nothing</div>';- await page.setContent(contentHTML);- await utils.infiniteScroll({ page });- await browser.close();- })();- });-- it('exits after reaches the bottom', () => {- (async () => {- const browser = await puppeteer.launch({- headless: true,- });- const page = await browser.newPage();- // Note: external website- await page.goto('https://twitter.com/search?src=typd&q=%23fingervein&lang=sv', {- waitUntil: 'networkidle2',- });- await utils.infiniteScroll({ page });- await browser.close();- })();- });-- it('times out if limit is set', () => {- (async () => {- const browser = await puppeteer.launch({- headless: true,- });- const page = await browser.newPage();- // Note: external website- await page.goto('https://medium.com/search?q=biometrics', {- waitUntil: 'networkidle2',- });- const timeoutSecs = 10; // seconds- await utils.infiniteScroll({ page, timeoutSecs });- await browser.close();- })();- });-});