Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fixed tests and eslint

vdrmota48b5b1e45664
diff --git a/src/utils.js b/src/utils.js
index 9ebb8a86d..ee839cd5e 100644
--- a/src/utils.js
+++ b/src/utils.js
@@ -625,19 +625,18 @@ export const snakeCaseToCamelCase = (snakeCaseStr) => {
* @memberOf utils
*/
const infiniteScroll = async function (page, maxTimeout = 0, waitForDynamicContent = 4) {
-
- var finished;
- const MAX_TIMEOUT = maxTimeout;
+ let finished;
+ const MAX_TIMEOUT = maxTimeout;
const WAIT_FOR_DYNAMIC_CONTENT = waitForDynamicContent;
- const startTime = Date.now()
+ const startTime = Date.now();
const maybeResourceTypesInfiniteScroll = ['xhr', 'fetch', 'websocket', 'other'];
const resourcesStats = {
newRequested: 0,
oldRequested: 0,
- matchNumber: 0
+ matchNumber: 0,
};
- new Promise((resolve, reject) => {
+ const scroller = new Promise((resolve, reject) => {
try {
page.on('request', (msg) => {
if (maybeResourceTypesInfiniteScroll.includes(msg.resourceType())) {
@@ -645,11 +644,11 @@ const infiniteScroll = async function (page, maxTimeout = 0, waitForDynamicConte
}
});
const scrollDown = setInterval(() => {
- //console.log(resourcesStats)
+ // console.log(resourcesStats)
if (resourcesStats.oldRequested === resourcesStats.newRequested) {
resourcesStats.matchNumber++;
if (resourcesStats.matchNumber >= WAIT_FOR_DYNAMIC_CONTENT) {
- clearInterval(scrollDown)
+ clearInterval(scrollDown);
resolve(finished = true);
}
} else {
@@ -657,27 +656,33 @@ const infiniteScroll = async function (page, maxTimeout = 0, waitForDynamicConte
resourcesStats.oldRequested = resourcesStats.newRequested;
}
// check if timeout has been reached
- if (MAX_TIMEOUT != 0 && (Date.now() - startTime) / 1000 > MAX_TIMEOUT) {
- //console.log("Timeout limit reached, exiting infinite scroll.")
- clearInterval(scrollDown)
+ if (MAX_TIMEOUT !== 0 && (Date.now() - startTime) / 1000 > MAX_TIMEOUT) {
+ // console.log("Timeout limit reached, exiting infinite scroll.")
+ clearInterval(scrollDown);
resolve(finished = true);
}
- }, 1000)
+ }, 1000);
} catch (error) {
- reject(error)
+ reject(error);
}
- })
+ });
+
+ scroller();
+
+ /* global window document */
+
+ const doScroll = await page.evaluate(async () => {
+ const delta = document.body.scrollHeight === 0 ? 10000 : document.body.scrollHeight; // in case scrollHeight fixed to 0
+ window.scrollBy(0, delta);
+ });
while (true) {
- await page.evaluate(async () => {
- let delta = document.body.scrollHeight === 0 ? 10000 : document.body.scrollHeight // in case scrollHeight fixed to 0
- window.scrollBy(0, delta);
- });
+ doScroll();
if (finished) {
break;
}
}
-}
+};
/**
* A namespace that contains various utilities.
@@ -704,5 +709,5 @@ export const publicUtils = {
URL_NO_COMMAS_REGEX,
URL_WITH_COMMAS_REGEX,
createRequestDebugInfo,
- infiniteScroll
+ infiniteScroll,
};
diff --git a/test/utils.js b/test/utils.js
index 1a0ca6a2d..8764cda18 100644
--- a/test/utils.js
+++ b/test/utils.js
@@ -10,6 +10,7 @@ import LruCache from 'apify-shared/lru_cache';
import { ENV_VARS, LOCAL_ENV_VARS } from 'apify-shared/consts';
import * as utils from '../build/utils';
import Apify from '../build/index';
+
const puppeteer = require('puppeteer');
/* global process, describe, it */
@@ -880,47 +881,46 @@ describe('utils.addTimeoutToPromise()', () => {
});
});
-describe('utils.infiniteScroll()', () => {
-
+describe.only('utils.infiniteScroll()', () => {
it('exits after no more to scroll', () => {
- (async () => {
+ (async () => {
const browser = await puppeteer.launch({
- headless: true
+ headless: true,
});
const page = await browser.newPage();
- const contentHTML = "<div>nothing</div>";
+ const contentHTML = '<div>nothing</div>';
await page.setContent(contentHTML);
- await infiniteScroll(page);
+ await utils.infiniteScroll(page);
await browser.close();
})();
- })
+ });
it('exits after reaches the bottom', () => {
(async () => {
const browser = await puppeteer.launch({
- headless: true
+ headless: true,
});
const page = await browser.newPage();
await page.goto('https://twitter.com/search?src=typd&q=%23fingervein&lang=sv', {
- waitUntil: "networkidle2"
+ waitUntil: 'networkidle2',
});
- await infiniteScroll(page);
+ await utils.infiniteScroll(page);
await browser.close();
})();
- })
+ });
it('times out if limit is set', () => {
(async () => {
const browser = await puppeteer.launch({
- headless: true
+ headless: true,
});
const page = await browser.newPage();
await page.goto('https://medium.com/search?q=biometrics', {
- waitUntil: "networkidle2"
+ waitUntil: 'networkidle2',
});
const TIMEOUT_AFTER = 10; // seconds
- await infiniteScroll(page, TIMEOUT_AFTER);
+ await utils.infiniteScroll(page, TIMEOUT_AFTER);
await browser.close();
})();
- })
-})
+ });
+});