Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
expanded comments
vdrmota5b1ccf421f34
diff --git a/src/utils.js b/src/utils.jsindex 21a108a9d..3a2abb4ea 100644--- a/src/utils.js+++ b/src/utils.js@@ -618,6 +618,7 @@ export const snakeCaseToCamelCase = (snakeCaseStr) => {/*** Scrolls to the bottom of a page, or until it times out.+ * Loads dynamic content when it hits the bottom of a page, and then continues scrolling.* @param {Object} options* @param {Object} options.page The page object returned from browser.newPage()* @param {Number} [options.timeoutSecs=0] How many seconds to scroll for. If 0, will scroll until bottom of page.diff --git a/test/utils.js b/test/utils.jsindex 5c535dff1..ff43274ba 100644--- a/test/utils.js+++ b/test/utils.js@@ -901,6 +901,7 @@ describe('utils.infiniteScroll()', () => {headless: true,});const page = await browser.newPage();+ // Note: external websiteawait page.goto('https://twitter.com/search?src=typd&q=%23fingervein&lang=sv', {waitUntil: 'networkidle2',});@@ -915,6 +916,7 @@ describe('utils.infiniteScroll()', () => {headless: true,});const page = await browser.newPage();+ // Note: external websiteawait page.goto('https://medium.com/search?q=biometrics', {waitUntil: 'networkidle2',});