Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Refactoring to improve reliability of tests
Ondra Urbanea319d0f18c3
diff --git a/src/autoscaled_pool.js b/src/autoscaled_pool.jsindex ad576630e..d934bfffd 100644--- a/src/autoscaled_pool.js+++ b/src/autoscaled_pool.js@@ -226,8 +226,10 @@ export default class AutoscaledPool {/*** Stops the auto-scaled pool, discards all currently running tasks and destroys it.+ *+ * @return {Promise}*/- stop() {+ async stop() {if (this.resolve) this.resolve();// _destroy gets called 10 lines above so no need to do it here}diff --git a/src/basic_crawler.js b/src/basic_crawler.jsindex f0a67ed59..654a58245 100644--- a/src/basic_crawler.js+++ b/src/basic_crawler.js@@ -188,10 +188,12 @@ export default class BasicCrawler {/*** Stops the crawler by preventing crawls of additional pages. Pages already running are NOT terminated.+ *+ * @return {Promise}*/- stop() {+ async stop() {this.isRunning = false;- this.autoscaledPool.stop();+ await this.autoscaledPool.stop();this.rejectOnStop(new Error('BasicCrawler: .stop() function has been called. Stopping the crawler.'));}diff --git a/src/cheerio_crawler.js b/src/cheerio_crawler.jsindex 29275158e..dd83637c0 100644--- a/src/cheerio_crawler.js+++ b/src/cheerio_crawler.js@@ -210,11 +210,13 @@ export default class CheerioCrawler {/*** Stops the crawler by preventing crawls of additional pages. Pages already running are terminated.+ *+ * @return {Promise}*/- stop() {+ async stop() {this.isRunning = false;- this.basicCrawler.stop();- this.rejectOnStop(new Error('PuppeteerCrawler: .stop() function has been called. Stopping the crawler.'));+ await this.basicCrawler.stop();+ this.rejectOnStop(new Error('CheerioCrawler: .stop() function has been called. Stopping the crawler.'));}/**diff --git a/src/puppeteer_crawler.js b/src/puppeteer_crawler.jsindex 0d623275b..c6b77b63f 100644--- a/src/puppeteer_crawler.js+++ b/src/puppeteer_crawler.js@@ -226,10 +226,12 @@ export default class PuppeteerCrawler {/*** Stops the crawler by preventing crawls of additional pages. Pages already running are terminated.+ *+ * @return {Promise}*/- stop() {+ async stop() {this.isRunning = false;- this.basicCrawler.stop();+ await this.basicCrawler.stop();this.rejectOnStop(new Error('PuppeteerCrawler: .stop() function has been called. Stopping the crawler.'));}diff --git a/src/puppeteer_pool.js b/src/puppeteer_pool.jsindex 89b3cf73c..f23c755a5 100644--- a/src/puppeteer_pool.js+++ b/src/puppeteer_pool.js@@ -302,6 +302,7 @@ export default class PuppeteerPool {/*** Closes all the browsers.+ * @return {Promise}*/destroy() {clearInterval(this.instanceKillerInterval);@@ -329,6 +330,7 @@ export default class PuppeteerPool {/*** Finds a PuppeteerInstance given a Puppeteer Browser running in the instance.* @param {Puppeteer.Browser} browser+ * @return {Promise}* @ignore*/_findInstanceByBrowser(browser) {@@ -351,6 +353,7 @@ export default class PuppeteerPool {* to process open pages so that they may gracefully finish. This is unlike browser.close()* which will forcibly terminate the browser and all open pages will be closed.* @param {Puppeteer.Browser} browser+ * @return {Promise}*/retire(browser) {return this._findInstanceByBrowser(browser)diff --git a/test/basic_crawler.js b/test/basic_crawler.jsindex 86b3e6472..bad335fd1 100644--- a/test/basic_crawler.js+++ b/test/basic_crawler.js@@ -51,19 +51,20 @@ describe('BasicCrawler', () => {});it('should run, stop and then finish', async () => {- const sources = _.range(1, 500).map(index => ({ url: `https://example.com/${index}` }));+ const sources = _.range(500).map(index => ({ url: `https://example.com/${index + 1}` }));let basicCrawler;let isStopped;const processed = [];const requestList = new Apify.RequestList({ sources });const handleRequestFunction = async ({ request }) => {- if (request.url.endsWith('200')) {- if (!isStopped) basicCrawler.stop();+ if (request.url.endsWith('200') && !isStopped) {+ await basicCrawler.stop();isStopped = true;+ } else {+ await delayPromise(10);+ processed.push(_.pick(request, 'url'));}- await delayPromise(10);- processed.push(_.pick(request, 'url'));};basicCrawler = new Apify.BasicCrawler({@@ -78,7 +79,7 @@ describe('BasicCrawler', () => {// The crawler will stop after 200 requestsawait basicCrawler.run();- expect(processed.length).to.be.eql(175);+ expect(processed.length).to.be.within(175, 200);expect(await requestList.isFinished()).to.be.eql(false);expect(await requestList.isEmpty()).to.be.eql(false);diff --git a/test/cheerio_crawler.js b/test/cheerio_crawler.jsindex 7065ed34f..0c01e8cd5 100644--- a/test/cheerio_crawler.js+++ b/test/cheerio_crawler.js@@ -1,11 +1,18 @@import chai, { expect } from 'chai';import chaiAsPromised from 'chai-as-promised';import log from 'apify-shared/log';+import _ from 'underscore';import * as Apify from '../build/index';chai.use(chaiAsPromised);describe('CheerioCrawler', () => {+ const comparator = (a, b) => {+ a = Number(/q=(\d+)$/.exec(a.url)[1]);+ b = Number(/q=(\d+)$/.exec(b.url)[1]);+ return a - b;+ };+let logLevel;before(() => {@@ -37,8 +44,8 @@ describe('CheerioCrawler', () => {const cheerioCrawler = new Apify.CheerioCrawler({requestList,- minConcurrency: 1,- maxConcurrency: 1,+ minConcurrency: 2,+ maxConcurrency: 2,handlePageFunction,handleFailedRequestFunction: ({ request }) => failed.push(request),});@@ -49,6 +56,7 @@ describe('CheerioCrawler', () => {expect(processed).to.have.lengthOf(6);expect(failed).to.have.lengthOf(0);+ processed.sort(comparator);processed.forEach((request, id) => {expect(request.url).to.be.eql(sources[id].url);expect(request.userData.title).to.be.eql('Example Domain');@@ -56,4 +64,54 @@ describe('CheerioCrawler', () => {expect(request.userData.html.length).not.to.be.eql(0);});});++ it('should stop and resume', async () => {+ const sources = _.range(30).map(index => ({ url: `https://example.com/?q=${index + 1}` }));+ let cheerioCrawler;+ let isStopped = false;+ const processed = [];+ const failed = [];+ const requestList = new Apify.RequestList({ sources });+ const handlePageFunction = async ({ $, html, request }) => {+ if (request.url.endsWith('15') && !isStopped) {+ cheerioCrawler.stop();+ isStopped = true;+ } else {+ request.userData.title = $('title').text();+ request.userData.html = html;+ processed.push(request);+ }+ };++ cheerioCrawler = new Apify.CheerioCrawler({+ requestList,+ minConcurrency: 3,+ maxConcurrency: 3,+ handlePageFunction,+ handleFailedRequestFunction: ({ request }) => failed.push(request),+ });++ await requestList.initialize();+ await cheerioCrawler.run();++ expect(processed.length).to.be.within(12, 15);+ expect(failed).to.have.lengthOf(0);++ processed.sort(comparator);++ processed.forEach((request, id) => {+ expect(request.url).to.be.eql(sources[id].url);+ expect(request.userData.title).to.be.eql('Example Domain');+ });++ await Apify.utils.sleep(10); // Wait for event loop to unwind.+ await cheerioCrawler.run();++ expect(processed.length).to.be.within(30, 33);+ expect(failed).to.have.lengthOf(0);+ expect(new Set(processed.map(p => p.url))).to.be.eql(new Set(sources.map(s => s.url)));+ processed.forEach((request) => {+ expect(request.userData.title).to.be.eql('Example Domain');+ });+ });});diff --git a/test/puppeteer_crawler.js b/test/puppeteer_crawler.jsindex f0d9be506..8ad5df744 100644--- a/test/puppeteer_crawler.js+++ b/test/puppeteer_crawler.js@@ -70,7 +70,7 @@ describe('PuppeteerCrawler', () => {b = Number(/q=(\d+)$/.exec(b.url)[1]);return a - b;};- const sources = _.range(30).map(index => ({ url: `https://example.com/?q=${index}` }));+ const sources = _.range(30).map(index => ({ url: `https://example.com/?q=${index + 1}` }));let puppeteerCrawler;let isStopped = false;const processed = [];@@ -80,11 +80,12 @@ describe('PuppeteerCrawler', () => {if (request.url.endsWith('15') && !isStopped) {await puppeteerCrawler.stop();isStopped = true;+ } else {+ await page.waitFor('title');+ expect(await response.status()).to.be.eql(200);+ request.userData.title = await page.title();+ processed.push(request);}- await page.waitFor('title');- expect(await response.status()).to.be.eql(200);- request.userData.title = await page.title();- processed.push(request);};puppeteerCrawler = new Apify.PuppeteerCrawler({@@ -98,8 +99,7 @@ describe('PuppeteerCrawler', () => {await requestList.initialize();await puppeteerCrawler.run();- expect(processed.length).to.be.above(12);- expect(processed.length).to.be.below(16);+ expect(processed.length).to.be.within(12, 15);expect(failed).to.have.lengthOf(0);processed.sort(comparator);@@ -112,13 +112,10 @@ describe('PuppeteerCrawler', () => {await Apify.utils.sleep(10); // Wait for event loop to unwind.await puppeteerCrawler.run();- expect(processed).to.have.lengthOf(30);+ expect(processed.length).to.be.within(30, 33);expect(failed).to.have.lengthOf(0);-- processed.sort(comparator);-- processed.forEach((request, id) => {- expect(request.url).to.be.eql(sources[id].url);+ expect(new Set(processed.map(p => p.url))).to.be.eql(new Set(sources.map(s => s.url)));+ processed.forEach((request) => {expect(request.userData.title).to.be.eql('Example Domain');});});