Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Speed up CheerioCrawler tests
Ondra Urban8bb698b0832a
diff --git a/package.json b/package.jsonindex 835b6e968..14700065d 100644--- a/package.json+++ b/package.json@@ -41,6 +41,7 @@"build-doc": "npm run clean && npm run build && node ./node_modules/jsdoc/jsdoc.js --package ./package.json -c ./jsdoc/conf.json -d docs","build-toc": "./node_modules/.bin/markdown-toc README.md -i","test": "npm run build && nyc --reporter=html --reporter=text mocha --timeout 60000 --compilers js:babel-core/register --recursive",+ "tst": "mocha --timeout 60000 --compilers js:babel-core/register --recursive","prepare": "npm run build-toc && npm run build","prepublishOnly": "(test $RUNNING_FROM_SCRIPT || (echo \"You must use publish.sh instead of 'npm publish' directly!\"; exit 1)) && npm test && npm run lint","clean": "rm -rf build",diff --git a/test/cheerio_crawler.js b/test/cheerio_crawler.jsindex 5609853cc..313847bfb 100644--- a/test/cheerio_crawler.js+++ b/test/cheerio_crawler.js@@ -2,6 +2,7 @@ import chai, { expect } from 'chai';import chaiAsPromised from 'chai-as-promised';import log from 'apify-shared/log';import _ from 'underscore';+import { delayPromise } from 'apify-shared/utilities';import * as Apify from '../build/index';chai.use(chaiAsPromised);@@ -65,16 +66,67 @@ describe('CheerioCrawler', () => {});});+ it('should work with custom request function', async () => {+ const sources = [+ { url: 'http://example.com/?q=0' },+ { url: 'http://example.com/?q=1' },+ { url: 'http://example.com/?q=2' },+ { url: 'http://example.com/?q=3' },+ { url: 'http://example.com/?q=4' },+ { url: 'http://example.com/?q=5' },+ { url: 'http://example.com/?q=6' },+ ];+ const processed = [];+ const failed = [];+ const requestList = new Apify.RequestList({ sources });+ const requestFunction = async ({ request }) => {+ await delayPromise(1);+ return `<html><head><title>${request.url[request.url.length - 1]}</title></head><body>Body</body></html>`;+ };+ const handlePageFunction = async ({ $, html, request }) => {+ request.userData.title = $('title').text();+ request.userData.html = html;+ processed.push(request);+ };++ const cheerioCrawler = new Apify.CheerioCrawler({+ requestList,+ minConcurrency: 2,+ maxConcurrency: 2,+ handlePageFunction,+ requestFunction,+ handleFailedRequestFunction: ({ request }) => failed.push(request),+ });++ await requestList.initialize();+ await cheerioCrawler.run();++ expect(processed).to.have.lengthOf(7);+ expect(failed).to.have.lengthOf(0);++ processed.sort(comparator);+ processed.forEach((request, id) => {+ expect(request.url).to.be.eql(sources[id].url);+ expect(request.userData.title).to.be.eql(String(id));+ expect(request.userData.html).to.be.a('string');+ expect(request.userData.html.length).not.to.be.eql(0);+ });+ });+it('should abort and resume', async () => {- const sources = _.range(30).map(index => ({ url: `https://example.com/?q=${index + 1}` }));+ const sources = _.range(100).map(index => ({ url: `https://example.com/?q=${index + 1}` }));let cheerioCrawler;let isStopped = false;const processed = [];const failed = [];const requestList = new Apify.RequestList({ sources });+ const requestFunction = async ({ request }) => {+ await delayPromise(2);+ return `<html><head><title>${request.url[request.url.length - 1]}</title></head><body>Body</body></html>`;+ };const handlePageFunction = async ({ $, html, request }) => {- if (request.url.endsWith('15') && !isStopped) {- cheerioCrawler.abort();+ if (request.url.endsWith('45') && !isStopped) {+ await cheerioCrawler.abort();isStopped = true;} else {request.userData.title = $('title').text();@@ -85,8 +137,9 @@ describe('CheerioCrawler', () => {cheerioCrawler = new Apify.CheerioCrawler({requestList,- minConcurrency: 3,- maxConcurrency: 3,+ minConcurrency: 5,+ maxConcurrency: 5,+ requestFunction,handlePageFunction,handleFailedRequestFunction: ({ request }) => failed.push(request),});@@ -94,7 +147,7 @@ describe('CheerioCrawler', () => {await requestList.initialize();await cheerioCrawler.run();- expect(processed.length).to.be.within(12, 15);+ expect(processed.length).to.be.within(40, 45);expect(failed).to.have.lengthOf(0);processed.sort(comparator);@@ -102,17 +155,17 @@ describe('CheerioCrawler', () => {for (let i = 0; i < 12; i++) {const request = processed[i];expect(request.url).to.be.eql(sources[i].url);- expect(request.userData.title).to.be.eql('Example Domain');+ expect(request.userData.title).to.be.eql(request.url[request.url.length - 1]);}await Apify.utils.sleep(10); // Wait for event loop to unwind.await cheerioCrawler.run();- expect(processed.length).to.be.within(30, 33);+ expect(processed.length).to.be.within(100, 105);expect(failed).to.have.lengthOf(0);expect(new Set(processed.map(p => p.url))).to.be.eql(new Set(sources.map(s => s.url)));processed.forEach((request) => {- expect(request.userData.title).to.be.eql('Example Domain');+ expect(request.userData.title).to.be.eql(request.url[request.url.length - 1]);});});});