Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Add test
Ondra Urban8801c01a7952
diff --git a/src/cheerio_crawler.js b/src/cheerio_crawler.jsindex b0787920e..d643ded7a 100644--- a/src/cheerio_crawler.js+++ b/src/cheerio_crawler.js@@ -3,10 +3,14 @@ import _ from 'underscore';import cheerio from 'cheerio';import { checkParamOrThrow } from 'apify-client/build/utils';import BasicCrawler from './basic_crawler';+import { isPromise } from './utils';const DEFAULT_OPTIONS = {requestFunction: ({ request }) => rp({ url: request.url, method: request.method, headers: request.headers }),pageOpsTimeoutMillis: 300000,+ requestOptions: {+ ignoreSslErrors: true,+ },};export default class CheerioCrawler {@@ -35,7 +39,9 @@ export default class CheerioCrawler {checkParamOrThrow(requestFunction, 'opts.requestFunction', 'Function');checkParamOrThrow(handlePageFunction, 'opts.handlePageFunction', 'Function');checkParamOrThrow(handleFailedRequestFunction, 'opts.handleFailedRequestFunction', 'Maybe Function');- checkParamOrThrow(requestOptions, 'opts.requestOptions', 'Maybe Object');+ checkParamOrThrow(requestOptions, 'opts.requestOptions', 'Object');++ if (requestOptions.ignoreSslErrors) process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0';this.handlePageFunction = handlePageFunction;this.requestFunction = requestFunction;diff --git a/test/cheerio_crawler.js b/test/cheerio_crawler.jsnew file mode 100644index 000000000..960cbf825--- /dev/null+++ b/test/cheerio_crawler.js@@ -0,0 +1,47 @@+import chai, { expect } from 'chai';+import chaiAsPromised from 'chai-as-promised';+import * as Apify from '../build/index';++chai.use(chaiAsPromised);++describe('CheerioCrawler', () => {+ it('should work', async () => {+ const sources = [+ { url: 'http://example.com/?q=1' },+ { url: 'http://example.com/?q=2' },+ { url: 'http://example.com/?q=3' },+ { url: 'http://example.com/?q=4' },+ { url: 'http://example.com/?q=5' },+ { url: 'http://example.com/?q=6' },+ ];+ const processed = [];+ const failed = [];+ const requestList = new Apify.RequestList({ sources });+ const handlePageFunction = async ({ $, html, request }) => {+ request.userData.title = $('title').text();+ request.userData.html = html;+ processed.push(request);+ };++ const cheerioCrawler = new Apify.CheerioCrawler({+ requestList,+ minConcurrency: 1,+ maxConcurrency: 1,+ handlePageFunction,+ handleFailedRequestFunction: ({ request }) => failed.push(request),+ });++ await requestList.initialize();+ await cheerioCrawler.run();++ expect(processed).to.have.lengthOf(6);+ expect(failed).to.have.lengthOf(0);++ processed.forEach((request, id) => {+ expect(request.url).to.be.eql(sources[id].url);+ expect(request.userData.title).to.be.eql('Example Domain');+ expect(request.userData.html).to.be.a('string');+ expect(request.userData.html.length).not.to.be.eql(0);+ });+ });+});