Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Add test

Ondra Urban8801c01a7952
diff --git a/src/cheerio_crawler.js b/src/cheerio_crawler.js
index b0787920e..d643ded7a 100644
--- a/src/cheerio_crawler.js
+++ b/src/cheerio_crawler.js
@@ -3,10 +3,14 @@ import _ from 'underscore';
import cheerio from 'cheerio';
import { checkParamOrThrow } from 'apify-client/build/utils';
import BasicCrawler from './basic_crawler';
+import { isPromise } from './utils';
const DEFAULT_OPTIONS = {
requestFunction: ({ request }) => rp({ url: request.url, method: request.method, headers: request.headers }),
pageOpsTimeoutMillis: 300000,
+ requestOptions: {
+ ignoreSslErrors: true,
+ },
};
export default class CheerioCrawler {
@@ -35,7 +39,9 @@ export default class CheerioCrawler {
checkParamOrThrow(requestFunction, 'opts.requestFunction', 'Function');
checkParamOrThrow(handlePageFunction, 'opts.handlePageFunction', 'Function');
checkParamOrThrow(handleFailedRequestFunction, 'opts.handleFailedRequestFunction', 'Maybe Function');
- checkParamOrThrow(requestOptions, 'opts.requestOptions', 'Maybe Object');
+ checkParamOrThrow(requestOptions, 'opts.requestOptions', 'Object');
+
+ if (requestOptions.ignoreSslErrors) process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0';
this.handlePageFunction = handlePageFunction;
this.requestFunction = requestFunction;
diff --git a/test/cheerio_crawler.js b/test/cheerio_crawler.js
new file mode 100644
index 000000000..960cbf825
--- /dev/null
+++ b/test/cheerio_crawler.js
@@ -0,0 +1,47 @@
+import chai, { expect } from 'chai';
+import chaiAsPromised from 'chai-as-promised';
+import * as Apify from '../build/index';
+
+chai.use(chaiAsPromised);
+
+describe('CheerioCrawler', () => {
+ it('should work', async () => {
+ const sources = [
+ { url: 'http://example.com/?q=1' },
+ { url: 'http://example.com/?q=2' },
+ { url: 'http://example.com/?q=3' },
+ { url: 'http://example.com/?q=4' },
+ { url: 'http://example.com/?q=5' },
+ { url: 'http://example.com/?q=6' },
+ ];
+ const processed = [];
+ const failed = [];
+ const requestList = new Apify.RequestList({ sources });
+ const handlePageFunction = async ({ $, html, request }) => {
+ request.userData.title = $('title').text();
+ request.userData.html = html;
+ processed.push(request);
+ };
+
+ const cheerioCrawler = new Apify.CheerioCrawler({
+ requestList,
+ minConcurrency: 1,
+ maxConcurrency: 1,
+ handlePageFunction,
+ handleFailedRequestFunction: ({ request }) => failed.push(request),
+ });
+
+ await requestList.initialize();
+ await cheerioCrawler.run();
+
+ expect(processed).to.have.lengthOf(6);
+ expect(failed).to.have.lengthOf(0);
+
+ processed.forEach((request, id) => {
+ expect(request.url).to.be.eql(sources[id].url);
+ expect(request.userData.title).to.be.eql('Example Domain');
+ expect(request.userData.html).to.be.a('string');
+ expect(request.userData.html.length).not.to.be.eql(0);
+ });
+ });
+});