Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Added option for adding content type to CheerioCrawler
drobnikj60de0a76cb36
diff --git a/package.json b/package.jsonindex 818c71bcd..33a839120 100644--- a/package.json+++ b/package.json@@ -54,6 +54,7 @@"apify-client": "^0.5.25","apify-shared": "^0.1.56","cheerio": "^1.0.0-rc.3",+ "concat-stream": "^2.0.0","content-type": "^1.0.4","express": "^4.17.1","fs-extra": "^8.1.0",@@ -89,6 +90,7 @@"chai-subset": "^1.6.0","csv-parse": "^4.4.3","eslint": "^5.16.0",+ "eslint-plugin-import": "^2.18.2","fpcollect": "^1.0.4","fpscanner": "^0.1.5","jsdoc": "^3.5.5",diff --git a/src/crawlers/cheerio_crawler.js b/src/crawlers/cheerio_crawler.jsindex 60fac93ef..aabf3a640 100644--- a/src/crawlers/cheerio_crawler.js+++ b/src/crawlers/cheerio_crawler.js@@ -7,12 +7,17 @@ import log from 'apify-shared/log';import { checkParamOrThrow } from 'apify-client/build/utils';import contentType from 'content-type';import readStreamToString from '@apify/http-request/src/read_stream_to_string';+import concatStream from 'concat-stream';import BasicCrawler from './basic_crawler';import { addTimeoutToPromise } from '../utils';import { getApifyProxyUrl } from '../actor';import { BASIC_CRAWLER_TIMEOUT_MULTIPLIER } from '../constants';import { requestAsBrowser } from '../utils_request';+/**+ * Default content types, which CheerioScraper accepts.+ */+const DEFAULT_CONTENT_TYPES = ['text/html'];const DEFAULT_OPTIONS = {requestTimeoutSecs: 30,@@ -86,7 +91,7 @@ const DEFAULT_OPTIONS = {* // Crawl the URLs* const crawler = new Apify.CheerioCrawler({* requestList,- * handlePageFunction: async ({ request, response, html, $ }) => {+ * handlePageFunction: async ({ request, response, body, html, $ }) => {* const data = [];** // Do some data extraction from the page with Cheerio.@@ -115,13 +120,20 @@ const DEFAULT_OPTIONS = {* The function receives the following object as an argument:* ```* {- * $: Cheerio, // the Cheerio object with parsed HTML+ * $: Cheerio, // the Cheerio object with parsed HTML or XML* html: String // the raw HTML of the page, lazy loaded only when used+ * body: String|Object|Buffer // the request body of the web page* request: Request,* response: Object // An instance of Node's http.IncomingMessage object,* autoscaledPool: AutoscaledPool* }* ```+ * Cheerio is available only for HTML and XML content types.+ * Type of `body` depends of content type of crawling URL.+ * - String: In case of HTML of XML content types+ * - Object: In case of JSON content type+ * - Buffer: For the rest content types+ ** With the {@link Request} object representing the URL to crawl.** If the function returns a promise, it is awaited by the crawler.@@ -217,6 +229,9 @@ const DEFAULT_OPTIONS = {** See <a href="https://github.com/apifytech/apify-js/blob/master/src/crawlers/cheerio_crawler.js#L13">source code</a>* for the default implementation of this function.+ * @param {String[]} [options.additionalContentTypes]+ * An array of content types you want to process.+ * By default only text/html content type is included in scraping.* @param {Number} [options.maxRequestRetries=3]* Indicates how many times the request is retried if either `requestFunction` or `handlePageFunction` fails.* @param {Number} [options.maxRequestsPerCrawl]@@ -249,6 +264,7 @@ class CheerioCrawler {apifyProxyGroups,apifyProxySession,proxyUrls,+ additionalContentTypes,// Autoscaled pool shorthandsminConcurrency,@@ -274,10 +290,15 @@ class CheerioCrawler {checkParamOrThrow(apifyProxySession, 'options.apifyProxySession', 'Maybe String');checkParamOrThrow(proxyUrls, 'options.proxyUrls', 'Maybe [String]');checkParamOrThrow(prepareRequestFunction, 'options.prepareRequestFunction', 'Maybe Function');+ checkParamOrThrow(apifyProxyGroups, 'options.additionalContentTypes', 'Maybe [String]');// Enforce valid proxy configurationif (proxyUrls && !proxyUrls.length) throw new Error('Parameter "options.proxyUrls" of type Array must not be empty');if (useApifyProxy && proxyUrls) throw new Error('Cannot combine "options.useApifyProxy" with "options.proxyUrls"!');+ // Check content types+ this.includedContentTypes = DEFAULT_CONTENT_TYPES;+ if (additionalContentTypes) this._extendsDefaultContentTypes(additionalContentTypes);+this.requestOptions = requestOptions;this.handlePageFunction = handlePageFunction;this.handlePageTimeoutMillis = handlePageTimeoutSecs * 1000;@@ -328,7 +349,7 @@ class CheerioCrawler {*/async _handleRequestFunction({ request, autoscaledPool }) {if (this.prepareRequestFunction) await this.prepareRequestFunction({ request });- const { dom, responseStream: response } = await addTimeoutToPromise(+ const { dom, xmlMode, body, responseStream: response } = await addTimeoutToPromise(this._requestFunction({ request }),this.requestTimeoutMillis,'CheerioCrawler: requestFunction timed out.',@@ -336,13 +357,19 @@ class CheerioCrawler {request.loadedUrl = response.url;- const $ = cheerio.load(dom);+ const $ = dom ? cheerio.load(dom, { xmlMode }) : undefined;const context = {$,// Using a getter here not to break the original API// and lazy load the HTML only when needed.get html() {- return $.html({ decodeEntities: false });+ return dom && !xmlMode ? $.html({ decodeEntities: false }) : undefined;+ },+ get body() {+ if (dom) {+ return (xmlMode) ? $.xml() : $.html({ decodeEntities: false });+ }+ return body;},request,response,@@ -368,8 +395,8 @@ class CheerioCrawler {const responseStream = await requestAsBrowser(opts);const { statusCode, headers } = responseStream;+ const { type, encoding } = contentType.parse(headers['content-type']);if (statusCode >= 500) {- const { type, encoding } = contentType.parse(headers['content-type']);const body = await readStreamToString(responseStream, encoding);// Errors are often sent as JSON, so attempt to parse them,@@ -383,9 +410,16 @@ class CheerioCrawler {// It's not a JSON so it's probably some text. Get the first 100 chars of it.throw new Error(`CheerioCrawler: ${statusCode} - Internal Server Error: ${body.substr(0, 100)}`);- } else {+ } else if (type === 'text/html' || type === 'application/xhtml+xml' || type === 'application/xml') {const dom = await this._parseHtmlToDom(responseStream);- return ({ dom, responseStream });+ return ({ dom, xmlMode: type.includes('xml'), responseStream });+ } else if (type === 'application/json') {+ const jsonString = await readStreamToString(responseStream, encoding);+ const body = JSON.parse(jsonString);+ return { body, responseStream };+ } else {+ const body = await this._concatStreamToBuffer(responseStream);+ return { body, responseStream };}}@@ -413,9 +447,10 @@ class CheerioCrawler {throw new Error(`CheerioCrawler: Resource ${request.url} is not available in HTML format. Skipping resource.`);}- if (type.toLowerCase() !== 'text/html' && statusCode < 500) {+ if (!this.includedContentTypes.includes(type) && statusCode < 500) {request.noRetry = true;- throw new Error(`CheerioCrawler: Resource ${request.url} served Content-Type ${type} instead of text/html. Skipping resource.`);+ throw new Error(`CheerioCrawler: Resource ${request.url} served Content-Type ${type}, `+ + `but only ${this.includedContentTypes.join(', ')} are allowed. Skipping resource.`);}return false;@@ -458,6 +493,36 @@ class CheerioCrawler {response.on('error', reject).pipe(parser);});}++ /**+ *+ * @param readStream+ * @return {Promise<unknown>}+ * @ignore+ */+ async _concatStreamToBuffer(readStream) {+ return new Promise((resolve, reject) => {+ const resultStream = concatStream(resolve);+ readStream.on('error', reject).pipe(resultStream);+ });+ }++ /**+ *+ * @param contentTypes+ * @return {*}+ * @ignore+ */+ _extendsDefaultContentTypes(contentTypes) {+ const normalizedTypes = contentTypes.map((type) => {+ try {+ return contentType.parse(type).type;+ } catch (err) {+ throw new Error(`CheerioCrawler: Content type ${type} in not correct content type.`);+ }+ });+ this.includedContentTypes.push(...normalizedTypes);+ }}diff --git a/test/crawlers/cheerio_crawler.js b/test/crawlers/cheerio_crawler.jsindex 75f9c53e0..c3c695c72 100644--- a/test/crawlers/cheerio_crawler.js+++ b/test/crawlers/cheerio_crawler.js@@ -1,6 +1,8 @@import { Readable } from 'stream';import EventEmitter from 'events';import rqst from 'request';+import fs from 'fs';+import path from 'path';import { expect } from 'chai';import log from 'apify-shared/log';import { delayPromise } from 'apify-shared/utilities';@@ -35,25 +37,25 @@ describe('CheerioCrawler', () => {let server;let port;- async function getRequestListForMock(mockData, path = 'mock') {+ async function getRequestListForMock(mockData, pathName = 'mock') {const sources = [{- url: `http://${HOST}:${port}/${path}?a=1`,+ url: `http://${HOST}:${port}/${pathName}?a=1`,payload: JSON.stringify(mockData),method: 'POST',headers: { 'Content-Type': 'application/json', Accept: 'application/json' },}, {- url: `http://${HOST}:${port}/${path}?a=2`,+ url: `http://${HOST}:${port}/${pathName}?a=2`,payload: JSON.stringify(mockData),method: 'POST',headers: { 'Content-Type': 'application/json', Accept: 'application/json' },}, {- url: `http://${HOST}:${port}/${path}?a=3`,+ url: `http://${HOST}:${port}/${pathName}?a=3`,payload: JSON.stringify(mockData),method: 'POST',headers: { 'Content-Type': 'application/json', Accept: 'application/json' },}, {- url: `http://${HOST}:${port}/${path}?a=4`,+ url: `http://${HOST}:${port}/${pathName}?a=4`,payload: JSON.stringify(mockData),method: 'POST',headers: { 'Content-Type': 'application/json', Accept: 'application/json' },@@ -110,6 +112,24 @@ describe('CheerioCrawler', () => {res.send('DATA');});+ app.get('/json-type', (req, res) => {+ res.json({ foo: 'bar' });+ });+ app.get('/xml-type', (req, res) => {+ res.type('application/xml');+ res.send('<?xml version="1.0" encoding="UTF-8"?>\n'+ + '<items>\n'+ + '<item>\n'+ + ' <url>https://apify.com</url>\n'+ + ' <title>Web Scraping, Data Extraction and Automation · Apify</title>\n'+ + '</item>\n'+ + '</items>');+ });+ app.get('/image-type', (req, res) => {+ res.type('image/png');+ res.send(fs.readFileSync(path.join(__dirname, 'data/apify.png')));+ });+server = await startExpressAppPromise(app, 0);port = server.address().port; //eslint-disable-line});@@ -342,7 +362,8 @@ describe('CheerioCrawler', () => {expect(handlePageInvocationCount).to.be.eql(0);expect(errorMessages).to.have.lengthOf(4);- errorMessages.forEach(msg => expect(msg).to.include('served Content-Type application/json instead of text/html. Skipping resource.'));+ errorMessages.forEach(msg => expect(msg)+ .to.include('Content-Type application/json, but only text/html are allowed. Skipping resource.'));expect(chunkReadCount).to.be.eql(0);});@@ -501,6 +522,51 @@ describe('CheerioCrawler', () => {});});+ describe('should works with all content types from option.additionalContentTypes', () => {+ const handlePageInvocationParams = [];+ let handleFailedInvocationCount = 0;+ before(async () => {+ const sources = [+ { url: `http://${HOST}:${port}/json-type` },+ { url: `http://${HOST}:${port}/xml-type` },+ { url: `http://${HOST}:${port}/image-type` },+ ];+ const requestList = new Apify.RequestList({ sources });+ await requestList.initialize();+ const crawler = new Apify.CheerioCrawler({+ requestList,+ additionalContentTypes: ['application/json', 'application/xml', 'image/png'],+ maxRequestRetries: 1,+ handlePageFunction: async (params) => {+ handlePageInvocationParams.push(params);+ },+ handleFailedRequestFunction: async () => {+ handleFailedInvocationCount++;+ },+ });+ await crawler.run();++ expect(handleFailedInvocationCount).to.be.eql(0);+ expect(handlePageInvocationParams.length).to.be.eql(sources.length);+ });+ it('when response is application/json', async () => {+ const jsonRequestParams = handlePageInvocationParams[0];+ expect(jsonRequestParams.body).to.be.an('object');+ expect(jsonRequestParams.$).to.be.an('null');+ });+ it('when response is application/xml', async () => {+ const xmlRequestParams = handlePageInvocationParams[1];+ expect(xmlRequestParams.body).to.be.an('string');+ expect(xmlRequestParams.html).to.be.an('null');+ expect(xmlRequestParams.$).to.be.an('function');+ });+ it('when response is image/png', async () => {+ const imageRequestParams = handlePageInvocationParams[2];+ expect(imageRequestParams.body).to.be.instanceof(Buffer);+ expect(imageRequestParams.html).to.be.an('null');+ });+ });+describe('proxy', () => {let requestList;beforeEach(async () => {