Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Added XML to default content type for CheerioCrawler, fixed tests
drobnikjf812be2d13af
diff --git a/src/crawlers/cheerio_crawler.js b/src/crawlers/cheerio_crawler.jsindex 05443c154..345131001 100644--- a/src/crawlers/cheerio_crawler.js+++ b/src/crawlers/cheerio_crawler.js@@ -15,9 +15,9 @@ import { BASIC_CRAWLER_TIMEOUT_MULTIPLIER } from '../constants';import { requestAsBrowser } from '../utils_request';/**- * Default content types, which CheerioScraper accepts.+ * Default content types, which CheerioScraper supports.*/-const DEFAULT_CONTENT_TYPES = ['text/html'];+const DEFAULT_CONTENT_TYPES = ['text/html', 'application/xml', 'application/xhtml+xml'];const DEFAULT_OPTIONS = {requestTimeoutSecs: 30,@@ -66,7 +66,7 @@ const DEFAULT_OPTIONS = {** The crawler finishes when there are no more {@link Request} objects to crawl.*- * By default, `CheerioCrawler` downloads HTML using the+ * By default, `CheerioCrawler` downloads HTML or XML using the* <a href="https://www.npmjs.com/package/request" target="_blank">request</a> NPM package.* You can use the `requestOptions` parameter to pass additional options to `request`.*@@ -232,7 +232,7 @@ const DEFAULT_OPTIONS = {* for the default implementation of this function.* @param {String[]} [options.additionalContentTypes]* An array of content types you want to process.- * By default only text/html content type is included in scraping.+ * By default `text/html`, `application/xml`, `application/xhtml+xml` content types are supported.* @param {Number} [options.maxRequestRetries=3]* Indicates how many times the request is retried if either `requestFunction` or `handlePageFunction` fails.* @param {Number} [options.maxRequestsPerCrawl]@@ -297,8 +297,8 @@ class CheerioCrawler {if (useApifyProxy && proxyUrls) throw new Error('Cannot combine "options.useApifyProxy" with "options.proxyUrls"!');// Check content types- this.includedContentTypes = DEFAULT_CONTENT_TYPES;- if (additionalContentTypes) this._extendsDefaultContentTypes(additionalContentTypes);+ this.supportedContentTypes = DEFAULT_CONTENT_TYPES;+ if (additionalContentTypes) this._extendSupportedContentTypes(additionalContentTypes);this.requestOptions = requestOptions;this.handlePageFunction = handlePageFunction;@@ -386,7 +386,7 @@ class CheerioCrawler {/*** Function to make the HTTP request. It performs optimizations* on the request such as only downloading the request body if the- * received content type matches text/html.+ * received content type matches text/html, application/xml, application/xhtml+xml.* @ignore*/async _requestFunction({ request }) {@@ -448,10 +448,10 @@ class CheerioCrawler {throw new Error(`CheerioCrawler: Resource ${request.url} is not available in HTML format. Skipping resource.`);}- if (!this.includedContentTypes.includes(type) && statusCode < 500) {+ if (!this.supportedContentTypes.includes(type) && statusCode < 500) {request.noRetry = true;throw new Error(`CheerioCrawler: Resource ${request.url} served Content-Type ${type}, `- + `but only ${this.includedContentTypes.join(', ')} are allowed. Skipping resource.`);+ + `but only ${this.supportedContentTypes.join(', ')} are allowed. Skipping resource.`);}return false;@@ -508,15 +508,15 @@ class CheerioCrawler {}/**- * Checks and extends included content types to crawler+ * Checks and extends supported content types* @param contentTypes* @ignore*/- _extendsDefaultContentTypes(contentTypes) {+ _extendSupportedContentTypes(contentTypes) {contentTypes.forEach((type) => {try {const parsedType = contentType.parse(type);- this.includedContentTypes.push(parsedType.type);+ this.supportedContentTypes.push(parsedType.type);} catch (err) {throw new Error(`CheerioCrawler: Can not parse content type ${type} from "options.additionalContentTypes".`);}diff --git a/test/crawlers/cheerio_crawler.js b/test/crawlers/cheerio_crawler.jsindex b2ca4fa3a..0ca68b57c 100644--- a/test/crawlers/cheerio_crawler.js+++ b/test/crawlers/cheerio_crawler.js@@ -77,6 +77,19 @@ describe('CheerioCrawler', () => {return requestList;}+ const responseSamples = {+ json: { foo: 'bar' },+ xml: '<?xml version="1.0" encoding="UTF-8"?>\n'+ + '<items>\n'+ + '<item>\n'+ + ' <url>https://apify.com</url>\n'+ + ' <title>Web Scraping, Data Extraction and Automation · Apify</title>\n'+ + '</item>\n'+ + '</items>',+ image: fs.readFileSync(path.join(__dirname, 'data/apify.png')),+ };++before(async () => {logLevel = log.getLevel();log.setLevel(log.LEVELS.ERROR);@@ -113,21 +126,15 @@ describe('CheerioCrawler', () => {});app.get('/json-type', (req, res) => {- res.json({ foo: 'bar' });+ res.json(responseSamples.json);});app.get('/xml-type', (req, res) => {res.type('application/xml');- res.send('<?xml version="1.0" encoding="UTF-8"?>\n'- + '<items>\n'- + '<item>\n'- + ' <url>https://apify.com</url>\n'- + ' <title>Web Scraping, Data Extraction and Automation · Apify</title>\n'- + '</item>\n'- + '</items>');+ res.send(responseSamples.xml);});app.get('/image-type', (req, res) => {res.type('image/png');- res.send(fs.readFileSync(path.join(__dirname, 'data/apify.png')));+ res.send(responseSamples.image);});server = await startExpressAppPromise(app, 0);@@ -363,7 +370,8 @@ describe('CheerioCrawler', () => {expect(handlePageInvocationCount).to.be.eql(0);expect(errorMessages).to.have.lengthOf(4);errorMessages.forEach(msg => expect(msg)- .to.include('Content-Type application/json, but only text/html are allowed. Skipping resource.'));+ .to.include('Content-Type application/json, but only text/html, application/xml, '+ + 'application/xhtml+xml are allowed. Skipping resource.'));expect(chunkReadCount).to.be.eql(0);});@@ -522,7 +530,30 @@ describe('CheerioCrawler', () => {});});- describe('should works with all content types from option.additionalContentTypes', () => {+ it('should work with all defaults content types', async () => {+ let handledRequests = 0;+ const contentTypes = ['text/html', 'application/xml', 'application/xhtml+xml'];+ const sources = contentTypes.map(contentType => ({+ url: `http://${HOST}:${port}/mock?ct=${contentType}`,+ payload: JSON.stringify({ headers: { 'Content-Type': contentType }, statusCode: 200 }),+ method: 'POST',+ headers: { 'Content-Type': 'application/json' },+ }));+ const requestList = new Apify.RequestList({+ sources,+ });+ await requestList.initialize();+ const crawler = new Apify.CheerioCrawler({+ requestList,+ handlePageFunction: async () => {+ handledRequests++;+ },+ });+ await crawler.run();+ expect(handledRequests).to.be.eql(contentTypes.length);+ });++ describe('should works with all content types from options.additionalContentTypes', () => {const handlePageInvocationParams = [];let handleFailedInvocationCount = 0;before(async () => {@@ -535,7 +566,7 @@ describe('CheerioCrawler', () => {await requestList.initialize();const crawler = new Apify.CheerioCrawler({requestList,- additionalContentTypes: ['application/json', 'application/xml', 'image/png'],+ additionalContentTypes: ['application/json', 'image/png'],maxRequestRetries: 1,handlePageFunction: async (params) => {handlePageInvocationParams.push(params);@@ -552,15 +583,18 @@ describe('CheerioCrawler', () => {it('when response is application/json', async () => {const jsonRequestParams = handlePageInvocationParams[0];expect(jsonRequestParams.body).to.be.an('object');+ expect(jsonRequestParams.body).to.be.eql(responseSamples.json);});it('when response is application/xml', async () => {const xmlRequestParams = handlePageInvocationParams[1];expect(xmlRequestParams.body).to.be.an('string');+ expect(xmlRequestParams.body).to.be.eql(responseSamples.xml);expect(xmlRequestParams.$).to.be.an('function');});it('when response is image/png', async () => {const imageRequestParams = handlePageInvocationParams[2];expect(imageRequestParams.body).to.be.instanceof(Buffer);+ expect(imageRequestParams.body).to.be.eql(responseSamples.image);});});