Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Improve content-type matching implementation, add tests
Ondra Urbana4bf69b635e6
diff --git a/src/cheerio_crawler.js b/src/cheerio_crawler.jsindex 488fb44ce..a0f21f5a2 100644--- a/src/cheerio_crawler.js+++ b/src/cheerio_crawler.js@@ -244,6 +244,10 @@ class CheerioCrawler {maxConcurrency,autoscaledPoolOptions,});++ // Set Request as an internal property to enable mocking in tests,+ // since Sinon can't mock it due to it being a Function.+ this.rqst = rqst;}/**@@ -318,57 +322,62 @@ class CheerioCrawler {return new Promise((resolve, reject) => {// Using the streaming API of Request to be able to// handle the response based on headers receieved.- rqst(this._getRequestOptions(request))+ this.rqst(this._getRequestOptions(request)).on('response', async (res) => {- // Throw away the body if it's not loadable by Cheerio.- // Request will not be retried since it's safe to assume+ // First check what kind of response we received.let cType;try {cType = contentType.parse(res);} catch (err) {- log.info(`CheerioCrawler: Received invalid Content-Type for URL: ${request.url} Skipping.`);- res.destroy();- return resolve(null);- }- if (cType.type !== 'text/html') {- log.info(`CheerioCrawler: Received Content-Type: ${cType.type} for URL: ${request.url} Skipping.`);res.destroy();- return resolve(null);+ // No reason to parse the body if the Content-Type header is invalid.+ return reject(new Error(`CheerioCrawler: Invalid Content-Type header for URL: ${request.url}`));}- // Read the body into a string since Cheerio does not support streaming+ const { type, encoding } = cType;+ // Read the body into a string since Cheerio does not support streaming.let body;try {- body = await this._readStreamIntoString(res, cType.encoding);+ body = await this._readStreamIntoString(res, encoding);} catch (err) {- // Error in reading the body+ // Error in reading the body.return reject(err);}// 500 codes are handled as errors, requests will be retried.- // Some message extraction is attempted.- if (res.statusCode >= 500) {- try {- // Error responses are often JSON so attempt to parse it.+ const status = res.statusCode;+ if (status >= 500) {+ // Errors are often sent as JSON, so attempt to parse them,+ // despite Accept header being set to text/html.+ if (type === 'application/json') {const errorResponse = JSON.parse(body);let { message } = errorResponse;if (!message) message = util.inspect(errorResponse, { depth: 1, maxArrayLength: 10 });- return reject(new Error(message));- } catch (err) {- // If not a JSON, just take the string and trim it to 100 characters.- return reject(new Error(body.substr(0, 100)));+ return reject(new Error(`${status} - ${message}`));}+ // It's not a JSON so it's probably some text. Get the first 100 chars of it.+ return reject(new Error(`${status} - Internal Server Error: ${body.substr(0, 100)}`));+ }++ // Handle situations where the server explicitly states that+ // it will not serve the resource as text/html by skipping.+ if (status === 406) {+ return log.error(`CheerioCrawler: Resource ${request.url} is not available in HTML format. Skipping resource.`);}- // 200-499 responses are considered OK and will not be retried- resolve(body);+ // Other 200-499 responses are considered OK, but first check the content type.+ if (type === 'text/html') {+ resolve(body);+ } else {+ log.error(`CheerioCrawler: Resource ${request.url} served Content-Type ${type} instead of text/html. Skipping resource.`);+ }}).on('error', err => reject(err));});}/**- * Combines the provided `requestOptions` with default values.+ * Combines the provided `requestOptions` with mandatory (non-overridable) values.* @param {Request} request* @ignore*/@@ -405,6 +414,15 @@ class CheerioCrawler {return null;}+ /**+ * Flushes the provided stream into a Buffer and transforms+ * it to a String using the provided encoding or utf-8 as default.+ *+ * @param {stream.Readable} stream+ * @param {String} [encoding]+ * @returns {Promise<String>}+ * @private+ */async _readStreamIntoString(stream, encoding) { // eslint-disable-line class-methods-use-thisreturn new Promise((resolve, reject) => {const chunks = [];diff --git a/test/cheerio_crawler.js b/test/cheerio_crawler.jsindex 6630b3195..7ee598712 100644--- a/test/cheerio_crawler.js+++ b/test/cheerio_crawler.js@@ -1,3 +1,5 @@+import { Readable } from 'stream';+import EventEmitter from 'events';import chai, { expect } from 'chai';import chaiAsPromised from 'chai-as-promised';import log from 'apify-shared/log';@@ -8,6 +10,7 @@ import Apify from '../build/index';chai.use(chaiAsPromised);+/* eslint-disable no-underscore-dangle */describe('CheerioCrawler', () => {const comparator = (a, b) => {a = Number(/q=(\d+)$/.exec(a.url)[1]);@@ -260,6 +263,130 @@ describe('CheerioCrawler', () => {});});+ describe('should ensure text/html Content-Type', () => {+ let requestList;+ beforeEach(async () => {+ requestList = new Apify.RequestList({+ sources: [+ { url: 'http://example.com/?q=0' },+ { url: 'http://example.com/?q=1' },+ { url: 'http://example.com/?q=2' },+ { url: 'http://example.com/?q=3' },+ ],+ });+ await requestList.initialize();+ });++ afterEach(() => {+ requestList = null;+ });++ it('by setting a correct Accept header', async () => {+ const headers = [];+ const crawler = new Apify.CheerioCrawler({+ requestList,+ handlePageFunction: async () => {},+ requestFunction: async ({ request }) => {+ const opts = crawler._getRequestOptions(request);+ headers.push(opts.headers);+ // it needs to return something valid+ return 'html';+ },+ });++ await crawler.run();+ headers.forEach(h => expect(h.Accept).to.be.eql('text/html'));+ });++ describe('by throwing', () => {+ let crawler;+ let handlePageInvocationCount = 0;+ let errorMessages = [];+ beforeEach(() => {+ // log.setLevel(log.LEVELS.OFF);+ crawler = new Apify.CheerioCrawler({+ requestList,+ maxRequestRetries: 1,+ handlePageFunction: async () => {+ handlePageInvocationCount++;+ },+ handleFailedRequestFunction: async ({ request }) => {+ errorMessages = [...errorMessages, ...request.errorMessages];+ },+ });+ });+ afterEach(async () => {+ // log.setLevel(log.LEVELS.ERROR);+ crawler = null;+ handlePageInvocationCount = 0;+ errorMessages = [];+ });+++ it('when invalid Content-Type header is received', async () => {+ // Mock Request to inject invalid response headers.+ crawler.rqst = () => {+ const response = new Readable({+ read() {+ this.push('x');+ this.push(null);+ },+ });+ response.headers = {+ 'content-type': '000',+ };++ const ee = new EventEmitter();++ setTimeout(() => {+ ee.emit('response', response);+ }, 0);++ return ee;+ };++ await crawler.run();++ expect(handlePageInvocationCount).to.be.eql(0);+ expect(errorMessages).to.have.lengthOf(8);+ errorMessages.forEach(msg => expect(msg).to.include('Invalid Content-Type header'));+ });++ it('when response stream emits an error event', async () => {+ // Mock Request to inject invalid response headers.+ crawler.rqst = () => {+ const start = Date.now();+ const response = new Readable({+ read() {+ if (Date.now() > start + 1) {+ this.emit('error', new Error('Error in stream.'));+ return;+ }+ this.push('x');+ },+ });+ response.headers = {+ 'content-type': 'text/html',+ };++ const ee = new EventEmitter();++ setTimeout(() => {+ ee.emit('response', response);+ }, 0);++ return ee;+ };++ await crawler.run();++ expect(handlePageInvocationCount).to.be.eql(0);+ expect(errorMessages).to.have.lengthOf(8);+ errorMessages.forEach(msg => expect(msg).to.include('Error in stream.'));+ });+ });+ });+describe('proxy', () => {let requestList;beforeEach(async () => {@@ -278,7 +405,6 @@ describe('CheerioCrawler', () => {requestList = null;});- /* eslint-disable no-underscore-dangle */it('should work with proxyUrls array', async () => {const proxies = [];const crawler = new Apify.CheerioCrawler({