Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Cheerio Scraper: Removed application/xml from default mine types
drobnikjb0c7a3cfba61
diff --git a/src/crawlers/cheerio_crawler.js b/src/crawlers/cheerio_crawler.jsindex 1f93f0c43..caf5c3d20 100644--- a/src/crawlers/cheerio_crawler.js+++ b/src/crawlers/cheerio_crawler.js@@ -17,7 +17,7 @@ import { requestAsBrowser } from '../utils_request';/*** Default mine types, which CheerioScraper supports.*/-const DEFAULT_MINE_TYPES = ['text/html', 'application/xml', 'application/xhtml+xml'];+const DEFAULT_MINE_TYPES = ['text/html', 'application/xhtml+xml'];const DEFAULT_OPTIONS = {requestTimeoutSecs: 30,@@ -233,7 +233,7 @@ const DEFAULT_OPTIONS = {* @param {String[]} [options.additionalMineTypes]* An array of <a href="https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Complete_list_of_MIME_types"* target="_blank">mine types</a> you want to process.- * By default `text/html`, `application/xml`, `application/xhtml+xml` mine types are supported.+ * By default `text/html`, `application/xhtml+xml` mine types are supported.* @param {Number} [options.maxRequestRetries=3]* Indicates how many times the request is retried if either `requestFunction` or `handlePageFunction` fails.* @param {Number} [options.maxRequestsPerCrawl]diff --git a/test/crawlers/cheerio_crawler.js b/test/crawlers/cheerio_crawler.jsindex 3baff7999..6864fd3c5 100644--- a/test/crawlers/cheerio_crawler.js+++ b/test/crawlers/cheerio_crawler.js@@ -370,7 +370,7 @@ describe('CheerioCrawler', () => {expect(handlePageInvocationCount).to.be.eql(0);expect(errorMessages).to.have.lengthOf(4);errorMessages.forEach(msg => expect(msg)- .to.include('Content-Type application/json, but only text/html, application/xml, '+ .to.include('Content-Type application/json, but only text/html, '+ 'application/xhtml+xml are allowed. Skipping resource.'));expect(chunkReadCount).to.be.eql(0);});@@ -532,7 +532,7 @@ describe('CheerioCrawler', () => {it('should work with all defaults content types', async () => {let handledRequests = 0;- const contentTypes = ['text/html', 'application/xml', 'application/xhtml+xml'];+ const contentTypes = ['text/html', 'application/xhtml+xml'];const sources = contentTypes.map(contentType => ({url: `http://${HOST}:${port}/mock?ct=${contentType}`,payload: JSON.stringify({ headers: { 'Content-Type': contentType }, statusCode: 200 }),@@ -566,7 +566,7 @@ describe('CheerioCrawler', () => {await requestList.initialize();const crawler = new Apify.CheerioCrawler({requestList,- additionalMineTypes: ['application/json', 'image/png'],+ additionalMineTypes: ['application/json', 'image/png', 'application/xml'],maxRequestRetries: 1,handlePageFunction: async (params) => {handlePageInvocationParams.push(params);