Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Improved comments
drobnikje155d1261683
diff --git a/src/crawlers/cheerio_crawler.js b/src/crawlers/cheerio_crawler.jsindex aabf3a640..05443c154 100644--- a/src/crawlers/cheerio_crawler.js+++ b/src/crawlers/cheerio_crawler.js@@ -128,11 +128,12 @@ const DEFAULT_OPTIONS = {* autoscaledPool: AutoscaledPool* }* ```+ * Type of `body` depends on web page `Content-Type` header.+ * - String for `text/html`, `application/xhtml+xml`, `application/xml`+ * - Object for `application/json`+ * - Buffer for the others+ ** Cheerio is available only for HTML and XML content types.- * Type of `body` depends of content type of crawling URL.- * - String: In case of HTML of XML content types- * - Object: In case of JSON content type- * - Buffer: For the rest content types** With the {@link Request} object representing the URL to crawl.*@@ -367,7 +368,7 @@ class CheerioCrawler {},get body() {if (dom) {- return (xmlMode) ? $.xml() : $.html({ decodeEntities: false });+ return xmlMode ? $.xml() : $.html({ decodeEntities: false });}return body;},@@ -495,9 +496,8 @@ class CheerioCrawler {}/**- *+ * Reads whole stream and concat buffer from it* @param readStream- * @return {Promise<unknown>}* @ignore*/async _concatStreamToBuffer(readStream) {@@ -508,20 +508,19 @@ class CheerioCrawler {}/**- *+ * Checks and extends included content types to crawler* @param contentTypes- * @return {*}* @ignore*/_extendsDefaultContentTypes(contentTypes) {- const normalizedTypes = contentTypes.map((type) => {+ contentTypes.forEach((type) => {try {- return contentType.parse(type).type;+ const parsedType = contentType.parse(type);+ this.includedContentTypes.push(parsedType.type);} catch (err) {- throw new Error(`CheerioCrawler: Content type ${type} in not correct content type.`);+ throw new Error(`CheerioCrawler: Can not parse content type ${type} from "options.additionalContentTypes".`);}});- this.includedContentTypes.push(...normalizedTypes);}}diff --git a/test/crawlers/cheerio_crawler.js b/test/crawlers/cheerio_crawler.jsindex c3c695c72..b2ca4fa3a 100644--- a/test/crawlers/cheerio_crawler.js+++ b/test/crawlers/cheerio_crawler.js@@ -552,18 +552,15 @@ describe('CheerioCrawler', () => {it('when response is application/json', async () => {const jsonRequestParams = handlePageInvocationParams[0];expect(jsonRequestParams.body).to.be.an('object');- expect(jsonRequestParams.$).to.be.an('null');});it('when response is application/xml', async () => {const xmlRequestParams = handlePageInvocationParams[1];expect(xmlRequestParams.body).to.be.an('string');- expect(xmlRequestParams.html).to.be.an('null');expect(xmlRequestParams.$).to.be.an('function');});it('when response is image/png', async () => {const imageRequestParams = handlePageInvocationParams[2];expect(imageRequestParams.body).to.be.instanceof(Buffer);- expect(imageRequestParams.html).to.be.an('null');});});diff --git a/test/crawlers/data/apify.png b/test/crawlers/data/apify.pngnew file mode 100644index 000000000..62c247d2cBinary files /dev/null and b/test/crawlers/data/apify.png differ