Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Moved streams functions to shared, added option json and contentType to handlePageFunction, mine -> mine
drobnikjfb26d868220d
diff --git a/docs/api/CheerioCrawler.md b/docs/api/CheerioCrawler.mdindex 003fc766f..4ea4f7a27 100644--- a/docs/api/CheerioCrawler.md+++ b/docs/api/CheerioCrawler.md@@ -27,8 +27,8 @@ that a single URL is not crawled multiple times.The crawler finishes when there are no more [`Request`](request) objects to crawl.-By default, `CheerioCrawler` downloads HTML or XML using the <a href="https://www.npmjs.com/package/request" target="_blank">request</a> NPM package.-You can use the `requestOptions` parameter to pass additional options to `request`.+By default, `CheerioCrawler` downloads HTML using the <a href="https://www.npmjs.com/package/request" target="_blank">request</a> NPM package. You can+use the `requestOptions` parameter to pass additional options to `request`.New requests are only dispatched when there is enough free CPU and memory available, using the functionality provided by the[`AutoscaledPool`](autoscaledpool) class. All [`AutoscaledPool`](autoscaledpool) configuration options can be passed to the `autoscaledPoolOptions`@@ -47,7 +47,7 @@ await requestList.initialize();// Crawl the URLsconst crawler = new Apify.CheerioCrawler({requestList,- handlePageFunction: async ({ request, response, body, $ }) => {+ handlePageFunction: async ({ request, response, body, contentType, $ }) => {const data = [];// Do some data extraction from the page with Cheerio.@@ -100,18 +100,20 @@ await crawler.run();loaded and parsed by the crawler.</p><p> The function receives the following object as an argument:</p><pre><code>{- $: Cheerio, // the Cheerio object with parsed HTML or XML- body: String|Object|Buffer // the request body of the web page+ $: Cheerio, // the Cheerio object with parsed HTML+ body: String|Buffer // the request body of the web pagerequest: Request,+ contentType: Object, // Parsed content typeresponse: Object // An instance of Node's http.IncomingMessage object,autoscaledPool: AutoscaledPool}</code></pre><p> Type of <code>body</code> depends on web page <code>Content-Type</code> header.</p><ul>-<li><p>String for <code>text/html</code>, <code>application/xhtml+xml</code>, <code>application/xml</code> mine types</p>+<li><p>String for <code>text/html</code>, <code>application/xhtml+xml</code>, <code>application/xml</code> mime types</p></li>-<li><p>Object for <code>application/json</code> mine type</p>-</li>-<li><p>Buffer for others mine types</p>+<li><p>Buffer for others mime types</p>+<p>Parsed <code>Content-Type</code> header using+<a href="https://www.npmjs.com/package/content-type" target="_blank">content-type package</a>+is stored in <code>contentType</code>.</p><p>Cheerio is available only for HTML and XML content types.</p><p>With the <a href="request"><code>Request</code></a> object representing the URL to crawl.</p><p>If the function returns a promise, it is awaited by the crawler.</p>@@ -230,12 +232,12 @@ The exceptions are logged to the request using the<p> See <a href="https://github.com/apifytech/apify-js/blob/master/src/crawlers/cheerio_crawler.js#L13">source code</a>for the default implementation of this function.</p></td></tr><tr>-<td><code>[options.additionalMineTypes]</code></td><td><code>Array<String></code></td><td></td>+<td><code>[options.additionalMimeTypes]</code></td><td><code>Array<String></code></td><td></td></tr><tr><td colspan="3"><p>An array of <a href="https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Complete_list_of_MIME_types"- target="_blank">mine types</a> you want to process.- By default <code>text/html</code>, <code>application/xhtml+xml</code> mine types are supported.</p>+ target="_blank">mime types</a> you want to process.+ By default <code>text/html</code>, <code>application/xhtml+xml</code> mime types are supported.</p></td></tr><tr><td><code>[options.maxRequestRetries]</code></td><td><code>Number</code></td><td><code>3</code></td></tr>diff --git a/package.json b/package.jsonindex b2906c8cd..53cb43d88 100644--- a/package.json+++ b/package.json@@ -52,9 +52,8 @@"@apify/http-request": "^1.1.2","@apify/ps-tree": "^1.1.3","apify-client": "^0.5.25",- "apify-shared": "^0.1.56",+ "apify-shared": "^0.1.66","cheerio": "^1.0.0-rc.3",- "concat-stream": "^2.0.0","content-type": "^1.0.4","express": "^4.17.1","fs-extra": "^8.1.0",diff --git a/src/crawlers/cheerio_crawler.js b/src/crawlers/cheerio_crawler.jsindex caf5c3d20..4b8341a89 100644--- a/src/crawlers/cheerio_crawler.js+++ b/src/crawlers/cheerio_crawler.js@@ -5,9 +5,8 @@ import htmlparser from 'htmlparser2';import util from 'util';import log from 'apify-shared/log';import { checkParamOrThrow } from 'apify-client/build/utils';-import contentType from 'content-type';-import readStreamToString from '@apify/http-request/src/read_stream_to_string';-import concatStream from 'concat-stream';+import contentTypeParser from 'content-type';+import { readStreamToString, concatStreamToBuffer } from 'apify-shared/streams_utilities';import BasicCrawler from './basic_crawler';import { addTimeoutToPromise } from '../utils';import { getApifyProxyUrl } from '../actor';@@ -15,9 +14,9 @@ import { BASIC_CRAWLER_TIMEOUT_MULTIPLIER } from '../constants';import { requestAsBrowser } from '../utils_request';/**- * Default mine types, which CheerioScraper supports.+ * Default mime types, which CheerioScraper supports.*/-const DEFAULT_MINE_TYPES = ['text/html', 'application/xhtml+xml'];+const DEFAULT_MIME_TYPES = ['text/html', 'application/xhtml+xml'];const DEFAULT_OPTIONS = {requestTimeoutSecs: 30,@@ -38,7 +37,7 @@ const DEFAULT_OPTIONS = {maxEventLoopOverloadedRatio: 0.7,},},- additionalMineTypes: [],+ additionalMimeTypes: [],};/**@@ -67,7 +66,7 @@ const DEFAULT_OPTIONS = {** The crawler finishes when there are no more {@link Request} objects to crawl.*- * By default, `CheerioCrawler` downloads HTML or XML using the+ * By default, `CheerioCrawler` downloads HTML using the* <a href="https://www.npmjs.com/package/request" target="_blank">request</a> NPM package.* You can use the `requestOptions` parameter to pass additional options to `request`.*@@ -92,7 +91,7 @@ const DEFAULT_OPTIONS = {* // Crawl the URLs* const crawler = new Apify.CheerioCrawler({* requestList,- * handlePageFunction: async ({ request, response, body, $ }) => {+ * handlePageFunction: async ({ request, response, body, contentType, $ }) => {* const data = [];** // Do some data extraction from the page with Cheerio.@@ -121,17 +120,21 @@ const DEFAULT_OPTIONS = {* The function receives the following object as an argument:* ```* {- * $: Cheerio, // the Cheerio object with parsed HTML or XML- * body: String|Object|Buffer // the request body of the web page+ * $: Cheerio, // the Cheerio object with parsed HTML+ * body: String|Buffer // the request body of the web page* request: Request,+ * contentType: Object, // Parsed content type* response: Object // An instance of Node's http.IncomingMessage object,* autoscaledPool: AutoscaledPool* }* ```* Type of `body` depends on web page `Content-Type` header.- * - String for `text/html`, `application/xhtml+xml`, `application/xml` mine types- * - Object for `application/json` mine type- * - Buffer for others mine types+ * - String for `text/html`, `application/xhtml+xml`, `application/xml` mime types+ * - Buffer for others mime types+ *+ * Parsed `Content-Type` header using+ * <a href="https://www.npmjs.com/package/content-type" target="_blank">content-type package</a>+ * is stored in `contentType`.** Cheerio is available only for HTML and XML content types.*@@ -230,10 +233,10 @@ const DEFAULT_OPTIONS = {** See <a href="https://github.com/apifytech/apify-js/blob/master/src/crawlers/cheerio_crawler.js#L13">source code</a>* for the default implementation of this function.- * @param {String[]} [options.additionalMineTypes]+ * @param {String[]} [options.additionalMimeTypes]* An array of <a href="https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Complete_list_of_MIME_types"- * target="_blank">mine types</a> you want to process.- * By default `text/html`, `application/xhtml+xml` mine types are supported.+ * target="_blank">mime types</a> you want to process.+ * By default `text/html`, `application/xhtml+xml` mime types are supported.* @param {Number} [options.maxRequestRetries=3]* Indicates how many times the request is retried if either `requestFunction` or `handlePageFunction` fails.* @param {Number} [options.maxRequestsPerCrawl]@@ -266,7 +269,7 @@ class CheerioCrawler {apifyProxyGroups,apifyProxySession,proxyUrls,- additionalMineTypes,+ additionalMimeTypes,// Autoscaled pool shorthandsminConcurrency,@@ -292,13 +295,13 @@ class CheerioCrawler {checkParamOrThrow(apifyProxySession, 'options.apifyProxySession', 'Maybe String');checkParamOrThrow(proxyUrls, 'options.proxyUrls', 'Maybe [String]');checkParamOrThrow(prepareRequestFunction, 'options.prepareRequestFunction', 'Maybe Function');- checkParamOrThrow(additionalMineTypes, 'options.additionalMineTypes', '[String]');+ checkParamOrThrow(additionalMimeTypes, 'options.additionalMimeTypes', '[String]');// Enforce valid proxy configurationif (proxyUrls && !proxyUrls.length) throw new Error('Parameter "options.proxyUrls" of type Array must not be empty');if (useApifyProxy && proxyUrls) throw new Error('Cannot combine "options.useApifyProxy" with "options.proxyUrls"!');- this.supportedMineTypes = new Set(DEFAULT_MINE_TYPES);- if (additionalMineTypes.length) this._extendSupportedMineTypes(additionalMineTypes);+ this.supportedMimeTypes = new Set(DEFAULT_MIME_TYPES);+ if (additionalMimeTypes.length) this._extendSupportedMimeTypes(additionalMimeTypes);this.requestOptions = requestOptions;this.handlePageFunction = handlePageFunction;@@ -350,7 +353,7 @@ class CheerioCrawler {*/async _handleRequestFunction({ request, autoscaledPool }) {if (this.prepareRequestFunction) await this.prepareRequestFunction({ request });- const { dom, isXmlOrHtml, body, responseStream: response } = await addTimeoutToPromise(+ const { dom, isXmlOrHtml, body, contentType, responseStream: response } = await addTimeoutToPromise(this._requestFunction({ request }),this.requestTimeoutMillis,'CheerioCrawler: requestFunction timed out.',@@ -364,15 +367,21 @@ class CheerioCrawler {// Using a getter here not to break the original API// and lazy load the HTML only when needed.get html() {- log.deprecated('CheerioCrawler: Parameter html for handlePageFunction is deprecated use body instead.');+ log.deprecated('CheerioCrawler: The "html" parameter of handlePageFunction is deprecated, use "body" instead.');return dom && !isXmlOrHtml ? $.html({ decodeEntities: false }) : undefined;},+ get json() {+ if (contentType.type !== 'application/json') return null;+ const jsonString = body.toString(contentType.encoding);+ return JSON.parse(jsonString);+ },get body() {if (dom) {return isXmlOrHtml ? $.xml() : $.html({ decodeEntities: false });}return body;},+ contentType,request,response,autoscaledPool,@@ -397,7 +406,8 @@ class CheerioCrawler {const responseStream = await requestAsBrowser(opts);const { statusCode, headers } = responseStream;- const { type, encoding } = contentType.parse(headers['content-type']);+ const contentType = contentTypeParser.parse(headers['content-type']);+ const { type, encoding } = contentType;if (statusCode >= 500) {const body = await readStreamToString(responseStream, encoding);@@ -414,14 +424,10 @@ class CheerioCrawler {throw new Error(`CheerioCrawler: ${statusCode} - Internal Server Error: ${body.substr(0, 100)}`);} else if (type === 'text/html' || type === 'application/xhtml+xml' || type === 'application/xml') {const dom = await this._parseHtmlToDom(responseStream);- return ({ dom, isXmlOrHtml: type.includes('xml'), responseStream });- } else if (type === 'application/json') {- const jsonString = await readStreamToString(responseStream, encoding);- const body = JSON.parse(jsonString);- return { body, responseStream };+ return ({ dom, isXmlOrHtml: type.includes('xml'), responseStream, contentType });} else {- const body = await this._concatStreamToBuffer(responseStream);- return { body, responseStream };+ const body = await concatStreamToBuffer(responseStream);+ return { body, responseStream, contentType };}}@@ -442,17 +448,17 @@ class CheerioCrawler {abortFunction: (res) => {const { statusCode, headers } = res;- const { type } = contentType.parse(headers['content-type']);+ const { type } = contentTypeParser.parse(headers['content-type']);if (statusCode === 406) {request.noRetry = true;throw new Error(`CheerioCrawler: Resource ${request.url} is not available in HTML format. Skipping resource.`);}- if (!this.supportedMineTypes.has(type) && statusCode < 500) {+ if (!this.supportedMimeTypes.has(type) && statusCode < 500) {request.noRetry = true;throw new Error(`CheerioCrawler: Resource ${request.url} served Content-Type ${type}, `- + `but only ${Array.from(this.supportedMineTypes).join(', ')} are allowed. Skipping resource.`);+ + `but only ${Array.from(this.supportedMimeTypes).join(', ')} are allowed. Skipping resource.`);}return false;@@ -497,29 +503,17 @@ class CheerioCrawler {}/**- * Reads whole stream and concat buffer from it- * @param readStream- * @ignore- */- async _concatStreamToBuffer(readStream) {- return new Promise((resolve, reject) => {- const resultStream = concatStream(resolve);- readStream.on('error', reject).pipe(resultStream);- });- }-- /**- * Checks and extends supported mine types- * @param additionalMineTypes+ * Checks and extends supported mime types+ * @param additionalMimeTypes* @ignore*/- _extendSupportedMineTypes(additionalMineTypes) {- additionalMineTypes.forEach((mineType) => {+ _extendSupportedMimeTypes(additionalMimeTypes) {+ additionalMimeTypes.forEach((mimeType) => {try {- const parsedType = contentType.parse(mineType);- this.supportedMineTypes.add(parsedType.type);+ const parsedType = contentTypeParser.parse(mimeType);+ this.supportedMimeTypes.add(parsedType.type);} catch (err) {- throw new Error(`CheerioCrawler: Can not parse mine type ${mineType} from "options.additionalMineTypes".`);+ throw new Error(`CheerioCrawler: Can not parse mime type ${mimeType} from "options.additionalMimeTypes".`);}});}diff --git a/test/crawlers/cheerio_crawler.js b/test/crawlers/cheerio_crawler.jsindex 6864fd3c5..1c59ec0e2 100644--- a/test/crawlers/cheerio_crawler.js+++ b/test/crawlers/cheerio_crawler.js@@ -553,7 +553,7 @@ describe('CheerioCrawler', () => {expect(handledRequests).to.be.eql(contentTypes.length);});- describe('should works with all content types from options.additionalMineTypes', () => {+ describe('should works with all content types from options.additionalMimeTypes', () => {const handlePageInvocationParams = [];let handleFailedInvocationCount = 0;before(async () => {@@ -566,7 +566,7 @@ describe('CheerioCrawler', () => {await requestList.initialize();const crawler = new Apify.CheerioCrawler({requestList,- additionalMineTypes: ['application/json', 'image/png', 'application/xml'],+ additionalMimeTypes: ['application/json', 'image/png', 'application/xml'],maxRequestRetries: 1,handlePageFunction: async (params) => {handlePageInvocationParams.push(params);@@ -582,19 +582,22 @@ describe('CheerioCrawler', () => {});it('when response is application/json', async () => {const jsonRequestParams = handlePageInvocationParams[0];- expect(jsonRequestParams.body).to.be.an('object');- expect(jsonRequestParams.body).to.be.eql(responseSamples.json);+ expect(jsonRequestParams.json).to.be.an('object');+ expect(jsonRequestParams.body).to.be.eql(Buffer.from(JSON.stringify(responseSamples.json)));+ expect(jsonRequestParams.contentType.type).to.be.eql('application/json');});it('when response is application/xml', async () => {const xmlRequestParams = handlePageInvocationParams[1];expect(xmlRequestParams.body).to.be.an('string');expect(xmlRequestParams.body).to.be.eql(responseSamples.xml);expect(xmlRequestParams.$).to.be.an('function');+ expect(xmlRequestParams.contentType.type).to.be.eql('application/xml');});it('when response is image/png', async () => {const imageRequestParams = handlePageInvocationParams[2];expect(imageRequestParams.body).to.be.instanceof(Buffer);expect(imageRequestParams.body).to.be.eql(responseSamples.image);+ expect(imageRequestParams.contentType.type).to.be.eql('image/png');});});