Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
save
Petr Pateka17d68795e2a
diff --git a/package.json b/package.jsonindex 12481ec47..f7aefc2f4 100644--- a/package.json+++ b/package.json@@ -81,6 +81,7 @@"@types/puppeteer": "^1.12.4","@types/socket.io": "^2.1.2","basic-auth-parser": "^0.0.2",+ "body-parser": "^1.19.0","chai": "^4.2.0","chai-as-promised": "^7.1.1","chai-subset": "^1.6.0",diff --git a/src/crawlers/cheerio_crawler.js b/src/crawlers/cheerio_crawler.jsindex 4aafe3609..441ca1fbe 100644--- a/src/crawlers/cheerio_crawler.js+++ b/src/crawlers/cheerio_crawler.js@@ -1,11 +1,6 @@/* eslint-disable class-methods-use-this */--import util from 'util';-import zlib from 'zlib';-import rqst from 'request';import _ from 'underscore';import cheerio from 'cheerio';-import contentType from 'content-type';import htmlparser from 'htmlparser2';import log from 'apify-shared/log';import { checkParamOrThrow } from 'apify-client/build/utils';@@ -13,6 +8,7 @@ import BasicCrawler from './basic_crawler';import { addTimeoutToPromise } from '../utils';import { getApifyProxyUrl } from '../actor';import { BASIC_CRAWLER_TIMEOUT_MULTIPLIER } from '../constants';+import { requestAsBrowser } from '../utils_request';const DEFAULT_OPTIONS = {requestTimeoutSecs: 30,@@ -306,8 +302,6 @@ class CheerioCrawler {autoscaledPoolOptions,});- // See the _suppressTunnelAgentAssertError function.- this.tunnelAgentExceptionListener = null;this.isRunningPromise = null;}@@ -319,11 +313,8 @@ class CheerioCrawler {async run() {if (this.isRunningPromise) return this.isRunningPromise;- this._suppressTunnelAgentAssertError();this.isRunningPromise = this.basicCrawler.run();await this.isRunningPromise;- process.removeListener('uncaughtException', this.tunnelAgentExceptionListener);- this.tunnelAgentExceptionListener = null;}/**@@ -333,13 +324,13 @@ class CheerioCrawler {*/async _handleRequestFunction({ request, autoscaledPool }) {if (this.prepareRequestFunction) await this.prepareRequestFunction({ request });- const { dom, response } = await addTimeoutToPromise(+ const { dom, responseStream: response } = await addTimeoutToPromise(this._requestFunction({ request }),this.requestTimeoutMillis,'CheerioCrawler: requestFunction timed out.',);- request.loadedUrl = response.request.uri.href;+ request.loadedUrl = response.url;const $ = cheerio.load(dom);const context = {@@ -367,75 +358,13 @@ class CheerioCrawler {* @ignore*/async _requestFunction({ request }) {- return new Promise((resolve, reject) => {- // Using the streaming API of Request to be able to- // handle the response based on headers receieved.- const opts = this._getRequestOptions(request);- const method = opts.method.toLowerCase();- rqst[method](opts)- .on('error', err => reject(err))- .on('response', async (response) => {- // First check what kind of response we received.- let cType;- try {- cType = contentType.parse(response);- } catch (err) {- response.destroy();- // No reason to parse the body if the Content-Type header is invalid.- return reject(new Error(`CheerioCrawler: Invalid Content-Type header for URL: ${request.url}`));- }-- const { type, encoding } = cType;-- // 500 codes are handled as errors, requests will be retried.- const status = response.statusCode;- if (status >= 500) {- let body;- try {- body = await this._stringifyResponseBody(this._decompressResponse(response), encoding);- } catch (err) {- // Error in reading the body.- return reject(err);- }- // Errors are often sent as JSON, so attempt to parse them,- // despite Accept header being set to text/html.- if (type === 'application/json') {- const errorResponse = JSON.parse(body);- let { message } = errorResponse;- if (!message) message = util.inspect(errorResponse, { depth: 1, maxArrayLength: 10 });- return reject(new Error(`${status} - ${message}`));- }- // It's not a JSON so it's probably some text. Get the first 100 chars of it.- return reject(new Error(`CheerioCrawler: ${status} - Internal Server Error: ${body.substr(0, 100)}`));- }+ // Using the streaming API of Request to be able to+ // handle the response based on headers receieved.+ const opts = this._getRequestOptions(request);+ const responseStream = await requestAsBrowser(opts);- // Handle situations where the server explicitly states that- // it will not serve the resource as text/html by skipping.- if (status === 406) {- request.noRetry = true;- response.destroy();- return reject(new Error(`CheerioCrawler: Resource ${request.url} is not available in HTML format. Skipping resource.`));- }-- // Other 200-499 responses are considered OK, but first check the content type.- if (type !== 'text/html') {- request.noRetry = true;- response.destroy();- return reject(new Error(- `CheerioCrawler: Resource ${request.url} served Content-Type ${type} instead of text/html. Skipping resource.`,- ));- }-- // Content-Type is fine. Read the body and respond.- try {- const dom = await this._parseHtmlToDom(this._decompressResponse(response));- resolve({ dom, response });- } catch (err) {- // Error in reading the body.- reject(err);- }- });- });+ const dom = await this._parseHtmlToDom(responseStream);+ return ({ dom, responseStream });}/**@@ -447,15 +376,13 @@ class CheerioCrawler {const mandatoryRequestOptions = {url: request.url,method: request.method,- headers: Object.assign({}, request.headers, {- Accept: 'text/html',- 'Accept-Encoding': 'gzip, deflate',- }),- strictSSL: !this.ignoreSslErrors,- proxy: this._getProxyUrl(),+ headers: Object.assign({}, request.headers),+ ignoreSslErrors: this.ignoreSslErrors,+ proxyUrl: this._getProxyUrl(),+ stream: true,};- if (/PATCH|POST|PUT/.test(request.method)) mandatoryRequestOptions.body = request.payload;+ if (/PATCH|POST|PUT/.test(request.method)) mandatoryRequestOptions.payload = request.payload;return Object.assign({}, this.requestOptions, mandatoryRequestOptions);}@@ -480,27 +407,6 @@ class CheerioCrawler {return null;}- /**- * Flushes the provided stream into a Buffer and transforms- * it to a String using the provided encoding or utf-8 as default.- *- * @param {http.IncomingMessage} response- * @param {String} [encoding]- * @private- */- async _stringifyResponseBody(response, encoding) {- return new Promise((resolve, reject) => {- const chunks = [];- response- .on('data', chunk => chunks.push(chunk))- .on('error', reject)- .on('end', () => {- const buffer = Buffer.concat(chunks);- resolve(buffer.toString(encoding));- });- });- }-async _parseHtmlToDom(response) {return new Promise((resolve, reject) => {@@ -512,58 +418,7 @@ class CheerioCrawler {response.on('error', reject).pipe(parser);});}-- /**- * If the stream data is compressed, decompresses it using- * the Content-Encoding header.- *- * @param {http.IncomingMessage} response- * @return {http.IncomingMessage}- * @private- */- _decompressResponse(response) {- const compression = response.headers['content-encoding'];- if (!compression) return response;- let decompressor;- if (compression === 'gzip') decompressor = zlib.createGunzip();- else if (compression === 'deflate') decompressor = zlib.createInflate();- else throw new Error(`CheerioCrawler: Invalid Content-Encoding header. Expected gzip or deflate, but received: ${compression}`);- return response.pipe(decompressor);- }-- /**- * The handler this function attaches overcomes a long standing bug in- * the tunnel-agent NPM package that is used by the Request package internally.- * The package throws an assertion error in a callback scope that cannot be- * caught by conventional means and shuts down the running process.- * @ignore- */- _suppressTunnelAgentAssertError() {- // Only set the handler if it's not already set.- if (this.tunnelAgentExceptionListener) return;- this.tunnelAgentExceptionListener = (err) => {- try {- const code = err.code === 'ERR_ASSERTION';- const name = err.name === 'AssertionError [ERR_ASSERTION]';- const operator = err.operator === '==';- const value = err.expected === 0;- const stack = err.stack.includes('/tunnel-agent/index.js');- // If this passes, we can be reasonably sure that it's- // the right error from tunnel-agent.- if (code && name && operator && value && stack) {- log.error('CheerioCrawler: Tunnel-Agent assertion error intercepted. The affected request will timeout.');- return;- }- } catch (caughtError) {- // Catch any exception resulting from the duck-typing- // check. It only means that the error is not the one- // we're looking for.- }- // Rethrow the original error if it's not a match.- throw err;- };- process.on('uncaughtException', this.tunnelAgentExceptionListener);- }}+export default CheerioCrawler;diff --git a/src/utils_request.js b/src/utils_request.jsindex 232be7b59..684bfc3d9 100644--- a/src/utils_request.js+++ b/src/utils_request.js@@ -19,6 +19,7 @@ export const REQUEST_AS_BROWSER_DEFAULT_OPTIONS = {return res.statusCode === 406 || type.toLowerCase() !== 'text/html';},useCaseSensitiveHeaders: true,+ useStream: false,};/*** Sends a HTTP request that looks like a request sent by a web browser,@@ -58,7 +59,7 @@ export const REQUEST_AS_BROWSER_DEFAULT_OPTIONS = {* If function returns true request gets aborted. This function is passed to the* (@apify/http-request)[https://www.npmjs.com/package/@apify/http-request] NPM package.*- * @return {http.IncomingMessage}+ * @return {Promise<http.IncomingMessage>|Promise<Stream>}* @memberOf utils* @name requestAsBrowser*/diff --git a/test/crawlers/cheerio_crawler.js b/test/crawlers/cheerio_crawler.jsindex 1610f38c7..0e4350f38 100644--- a/test/crawlers/cheerio_crawler.js+++ b/test/crawlers/cheerio_crawler.js@@ -8,17 +8,24 @@ import { expect } from 'chai';import log from 'apify-shared/log';import { delayPromise } from 'apify-shared/utilities';import { ENV_VARS } from 'apify-shared/consts';+import express from 'express'+import bodyParser from 'body-parser';import Apify from '../../build';// Add common props to mocked request responses.const responseMock = {request: {- uri: {- href: 'loadedUrl',- },+ url: 'loadedUrl',},};+const startExpressAppPromise = (app, port) => {+ return new Promise((resolve) => {+ const server = app.listen(port, () => resolve(server));+ });+};++/* eslint-disable no-underscore-dangle */describe('CheerioCrawler', () => {const comparator = (a, b) => {@@ -32,6 +39,14 @@ describe('CheerioCrawler', () => {before(() => {logLevel = log.getLevel();log.setLevel(log.LEVELS.ERROR);+ const app = express();+ app.use(bodyParser.urlencoded({+ extended: true,+ }));+ app.use(bodyParser.json());+ app.get('/mock', (req, res) => {++ });});after(() => {@@ -796,71 +811,4 @@ describe('CheerioCrawler', () => {});});});-- // It would make sense to also test that the listener does not prevent other- // uncaught exceptions from exiting the process, but since the exceptions- // thrown from within the listener cannot be caught anymore, there's no way- // to do this, because the re-thrown error will always crash the test process.- describe('tunnel-agent error handler', () => {- const throwNextTick = (err) => {- process.nextTick(() => {- throw err;- });- };-- let mochaListener;-- before(() => {- log.setLevel(log.LEVELS.OFF);- mochaListener = process.listeners('uncaughtException').shift();- process.removeListener('uncaughtException', mochaListener);- });- after(() => {- log.setLevel(log.LEVELS.ERROR);- process.on('uncaughtException', mochaListener);- });-- it('should suppress tunnel-agent errors', async () => {- let handlePageCalled = false;- let handleFailedRequestCallCount = 0;-- const requestList = new Apify.RequestList({- sources: [- { url: 'http://example.com/?q=0' },- { url: 'http://example.com/?q=1' },- { url: 'http://example.com/?q=2' },- { url: 'http://example.com/?q=3' },- ],- });-- const crawler = new Apify.CheerioCrawler({- requestList,- requestTimeoutSecs: 1 / 1000,- handlePageFunction: () => {- handlePageCalled = true;- },- handleFailedRequestFunction: () => {- handleFailedRequestCallCount++;- },- });-- crawler._requestFunction = async () => {- const err = new Error();- err.code = 'ERR_ASSERTION';- err.name = 'AssertionError [ERR_ASSERTION]';- err.operator = '==';- err.expected = 0;- err.stack = ('xxx/tunnel-agent/index.js/yyyy');- throwNextTick(err);- // will never resolve- await new Promise(() => {});- };-- await requestList.initialize();- await crawler.run();-- expect(handlePageCalled).to.be.eql(false);- expect(handleFailedRequestCallCount).to.be.eql(4);- });- });});