Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Improved test and streaming logic...

Petr Patek21fa4756ae6a
diff --git a/package.json b/package.json
index 4119978e8..0d35b4dba 100644
--- a/package.json
+++ b/package.json
@@ -50,10 +50,10 @@
"lint:fix": "./node_modules/.bin/eslint ./src ./test --ext .js,.jsx --fix"
},
"dependencies": {
- "@apify/http-request": "^1.0.6",
+ "@apify/http-request": "^1.1.0",
"@apify/ps-tree": "^1.1.3",
"apify-client": "^0.5.22",
- "apify-shared": "^0.1.55",
+ "apify-shared": "^0.1.56",
"cheerio": "^1.0.0-rc.2",
"content-type": "^1.0.4",
"express": "^4.17.1",
@@ -64,7 +64,7 @@
"proxy-chain": "^0.2.7",
"request": "^2.88.0",
"request-promise-native": "^1.0.5",
- "rimraf": "^2.6.2",
+ "rimraf": "^2.7.1",
"semver": "^6.3.0",
"socket.io": "^2.2.0",
"underscore": "^1.9.1",
@@ -78,7 +78,7 @@
"@babel/preset-env": "^7.0.0",
"@babel/register": "^7.0.0",
"@types/fs-extra": "^7.0.0",
- "@types/puppeteer": "^1.12.4",
+ "@types/puppeteer": "^1.19.1",
"@types/socket.io": "^2.1.2",
"basic-auth-parser": "^0.0.2",
"body-parser": "^1.19.0",
@@ -94,7 +94,7 @@
"nyc": "^14.0.0",
"portastic": "^1.0.1",
"proxy": "^0.2.4",
- "sinon": "^7.1.1",
+ "sinon": "^7.4.1",
"sinon-stub-promise": "^4.0.0",
"socket.io-client": "^2.2.0",
"why-is-node-running": "^2.0.3"
diff --git a/src/crawlers/cheerio_crawler.js b/src/crawlers/cheerio_crawler.js
index 846e93673..ff413962c 100644
--- a/src/crawlers/cheerio_crawler.js
+++ b/src/crawlers/cheerio_crawler.js
@@ -2,14 +2,18 @@
import _ from 'underscore';
import cheerio from 'cheerio';
import htmlparser from 'htmlparser2';
+import util from 'util';
import log from 'apify-shared/log';
import { checkParamOrThrow } from 'apify-client/build/utils';
+import contentType from 'content-type';
+import readStreamToString from '@apify/http-request/src/read_stream_to_string';
import BasicCrawler from './basic_crawler';
import { addTimeoutToPromise } from '../utils';
import { getApifyProxyUrl } from '../actor';
import { BASIC_CRAWLER_TIMEOUT_MULTIPLIER } from '../constants';
import { requestAsBrowser } from '../utils_request';
+
const DEFAULT_OPTIONS = {
requestTimeoutSecs: 30,
handlePageTimeoutSecs: 60,
@@ -330,7 +334,7 @@ class CheerioCrawler {
'CheerioCrawler: requestFunction timed out.',
);
- request.loadedUrl = response.url;
+ request.loadedUrl = response.url; // @TODO: add test for redirects
const $ = cheerio.load(dom);
const context = {
@@ -361,18 +365,28 @@ class CheerioCrawler {
// Using the streaming API of Request to be able to
// handle the response based on headers receieved.
const opts = this._getRequestOptions(request);
- let responseStream;
- try {
- responseStream = await requestAsBrowser(opts);
- } catch (e) {
- if (e.message === `Request for ${request.url} aborted due to abortFunction`) {
- request.noRetry = true;
+
+ const responseStream = await requestAsBrowser(opts);
+ const { statusCode, headers } = responseStream;
+ if (statusCode >= 500) {
+ const { type, encoding } = contentType.parse(headers['content-type']);
+ const body = await readStreamToString(responseStream, encoding);
+
+ // Errors are often sent as JSON, so attempt to parse them,
+ // despite Accept header being set to text/html.
+ if (type === 'application/json') {
+ const errorResponse = JSON.parse(body);
+ let { message } = errorResponse;
+ if (!message) message = util.inspect(errorResponse, { depth: 1, maxArrayLength: 10 });
+ throw new Error(`${statusCode} - ${message}`);
}
- throw new Error(e.message);
- }
- const dom = await this._parseHtmlToDom(responseStream);
- return ({ dom, responseStream });
+ // It's not a JSON so it's probably some text. Get the first 100 chars of it.
+ throw new Error(`CheerioCrawler: ${statusCode} - Internal Server Error: ${body.substr(0, 100)}`);
+ } else {
+ const dom = await this._parseHtmlToDom(responseStream);
+ return ({ dom, responseStream });
+ }
}
/**
@@ -388,6 +402,24 @@ class CheerioCrawler {
ignoreSslErrors: this.ignoreSslErrors,
proxyUrl: this._getProxyUrl(),
stream: true,
+ useCaseSensitiveHeaders: true,
+ abortFunction: (res) => {
+ const { statusCode, headers } = res;
+
+ const { type } = contentType.parse(headers['content-type']);
+
+ if (statusCode === 406) {
+ request.noRetry = true;
+ throw new Error(`CheerioCrawler: Resource ${request.url} is not available in HTML format. Skipping resource.`);
+ }
+
+ if (type.toLowerCase() !== 'text/html' && statusCode < 500) {
+ request.noRetry = true;
+ throw new Error(`CheerioCrawler: Resource ${request.url} served Content-Type ${type} instead of text/html. Skipping resource.`);
+ }
+
+ return false;
+ },
};
if (/PATCH|POST|PUT/.test(request.method)) mandatoryRequestOptions.payload = request.payload;
diff --git a/src/utils_request.js b/src/utils_request.js
index 3cb25549f..712910bea 100644
--- a/src/utils_request.js
+++ b/src/utils_request.js
@@ -16,11 +16,11 @@ export const REQUEST_AS_BROWSER_DEFAULT_OPTIONS = {
json: false,
abortFunction: (res) => {
const { type } = contentType.parse(res.headers['content-type']);
- return res.statusCode === 406 || type.toLowerCase() !== 'text/html' || res.statusCode >= 500;
+ return res.statusCode === 406 || type.toLowerCase() !== 'text/html';
},
useCaseSensitiveHeaders: true,
useStream: false,
- payload: '',
+ proxyUrl: null,
};
/**
* Sends a HTTP request that looks like a request sent by a web browser,
@@ -60,7 +60,7 @@ export const REQUEST_AS_BROWSER_DEFAULT_OPTIONS = {
* If function returns true request gets aborted. This function is passed to the
* (@apify/http-request)[https://www.npmjs.com/package/@apify/http-request] NPM package.
*
- * @return {Promise<http.IncomingMessage>|Promise<Stream>}
+ * @return {Promise<http.IncomingMesage|stream.Readable>}
* @memberOf utils
* @name requestAsBrowser
*/
diff --git a/test/crawlers/cheerio_crawler.js b/test/crawlers/cheerio_crawler.js
index 67ceabaf7..375ff766b 100644
--- a/test/crawlers/cheerio_crawler.js
+++ b/test/crawlers/cheerio_crawler.js
@@ -7,8 +7,12 @@ import { delayPromise } from 'apify-shared/utilities';
import { ENV_VARS } from 'apify-shared/consts';
import express from 'express';
import bodyParser from 'body-parser';
+import sinon from 'sinon';
+import * as utilsRequest from '../../build/utils_request';
import Apify from '../../build';
+const Stream = require('stream');
+
// Add common props to mocked request responses.
const responseMock = {
url: 'loadedUrl',
@@ -35,28 +39,28 @@ describe('CheerioCrawler', () => {
let server;
let port;
- async function getRequestListForMock(mockData) {
+ async function getRequestListForMock(mockData, path = 'mock') {
const sources = [
{
- url: `http://${HOST}:${port}/mock?a=1`,
+ url: `http://${HOST}:${port}/${path}?a=1`,
payload: JSON.stringify(mockData),
method: 'POST',
- headers: { 'Content-Type': 'application/json' },
+ headers: { 'Content-Type': 'application/json', Accept: 'application/json' },
}, {
- url: `http://${HOST}:${port}/mock?a=2`,
+ url: `http://${HOST}:${port}/${path}?a=2`,
payload: JSON.stringify(mockData),
method: 'POST',
- headers: { 'Content-Type': 'application/json' },
+ headers: { 'Content-Type': 'application/json', Accept: 'application/json' },
}, {
- url: `http://${HOST}:${port}/mock?a=3`,
+ url: `http://${HOST}:${port}/${path}?a=3`,
payload: JSON.stringify(mockData),
method: 'POST',
- headers: { 'Content-Type': 'application/json' },
+ headers: { 'Content-Type': 'application/json', Accept: 'application/json' },
}, {
- url: `http://${HOST}:${port}/mock?a=4`,
+ url: `http://${HOST}:${port}/${path}?a=4`,
payload: JSON.stringify(mockData),
method: 'POST',
- headers: { 'Content-Type': 'application/json' },
+ headers: { 'Content-Type': 'application/json', Accept: 'application/json' },
},
];
const requestList = new Apify.RequestList({ sources });
@@ -84,16 +88,14 @@ describe('CheerioCrawler', () => {
}));
app.use(bodyParser.json());
app.post('/mock', (req, res) => {
- const { headers, body, statusCode, error = false } = req.body;
+ const { headers, statusCode, error = false, body } = req.body;
+
if (error) {
throw new Error(error);
}
+
Object.entries(headers).forEach(([key, value]) => res.setHeader(key, value));
- if (headers.contentType) {
- res.set({
- 'Content-Type': headers.contentType,
- });
- }
+
res.status(statusCode).send(body);
});
@@ -101,6 +103,12 @@ describe('CheerioCrawler', () => {
res.send({ some: 'json' });
});
+ app.get('/jsonError', (req, res) => {
+ res
+ .status(500)
+ .json({ message: 'json' });
+ });
+
app.get('/mirror', (req, res) => {
res.send('DATA');
@@ -338,7 +346,7 @@ describe('CheerioCrawler', () => {
expect(handlePageInvocationCount).to.be.eql(0);
expect(errorMessages).to.have.lengthOf(4);
- errorMessages.forEach(msg => expect(msg).to.include('aborted due to abortFunction'));
+ errorMessages.forEach(msg => expect(msg).to.include('served Content-Type application/json instead of text/html. Skipping resource.'));
expect(chunkReadCount).to.be.eql(0);
});
@@ -400,10 +408,11 @@ describe('CheerioCrawler', () => {
// sometimes if you get blocked you can get 500+ with some html inside
crawler = new Apify.CheerioCrawler({
requestList: await getRequestListForMock({
- statusCode: 500,
+ statusCode: 508,
headers: {
'content-type': 'text/html',
},
+ body: 'DATABASE ERRROR',
}),
maxRequestRetries: 1,
handlePageFunction: async () => {
@@ -416,18 +425,31 @@ describe('CheerioCrawler', () => {
await crawler.run();
expect(handlePageInvocationCount).to.be.eql(0);
- expect(errorMessages).to.have.lengthOf(4);
- errorMessages.forEach(msg => expect(msg).to.include('aborted due to abortFunction'));
+ expect(errorMessages).to.have.lengthOf(8);
+ errorMessages.forEach(msg => expect(msg).to.include('Internal Server Error'));
});
it('when statusCode >= 500 and application/json is received', async () => {
+ const save = sinon.stub(utilsRequest, 'requestAsBrowser');
+ const errorMessage = 'TEST_ERROR';
+ save.callsFake(() => new Promise((resolve) => {
+ const readableStream = new Stream.Readable();
+ readableStream._read = function () {
+ this.push(Buffer.from(JSON.stringify({ message: errorMessage })));
+ this.push(null);
+ };
+
+ const finalStream = new Stream.PassThrough();
+ finalStream.statusCode = 500;
+ finalStream.headers = {
+ 'content-type': 'application/json',
+ };
+ resolve(readableStream.pipe(finalStream));
+ }));
+
+
crawler = new Apify.CheerioCrawler({
- requestList: await getRequestListForMock({
- statusCode: 500,
- headers: {
- 'content-type': 'application/json',
- },
- }),
+ requestList: await getRequestListForMock({}, 'jsonError'),
maxRequestRetries: 1,
handlePageFunction: async () => {
handlePageInvocationCount++;
@@ -437,10 +459,11 @@ describe('CheerioCrawler', () => {
},
});
await crawler.run();
+ save.restore();
expect(handlePageInvocationCount).to.be.eql(0);
- expect(errorMessages).to.have.lengthOf(4);
- errorMessages.forEach(msg => expect(msg).to.include('aborted due to abortFunction'));
+ expect(errorMessages).to.have.lengthOf(8);
+ errorMessages.forEach(msg => expect(msg).to.include(errorMessage));
});
it('when 406 is received', async () => {
@@ -464,7 +487,7 @@ describe('CheerioCrawler', () => {
expect(handlePageInvocationCount).to.be.eql(0);
expect(errorMessages).to.have.lengthOf(4);
- errorMessages.forEach(msg => expect(msg).to.include('aborted due to abortFunction'));
+ errorMessages.forEach(msg => expect(msg).to.include('is not available in HTML format. Skipping resource.'));
});
xit('when status is ok, but a wrong content type is received', async () => {
@@ -506,7 +529,7 @@ describe('CheerioCrawler', () => {
beforeEach(async () => {
requestList = new Apify.RequestList({
sources: [
- { url: 'http://example.com/?q=0' },
+ { url: 'https://unicorn.com/cz/career' },
{ url: 'http://example.com/?q=1' },
{ url: 'http://example.com/?q=2' },
{ url: 'http://example.com/?q=3' },