Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Added param throwOnHttpError. Fixed error handling to do not throw on 400+ error codes by default. Added better contentType check.

Petr Patekb0467300eb4c
diff --git a/src/utils_request.js b/src/utils_request.js
index 5a5ef27f2..788177f68 100644
--- a/src/utils_request.js
+++ b/src/utils_request.js
@@ -77,6 +77,9 @@ const FIREFOX_DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14
* [`http.IncomingMessage`](https://nodejs.org/api/http.html#http_class_http_incomingmessage)
* class and it should return `true` if request should be aborted, or `false` otherwise.
*
+ * @param [options.throwOnHttpError=false]
+ * If set to true function throws and error on 4XX and 5XX response codes.
+ *
* @return {{ response, body }}
* Returns an object with two properties: `response` is the instance of
* Node's [`http.IncomingMessage`](https://nodejs.org/api/http.html#http_class_http_incomingmessage) class,
@@ -94,6 +97,7 @@ export const requestBetter = async (options) => {
removeRefererHeader: false,
headers: {},
method: 'GET',
+ throwOnHttpError: false,
};
const opts = Object.assign(defaultOptions, options);
@@ -101,12 +105,12 @@ export const requestBetter = async (options) => {
rqst[method](opts)
.on('error', err => reject(err))
.on('response', async (res) => {
- // no need to catch invalid content header - it is already caught by request
+ // No need to catch invalid content header - it is already caught by request
const { type, encoding } = contentType.parse(res);
- // 500 codes are handled as errors, requests will be retried.
+ // 5XX codes are handled as errors, requests will be retried.
const status = res.statusCode;
- if (status >= 500) {
+ if (status >= 400 && opts.throwOnHttpError) {
let body;
try {
body = await readStreamIntoString(res, encoding);
@@ -244,7 +248,7 @@ export const requestLikeBrowser = async (options) => {
opts.headers = Object.assign(browserHeaders, opts.headers);
const response = await requestBetter(opts);
- const receivedContentType = response.headers['content-type'];
+ const { type } = contentType.parse(response.headers['content-type']);
// Handle situations where the server explicitly states that
// it will not serve the resource as text/html by skipping.
@@ -252,8 +256,8 @@ export const requestLikeBrowser = async (options) => {
throw new Error(`requestLikeBrowser: Resource ${options.url} is not available in HTML format. Skipping resource.`);
}
- if (receivedContentType.toLowerCase() !== 'text/html; charset=utf-8') {
- throw new Error(`Received unexpected Content-Type: ${receivedContentType}`);
+ if (type.toLowerCase() !== 'text/html') {
+ throw new Error(`Received unexpected Content-Type: ${type}`);
}
const { body } = response;
diff --git a/test/utils_request.js b/test/utils_request.js
index 3541f18a3..c79fd572b 100644
--- a/test/utils_request.js
+++ b/test/utils_request.js
@@ -145,10 +145,25 @@ describe('Apify.utils_request', () => {
expect(response.body).to.eql(CONTENT);
});
- it('it throws error for 500+', async () => {
+ it('it does not throw error for 400+ error codes when throwOnHttpError is false', async () => {
const options = {
url: `http://${HOST}:${port}/500`,
+ };
+ let error;
+ try {
+ await requestBetter(options);
+ } catch (e) {
+ error = e;
+ }
+ expect(error).to.be.undefined; // eslint-disable-line
+ });
+
+ it('it does not throw error for 400+ error codes when throwOnHttpError is true', async () => {
+ const options = {
+ url: `http://${HOST}:${port}/500`,
+ throwOnHttpError: true,
+
};
let error;
try {
@@ -160,9 +175,10 @@ describe('Apify.utils_request', () => {
expect(error.message.includes(ERROR_BODY)).to.be.eql(true);
});
- it('it throws error when the body cannot be parsed and the code is 500', async () => {
+ it('it throws error when the body cannot be parsed and the code is 500 when throwOnHttpError is true', async () => {
const options = {
url: `http://${HOST}:${port}/500/invalidBody`,
+ throwOnHttpError: true,
};
let error;
@@ -188,9 +204,10 @@ describe('Apify.utils_request', () => {
expect(error.message).to.exist; // eslint-disable-line
});
- it('it returns json when 500 even if content-type is different', async () => {
+ it('it returns json when 500 even if content-type is different, throwOnHttpError is true ', async () => {
const options = {
url: `http://${HOST}:${port}/500/json`,
+ throwOnHttpError: true,
};
let error;
@@ -292,7 +309,7 @@ describe('Apify.utils_request', () => {
expect(error.message).to.eql('The response body is empty');
});
- it('it throws for other contentType then - text/html; charset=utf-8', async () => {
+ it('it throws for other contentType then - text/html', async () => {
const options = {
url: `http://${HOST}:${port}/invalidContentType`,
};