Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

save

Petr Patek66b70cb8ac17
diff --git a/src/crawlers/cheerio_crawler.js b/src/crawlers/cheerio_crawler.js
index dc304e5f8..1c2e3ba79 100644
--- a/src/crawlers/cheerio_crawler.js
+++ b/src/crawlers/cheerio_crawler.js
@@ -11,7 +11,7 @@ import BasicCrawler from './basic_crawler';
import { addTimeoutToPromise } from '../utils';
import { getApifyProxyUrl } from '../actor';
import { BASIC_CRAWLER_TIMEOUT_MULTIPLIER } from '../constants';
-import { requestAsBrowser } from '../utils_request';
+import { requestAsBrowser, TimeoutError } from '../utils_request';
/**
* Default mime types, which CheerioScraper supports.
@@ -40,6 +40,8 @@ const DEFAULT_OPTIONS = {
additionalMimeTypes: [],
};
+const TIMEOUT_ERROR_MESSAGE = 'CheerioCrawler: requestFunction timed out.';
+
/**
* Provides a framework for the parallel crawling of web pages using plain HTTP requests and
* <a href="https://www.npmjs.com/package/cheerio" target="_blank">cheerio</a> HTML parser.
@@ -356,7 +358,7 @@ class CheerioCrawler {
const { dom, isXmlOrHtml, body, contentType, responseStream: response } = await addTimeoutToPromise(
this._requestFunction({ request }),
this.requestTimeoutMillis,
- 'CheerioCrawler: requestFunction timed out.',
+ TIMEOUT_ERROR_MESSAGE,
);
request.loadedUrl = response.url;
@@ -403,8 +405,17 @@ class CheerioCrawler {
// Using the streaming API of Request to be able to
// handle the response based on headers receieved.
const opts = this._getRequestOptions(request);
-
- const responseStream = await requestAsBrowser(opts);
+ let responseStream;
+
+ try {
+ responseStream = await requestAsBrowser(opts);
+ } catch (e) {
+ if (e instanceof TimeoutError) {
+ throw new Error(TIMEOUT_ERROR_MESSAGE);
+ } else {
+ throw e;
+ }
+ }
const { statusCode, headers } = responseStream;
const contentType = contentTypeParser.parse(headers['content-type']);
const { type, encoding } = contentType;
@@ -463,7 +474,7 @@ class CheerioCrawler {
return false;
},
- timeoutSecs: this.requestTimeoutMillis / 1000 + 1,
+ timeoutSecs: this.requestTimeoutMillis / 1000,
};
if (/PATCH|POST|PUT/.test(request.method)) mandatoryRequestOptions.payload = request.payload;
diff --git a/src/utils_request.js b/src/utils_request.js
index 712910bea..4d92760a4 100644
--- a/src/utils_request.js
+++ b/src/utils_request.js
@@ -2,10 +2,12 @@ import contentType from 'content-type';
import * as url from 'url';
import _ from 'underscore';
import httpRequest from '@apify/http-request';
+import errors from '@apify/http-request/src/errors';
export const FIREFOX_MOBILE_USER_AGENT = 'Mozilla/5.0 (Android; Mobile; rv:14.0) Gecko/14.0 Firefox/14.0';
export const FIREFOX_DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:68.0) Gecko/20100101 Firefox/68.0';
+
export const REQUEST_AS_BROWSER_DEFAULT_OPTIONS = {
countryCode: 'US',
languageCode: 'en',
@@ -78,5 +80,17 @@ export const requestAsBrowser = async (options) => {
Connection: 'keep-alive',
};
opts.headers = _.defaults({}, opts.headers, defaultHeaders);
- return httpRequest(opts);
+
+ try {
+ return await httpRequest(opts);
+ } catch (e) {
+ if (e instanceof errors.TimeoutError) {
+ throw new TimeoutError('Request Timed-out');
+ }
+ }
};
+
+/**
+ * TimeoutError helper class
+ */
+export class TimeoutError extends Error {}