Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

improved message.

Petr Patek2d9ce62c3ab0
diff --git a/src/crawlers/cheerio_crawler.js b/src/crawlers/cheerio_crawler.js
index 7de5e45ab..f2c420291 100644
--- a/src/crawlers/cheerio_crawler.js
+++ b/src/crawlers/cheerio_crawler.js
@@ -41,8 +41,6 @@ const DEFAULT_OPTIONS = {
additionalMimeTypes: [],
};
-const TIMEOUT_ERROR_MESSAGE = 'CheerioCrawler: requestFunction timed out.';
-
/**
* Provides a framework for the parallel crawling of web pages using plain HTTP requests and
* <a href="https://www.npmjs.com/package/cheerio" target="_blank">cheerio</a> HTML parser.
@@ -359,7 +357,7 @@ class CheerioCrawler {
const { dom, isXmlOrHtml, body, contentType, responseStream: response } = await addTimeoutToPromise(
this._requestFunction({ request }),
this.requestTimeoutMillis,
- TIMEOUT_ERROR_MESSAGE,
+ `CheerioCrawler: requestFunction timed out after ${this.requestTimeoutMillis}.`,
);
request.loadedUrl = response.url;
@@ -412,7 +410,7 @@ class CheerioCrawler {
responseStream = await requestAsBrowser(opts);
} catch (e) {
if (e instanceof TimeoutError) {
- throw new Error(TIMEOUT_ERROR_MESSAGE);
+ throw new Error(`CheerioCrawler: requestFunction timed out after ${this.requestTimeoutMillis}.`);
} else {
throw e;
}
diff --git a/src/utils_request.js b/src/utils_request.js
index f2c064c20..6d6f440b6 100644
--- a/src/utils_request.js
+++ b/src/utils_request.js
@@ -24,6 +24,7 @@ export const REQUEST_AS_BROWSER_DEFAULT_OPTIONS = {
useCaseSensitiveHeaders: true,
useStream: false,
proxyUrl: null,
+ timeoutSecs: 30,
};
/**
* Sends a HTTP request that looks like a request sent by a web browser,
@@ -87,7 +88,7 @@ export const requestAsBrowser = async (options) => {
return await httpRequest(opts);
} catch (e) {
if (e instanceof errors.TimeoutError) {
- throw new TimeoutError('Request Timed-out');
+ throw new TimeoutError(`Request Timed-out after ${opts.timeoutSecs} seconds.`);
}
throw e;