Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

moved abortFunction to DEFAULTS

Petr Patekb9f92367f1ef
diff --git a/src/utils_request.js b/src/utils_request.js
index 70c685b02..2ae549886 100644
--- a/src/utils_request.js
+++ b/src/utils_request.js
@@ -14,6 +14,10 @@ export const REQUEST_AS_BROWSER_DEFAULT_OPTIONS = {
useMobileVersion: false,
useBrotli: true,
json: false,
+ abortFunction: (res) => {
+ const { type } = contentType.parse(res.headers['content-type']);
+ return res.statusCode === 406 || type.toLowerCase() !== 'text/html';
+ },
};
/**
* Sends a HTTP request that looks like a request sent by a web browser,
@@ -55,10 +59,6 @@ export const requestAsBrowser = async (options) => {
const opts = _.defaults({}, options, REQUEST_AS_BROWSER_DEFAULT_OPTIONS);
const parsedUrl = url.parse(opts.url);
- const abortFunction = (res) => {
- const { type } = contentType.parse(res.headers['content-type']);
- return res.statusCode === 406 || type.toLowerCase() !== 'text/html';
- };
const browserHeaders = {
Host: parsedUrl.host,
@@ -69,7 +69,6 @@ export const requestAsBrowser = async (options) => {
Connection: 'keep-alive',
};
opts.headers = _.defaults({}, opts.headers, browserHeaders);
- opts.abortFunction = abortFunction;
return httpRequest(opts);
};