Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
moved abortFunction to DEFAULTS
Petr Patekb9f92367f1ef
diff --git a/src/utils_request.js b/src/utils_request.jsindex 70c685b02..2ae549886 100644--- a/src/utils_request.js+++ b/src/utils_request.js@@ -14,6 +14,10 @@ export const REQUEST_AS_BROWSER_DEFAULT_OPTIONS = {useMobileVersion: false,useBrotli: true,json: false,+ abortFunction: (res) => {+ const { type } = contentType.parse(res.headers['content-type']);+ return res.statusCode === 406 || type.toLowerCase() !== 'text/html';+ },};/*** Sends a HTTP request that looks like a request sent by a web browser,@@ -55,10 +59,6 @@ export const requestAsBrowser = async (options) => {const opts = _.defaults({}, options, REQUEST_AS_BROWSER_DEFAULT_OPTIONS);const parsedUrl = url.parse(opts.url);- const abortFunction = (res) => {- const { type } = contentType.parse(res.headers['content-type']);- return res.statusCode === 406 || type.toLowerCase() !== 'text/html';- };const browserHeaders = {Host: parsedUrl.host,@@ -69,7 +69,6 @@ export const requestAsBrowser = async (options) => {Connection: 'keep-alive',};opts.headers = _.defaults({}, opts.headers, browserHeaders);- opts.abortFunction = abortFunction;return httpRequest(opts);};