Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
renamed browserHeaders to defaultHeaders.
Petr Patekbab497d05aab
diff --git a/src/utils_request.js b/src/utils_request.jsindex 2ae549886..20a33934c 100644--- a/src/utils_request.js+++ b/src/utils_request.js@@ -60,7 +60,7 @@ export const requestAsBrowser = async (options) => {const parsedUrl = url.parse(opts.url);- const browserHeaders = {+ const defaultHeaders = {Host: parsedUrl.host,'User-Agent': opts.useMobileVersion ? FIREFOX_MOBILE_USER_AGENT : FIREFOX_DESKTOP_USER_AGENT,Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',@@ -68,7 +68,7 @@ export const requestAsBrowser = async (options) => {'Accept-Encoding': 'gzip, deflate, br',Connection: 'keep-alive',};- opts.headers = _.defaults({}, opts.headers, browserHeaders);+ opts.headers = _.defaults({}, opts.headers, defaultHeaders);return httpRequest(opts);};