Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fixed user-agents.
Petr Patek39bb6db9cffb
diff --git a/src/utils_request.js b/src/utils_request.jsindex e0f0a065f..21b3d0cb1 100644--- a/src/utils_request.js+++ b/src/utils_request.js@@ -3,8 +3,8 @@ import * as url from 'url';import _ from 'underscore';import httpRequest from '@apify/http-request';-const FIREFOX_MOBILE_USER_AGENT = 'Mozilla/5.0 (Android; Mobile; rv:14.0) Gecko/14.0 Firefox/14.0';-const FIREFOX_DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:68.0) Gecko/20100101 Firefox/68.0';+export const FIREFOX_MOBILE_USER_AGENT = 'Mozilla/5.0 (Android; Mobile; rv:14.0) Gecko/14.0 Firefox/14.0';+export const FIREFOX_DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:68.0) Gecko/20100101 Firefox/68.0';export const REQUEST_AS_BROWSER_DEFAULT_OPTIONS = {countryCode: 'US',@@ -27,7 +27,8 @@ export const REQUEST_AS_BROWSER_DEFAULT_OPTIONS = {* Currently, the function sends requests the same way as Firefox web browser does.* In the future, it might add support for other browsers too.*- * Internally, the function uses `@apify/httpRequest` function to perform the request.+ * Internally, the function uses `httpRequest` function from the+ * [`@apify/httpRequest`](https://github.com/apifytech/http-request) to perform the request.* All `options` not recognized by this function are passed to it,* so see it for more details.*@@ -65,9 +66,7 @@ export const requestAsBrowser = async (options) => {Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8','Accept-Language': `${opts.languageCode}-${opts.countryCode},${opts.languageCode};q=0.5`,'Accept-Encoding': 'gzip, deflate, br',- DNT: '1',Connection: 'keep-alive',- 'Upgrade-Insecure-Requests': '1',};opts.headers = _.defaults({}, opts.headers, browserHeaders);opts.abortFunction = abortFunction;diff --git a/test/utils_request.js b/test/utils_request.jsindex dc0b34801..8f2f7d94e 100644--- a/test/utils_request.js+++ b/test/utils_request.js@@ -1,7 +1,7 @@import { expect } from 'chai';import express from 'express';import { compress } from 'iltorb';-import { requestAsBrowser, REQUEST_AS_BROWSER_DEFAULT_OPTIONS } from '../build/utils_request';+import { requestAsBrowser, REQUEST_AS_BROWSER_DEFAULT_OPTIONS, FIREFOX_MOBILE_USER_AGENT, FIREFOX_DESKTOP_USER_AGENT } from '../build/utils_request';import { startExpressAppPromise } from './_helper';const CONTENT = 'CONTENT';@@ -61,7 +61,7 @@ describe('Apify.utils_request', () => {};const response = await requestAsBrowser(data);expect(response.statusCode).to.eql(200);- expect(JSON.parse(response.body)['user-agent']).to.be.eql('Mozilla/5.0 (Android 9.0; Mobile; rv:66.0) Gecko/66.0 Firefox/66.0');+ expect(JSON.parse(response.body)['user-agent']).to.be.eql(FIREFOX_MOBILE_USER_AGENT);});it('uses desktop user-agent by default ', async () => {@@ -73,7 +73,7 @@ describe('Apify.utils_request', () => {expect(JSON.parse(response.body)['user-agent']).to.be- .eql('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1.1 Safari/605.1.15');+ .eql(FIREFOX_DESKTOP_USER_AGENT);});it('sets correct hosts', async () => {