Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
added default user agent check.
Petr Patek2a3fba97fdf9
diff --git a/.travis.yml b/.travis.ymlindex 151199b63..a85cafd7d 100644--- a/.travis.yml+++ b/.travis.yml@@ -30,6 +30,11 @@ jobs:script:- "(cd website && npm install --no-package-lock)"- npm run lint && node .travis/check-docs.js+ - stage: test+ name: Check User-Agent+ node_js: '10'+ script:+ - node .travis/check-user-agent.js- stage: deployif: (branch = master AND type != pull_request) OR tag =~ /^v\d+\.\d+\.\d+$/script: skipdiff --git a/.travis/check-user-agent.js b/.travis/check-user-agent.jsnew file mode 100644index 000000000..68fb72875--- /dev/null+++ b/.travis/check-user-agent.js@@ -0,0 +1,33 @@+const {DEFAULT_USER_AGENT} = require('../build/constants');+const httpRequest = require("@apify/http-request");+const parse = require('csv-parse/lib/sync');+const UAParser = require("ua-parser-js");++const getMajor = (version) => {+ return version.split(".")[0];+};++async function main() {+ const {body} = await httpRequest({url: "https://omahaproxy.appspot.com/all?csv=1"});+ const csv = await parse(body, {columns: true});+ const latestStabel = csv.find(line => line.channel === "stable" && line.os === "linux");+ const UA = new UAParser(DEFAULT_USER_AGENT);+ const defaultUserAgentVersion = getMajor(UA.getBrowser().version);+ const latestStableVersion = getMajor(latestStabel.current_version);+ if (defaultUserAgentVersion === latestStableVersion) {+ console.log(`Default User-Agent has a correct Chrome version - ${defaultUserAgentVersion}.`)+ } else {+ console.warn("Default User-Agent has incorrect version of Chrome!");+ console.warn("This can cause more blocking in head-full mode.");+ console.warn(`Default User-Agent: ${defaultUserAgentVersion} X Latest Chrome: ${latestStableVersion} `);+ }++}++try {+ main();++} catch (e) {+ console.log("Could not check User-Agent");+ console.error(e)+}diff --git a/package.json b/package.jsonindex 51efd2d7e..6c207ed9a 100644--- a/package.json+++ b/package.json@@ -50,6 +50,7 @@"lint:fix": "./node_modules/.bin/eslint ./src ./test --ext .js,.jsx --fix"},"dependencies": {+ "@apify/http-request": "^1.0.1","@apify/ps-tree": "^1.1.3","apify-client": "^0.5.22","apify-shared": "^0.1.51",@@ -82,6 +83,7 @@"chai": "^4.2.0","chai-as-promised": "^7.1.1","chai-subset": "^1.6.0",+ "csv-parse": "^4.4.3","eslint": "^5.9.0","eslint-config-airbnb": "^17.1.0","eslint-config-airbnb-base": "^13.1.0",@@ -101,6 +103,7 @@"sinon": "^7.1.1","sinon-stub-promise": "^4.0.0","socket.io-client": "^2.2.0",+ "ua-parser-js": "^0.7.20","why-is-node-running": "^2.0.3"},"optionalDependencies": {