Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

added default user agent check.

Petr Patek2a3fba97fdf9
diff --git a/.travis.yml b/.travis.yml
index 151199b63..a85cafd7d 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -30,6 +30,11 @@ jobs:
script:
- "(cd website && npm install --no-package-lock)"
- npm run lint && node .travis/check-docs.js
+ - stage: test
+ name: Check User-Agent
+ node_js: '10'
+ script:
+ - node .travis/check-user-agent.js
- stage: deploy
if: (branch = master AND type != pull_request) OR tag =~ /^v\d+\.\d+\.\d+$/
script: skip
diff --git a/.travis/check-user-agent.js b/.travis/check-user-agent.js
new file mode 100644
index 000000000..68fb72875
--- /dev/null
+++ b/.travis/check-user-agent.js
@@ -0,0 +1,33 @@
+const {DEFAULT_USER_AGENT} = require('../build/constants');
+const httpRequest = require("@apify/http-request");
+const parse = require('csv-parse/lib/sync');
+const UAParser = require("ua-parser-js");
+
+const getMajor = (version) => {
+ return version.split(".")[0];
+};
+
+async function main() {
+ const {body} = await httpRequest({url: "https://omahaproxy.appspot.com/all?csv=1"});
+ const csv = await parse(body, {columns: true});
+ const latestStabel = csv.find(line => line.channel === "stable" && line.os === "linux");
+ const UA = new UAParser(DEFAULT_USER_AGENT);
+ const defaultUserAgentVersion = getMajor(UA.getBrowser().version);
+ const latestStableVersion = getMajor(latestStabel.current_version);
+ if (defaultUserAgentVersion === latestStableVersion) {
+ console.log(`Default User-Agent has a correct Chrome version - ${defaultUserAgentVersion}.`)
+ } else {
+ console.warn("Default User-Agent has incorrect version of Chrome!");
+ console.warn("This can cause more blocking in head-full mode.");
+ console.warn(`Default User-Agent: ${defaultUserAgentVersion} X Latest Chrome: ${latestStableVersion} `);
+ }
+
+}
+
+try {
+ main();
+
+} catch (e) {
+ console.log("Could not check User-Agent");
+ console.error(e)
+}
diff --git a/package.json b/package.json
index 51efd2d7e..6c207ed9a 100644
--- a/package.json
+++ b/package.json
@@ -50,6 +50,7 @@
"lint:fix": "./node_modules/.bin/eslint ./src ./test --ext .js,.jsx --fix"
},
"dependencies": {
+ "@apify/http-request": "^1.0.1",
"@apify/ps-tree": "^1.1.3",
"apify-client": "^0.5.22",
"apify-shared": "^0.1.51",
@@ -82,6 +83,7 @@
"chai": "^4.2.0",
"chai-as-promised": "^7.1.1",
"chai-subset": "^1.6.0",
+ "csv-parse": "^4.4.3",
"eslint": "^5.9.0",
"eslint-config-airbnb": "^17.1.0",
"eslint-config-airbnb-base": "^13.1.0",
@@ -101,6 +103,7 @@
"sinon": "^7.1.1",
"sinon-stub-promise": "^4.0.0",
"socket.io-client": "^2.2.0",
+ "ua-parser-js": "^0.7.20",
"why-is-node-running": "^2.0.3"
},
"optionalDependencies": {