Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
test failed
Petr Patek661e013adb87
diff --git a/.travis/check-user-agent.js b/.travis/check-user-agent.jsindex 4a83b1298..b8885e4d0 100644--- a/.travis/check-user-agent.js+++ b/.travis/check-user-agent.js@@ -1,35 +1,33 @@-const {DEFAULT_USER_AGENT} = require('../build/constants');-const httpRequest = require("@apify/http-request");+const httpRequest = require('@apify/http-request');const parse = require('csv-parse/lib/sync');-const UAParser = require("ua-parser-js");+const UAParser = require('ua-parser-js');+const { DEFAULT_USER_AGENT } = require('../build/constants');const getMajor = (version) => {- return version.split(".")[0];+ return version.split('.')[0];};async function main() {- const {body} = await httpRequest({url: "https://omahaproxy.appspot.com/all?csv=1"});- const csv = await parse(body, {columns: true});- const latestStabel = csv.find(line => line.channel === "stable" && line.os === "linux");+ const { body } = await httpRequest({ url: 'https://omahaproxy.appspot.com/all?csv=1' });+ const csv = await parse(body, { columns: true });+ const latestStabel = csv.find(line => line.channel === 'stable' && line.os === 'linux');const UA = new UAParser(DEFAULT_USER_AGENT);const defaultUserAgentVersion = getMajor(UA.getBrowser().version);const latestStableVersion = getMajor(latestStabel.current_version);- if (defaultUserAgentVersion === latestStableVersion) {- console.log(`Default User-Agent has a correct Chrome version - ${defaultUserAgentVersion}.`)- process.exit(0)+ if (defaultUserAgentVersion !== latestStableVersion) {+ console.log(`Default User-Agent has a correct Chrome version - ${defaultUserAgentVersion}.`);+ process.exit(0);} else {- console.warn("Default User-Agent has incorrect version of Chrome!");- console.warn("This can cause more blocking in head-full mode.");+ console.warn('Default User-Agent has incorrect version of Chrome!');+ console.warn('This can cause more blocking in head-full mode.');console.warn(`Default User-Agent: ${defaultUserAgentVersion} X Latest Chrome: ${latestStableVersion} `);- process.exit(1)+ process.exit(1);}-}try {main();-} catch (e) {- console.log("Could not check User-Agent");- console.error(e)+ console.log('Could not check User-Agent');+ console.error(e);}