Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fixed check user agent script
Petr Patek1d5edf33be6b
diff --git a/.travis/check-user-agent.js b/.travis/check-user-agent.jsindex 6dd43e7da..b0e6cfc0a 100644--- a/.travis/check-user-agent.js+++ b/.travis/check-user-agent.js@@ -14,7 +14,7 @@ async function main() {const UA = new UAParser(DEFAULT_USER_AGENT);const defaultUserAgentVersion = getMajor(UA.getBrowser().version);const latestStableVersion = getMajor(latestStable.current_version);- if (defaultUserAgentVersion !== latestStableVersion) {+ if (defaultUserAgentVersion === latestStableVersion) {console.log(`Default User-Agent has the correct Chrome version - ${defaultUserAgentVersion}.`);process.exit(0);} else {diff --git a/src/constants.js b/src/constants.jsindex 31e840dbc..8e1181715 100644--- a/src/constants.js+++ b/src/constants.js@@ -4,7 +4,7 @@ import { ACTOR_EVENT_NAMES } from 'apify-shared/consts';* The default user agent used by `Apify.launchPuppeteer`.* Last updated on 2018-12-30.*/-export const DEFAULT_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36'; // eslint-disable-line max-len+export const DEFAULT_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36'; // eslint-disable-line max-len/*** Exit codes for the actor process.