Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
added exit codes since the log is not vissible by default.
Petr Patekf79f13e467a7
diff --git a/.travis/check-user-agent.js b/.travis/check-user-agent.jsindex 68fb72875..4a83b1298 100644--- a/.travis/check-user-agent.js+++ b/.travis/check-user-agent.js@@ -16,10 +16,12 @@ async function main() {const latestStableVersion = getMajor(latestStabel.current_version);if (defaultUserAgentVersion === latestStableVersion) {console.log(`Default User-Agent has a correct Chrome version - ${defaultUserAgentVersion}.`)+ process.exit(0)} else {console.warn("Default User-Agent has incorrect version of Chrome!");console.warn("This can cause more blocking in head-full mode.");console.warn(`Default User-Agent: ${defaultUserAgentVersion} X Latest Chrome: ${latestStableVersion} `);+ process.exit(1)}}