Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
App - adding some basic system info
Marek Trunkat2077cf0355a9
diff --git a/src/index.js b/src/index.jsindex a6e58464c..3bde2fe5a 100644--- a/src/index.js+++ b/src/index.js@@ -15,7 +15,7 @@ import Request from './request';import { RequestList, openRequestList } from './request_list';import { openRequestQueue } from './request_queue';import SettingsRotator from './settings_rotator';-import { apifyClient, getMemoryInfo, isAtHome, publicUtils } from './utils';+import { apifyClient, getMemoryInfo, isAtHome, publicUtils, logSystemInfo } from './utils';import { browse, launchWebDriver } from './webdriver';import { puppeteerUtils } from './puppeteer_utils';import { socialUtils } from './utils_social';@@ -36,6 +36,8 @@ if (process.env[EMULATION_ENV_VAR]) {if (!process.env[ENV_VARS.LOCAL_STORAGE_DIR]) process.env[ENV_VARS.LOCAL_STORAGE_DIR] = process.env[EMULATION_ENV_VAR];}+logSystemInfo();+/*** The following section describes all functions and properties provided by the `apify` package,* except individual classes and namespaces that have their separate, detailed, documentation pagesdiff --git a/src/utils.js b/src/utils.jsindex b522478b8..0ba0d18cc 100644--- a/src/utils.js+++ b/src/utils.js@@ -8,14 +8,16 @@ import psTree from '@apify/ps-tree';import requestPromise from 'request-promise-native';import XRegExp from 'xregexp';import cheerio from 'cheerio';+import log from 'apify-shared/log';import { delayPromise, getRandomInt } from 'apify-shared/utilities';import { ENV_VARS, LOCAL_ENV_VARS } from 'apify-shared/consts';import { checkParamOrThrow } from 'apify-client/build/utils';+import { version as apifyClientVersion } from 'apify-client/package.json';+import { version as apifyVersion } from '../package.json';import { USER_AGENT_LIST } from './constants';/* globals process */-/*** Default regular expression to match URLs in a string that may be plain text, JSON, CSV or other. It supports common URL characters* and does not support URLs containing commas or spaces. The URLs also may contain Unicode letters (not symbols).@@ -53,6 +55,18 @@ export const newClient = () => {return new ApifyClient(opts);};+/**+ * Logs info about system, node version and apify package version.+ */+export const logSystemInfo = () => {+ log.info('System info', {+ apifyVersion,+ apifyClientVersion,+ osType: os.type(),+ nodeVersion: process.version,+ });+};+/*** Gets the default instance of the `ApifyClient` class provided* <a href="https://www.apify.com/docs/sdk/apify-client-js/latest"@@ -560,7 +574,6 @@ const htmlToText = (html) => {return text.trim();};-/*** A namespace that contains various utilities.*