Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Move logging from require time to Apify.main execution
Ondra Urband6937dc795af
diff --git a/src/actor.js b/src/actor.jsindex 48054e264..621ffc37d 100644--- a/src/actor.js+++ b/src/actor.js@@ -6,7 +6,7 @@ import { APIFY_PROXY_VALUE_REGEX } from 'apify-shared/regexs';import { ENV_VARS, INTEGER_ENV_VARS, LOCAL_ENV_VARS, ACT_JOB_TERMINAL_STATUSES, ACT_JOB_STATUSES } from 'apify-shared/consts';import { EXIT_CODES, COUNTRY_CODE_REGEX } from './constants';import { initializeEvents, stopEvents } from './events';-import { apifyClient, addCharsetToContentType, sleep, snakeCaseToCamelCase, isAtHome } from './utils';+import { apifyClient, addCharsetToContentType, sleep, snakeCaseToCamelCase, isAtHome, logSystemInfo, printOutdatedSdkWarning } from './utils';import { maybeStringify } from './key_value_store';import { ApifyCallError } from './errors';@@ -228,6 +228,12 @@ export const main = (userFunc) => {throw new Error(`Apify.main() accepts a single parameter that must be a function (was '${userFunc === null ? 'null' : typeof (userFunc)}').`);}+ // Logging some basic system info (apify and apify-client version, NodeJS version, ...).+ logSystemInfo();++ // Log warning if SDK is outdated.+ printOutdatedSdkWarning();+if (!process.env[ENV_VARS.LOCAL_STORAGE_DIR] && !process.env[ENV_VARS.TOKEN]) {const dir = path.join(process.cwd(), './apify_storage');process.env[ENV_VARS.LOCAL_STORAGE_DIR] = dir;diff --git a/src/index.js b/src/index.jsindex 172a32565..ce55cd791 100644--- a/src/index.js+++ b/src/index.js@@ -1,6 +1,5 @@import EventEmitter from 'events';import log from 'apify-shared/log';-import { ENV_VARS } from 'apify-shared/consts';import { main, getEnv, call, callTask, getApifyProxyUrl, metamorph, addWebhook } from './actor';import AutoscaledPool from './autoscaling/autoscaled_pool';import BasicCrawler from './crawlers/basic_crawler';@@ -14,7 +13,7 @@ import PuppeteerPool from './puppeteer_pool';import Request from './request';import { RequestList, openRequestList } from './request_list';import { openRequestQueue } from './request_queue';-import { apifyClient, getMemoryInfo, isAtHome, publicUtils, logSystemInfo, printOutdatedSdkWarning } from './utils';+import { apifyClient, getMemoryInfo, isAtHome, publicUtils } from './utils';import { puppeteerUtils } from './puppeteer_utils';import { socialUtils } from './utils_social';import { enqueueLinks } from './enqueue_links/enqueue_links';@@ -30,19 +29,6 @@ EventEmitter.defaultMaxListeners = 50;// Log as plain text not JSONlog.logJson = false;-// TODO: remove this when we release v1.0.0-const EMULATION_ENV_VAR = 'APIFY_LOCAL_EMULATION_DIR';-if (process.env[EMULATION_ENV_VAR]) {- log.warning(`Environment variable "${EMULATION_ENV_VAR}" is deprecated!!! Use "${ENV_VARS.LOCAL_STORAGE_DIR}" instead!`);- if (!process.env[ENV_VARS.LOCAL_STORAGE_DIR]) process.env[ENV_VARS.LOCAL_STORAGE_DIR] = process.env[EMULATION_ENV_VAR];-}--// Logging some basic system info (apify and apify-client version, NodeJS version, ...).-logSystemInfo();--// Log warning if SDK is outdated.-printOutdatedSdkWarning();-/*** The following section describes all functions and properties provided by the `apify` package,* except individual classes and namespaces that have their separate, detailed, documentation pages