Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Feature/new logger (#570)

Marek Trunkát758cff1ebf45
diff --git a/docs/api/log.md b/docs/api/log.md
index a417c67f4..99140c3ab 100644
--- a/docs/api/log.md
+++ b/docs/api/log.md
@@ -41,6 +41,12 @@ try {
} catch (e) {
log.exception(e, 'Exception occurred', { errorDetails: 'This is really bad!' }); // prints message
}
+
+log.setOptions({ prefix: 'My actor' });
+log.info('I am running!'); // prints "My actor: I am running"
+
+const childLog = log.child({ prefix: 'Crawler' });
+log.info('I am crawling!'); // prints "My actor:Crawler: I am crawling"
```
Another very useful way of setting the log level is by setting the `APIFY_LOG_LEVEL` environment variable, such as `APIFY_LOG_LEVEL=DEBUG`. This way,
@@ -86,6 +92,40 @@ level is not high enough at the moment, it doesn't make sense to execute the que
---
+<a name="setoptions"></a>
+
+## `log.setOptions(options)`
+
+Configures logger.
+
+**Params**
+
+- **`options`**: [`LoggerOptions`](/docs/typedefs/logger-options)
+
+---
+
+<a name="child"></a>
+
+## `log.child([options])`
+
+Creates a new instance of logger that inherits settings from a parent logger.
+
+**Params**
+
+- **`[options]`**: [`LoggerOptions`](/docs/typedefs/logger-options) - Supports the same options as the `setOptions()` method.
+
+---
+
+<a name="getoptions"></a>
+
+## `log.getOptions()`
+
+**Returns**: [`LoggerOptions`](/docs/typedefs/logger-options)
+
+Returns the logger configuration.
+
+---
+
<a name="debug"></a>
## `log.debug(message, [data])`
diff --git a/docs/typedefs/LoggerOptions.md b/docs/typedefs/LoggerOptions.md
new file mode 100644
index 000000000..1b3615a5e
--- /dev/null
+++ b/docs/typedefs/LoggerOptions.md
@@ -0,0 +1,66 @@
+---
+id: logger-options
+title: LoggerOptions
+---
+
+<a name="loggeroptions"></a>
+
+## Properties
+
+### `level`
+
+**Type**: `number` <code> = 4</code>
+
+Sets the log level to the given value, preventing messages from less important log levels from being printed to the console. Use in conjunction with
+the `log.LEVELS` constants.
+
+---
+
+### `maxDepth`
+
+**Type**: `number` <code> = 4</code>
+
+Max depth of data object that will be logged. Anything deeper than the limit will be stripped off.
+
+---
+
+### `maxStringLength`
+
+**Type**: `number` <code> = 2000</code>
+
+Max length of the string to be logged. Longer strings will be truncated.
+
+---
+
+### `prefix`
+
+**Type**: `string`
+
+Prefix to be prepended the each logged line.
+
+---
+
+### `suffix`
+
+**Type**: `string`
+
+Suffix that will be appended the each logged line.
+
+---
+
+### `logger`
+
+**Type**: `Object`
+
+Logger implementation to be used. Default one is log.LoggerText to log messages as easily readable strings. Optionally you can use `log.LoggerJson`
+that formats each log line as a JSON.
+
+---
+
+### `data`
+
+**Type**: `Object`
+
+Additional data to be added to each log line.
+
+---
diff --git a/package.json b/package.json
index 95cfaf639..011713e0d 100644
--- a/package.json
+++ b/package.json
@@ -58,7 +58,7 @@
"@types/puppeteer": "^2.0.0",
"@types/tough-cookie": "^2.3.6",
"apify-client": "^0.5.25",
- "apify-shared": "^0.1.66",
+ "apify-shared": "^0.2.5",
"cheerio": "^1.0.0-rc.3",
"content-type": "^1.0.4",
"express": "^4.17.1",
diff --git a/src/index.esm.js b/src/index.esm.js
index 7587a75d4..019d5a9f2 100644
--- a/src/index.esm.js
+++ b/src/index.esm.js
@@ -26,9 +26,6 @@ import { Session } from './session_pool/session';
// Increase the global limit for event emitter memory leak warnings.
EventEmitter.defaultMaxListeners = 50;
-// Log as plain text not JSON
-log.logJson = false;
-
const exportedUtils = Object.assign(publicUtils, {
puppeteer: puppeteerUtils,
social: socialUtils,
diff --git a/src/utils_log.d.ts b/src/utils_log.d.ts
index bbf38366e..e554c1f7c 100644
--- a/src/utils_log.d.ts
+++ b/src/utils_log.d.ts
@@ -4,9 +4,6 @@ declare module log {
OFF: number,
// For unexpected errors in Apify system
ERROR: number,
- // For situations where error is caused by user (e.g. Meteor.Error), i.e. when the error is not
- // caused by Apify system, avoid the word "ERROR" to simplify searching in log
- SOFT_FAIL: number,
WARNING: number,
INFO: number,
DEBUG: number,
@@ -14,11 +11,6 @@ declare module log {
PERF: number
};
- let logJson: boolean;
-
- // Indicates whether DEBUG messages will be printed or not
- let isDebugMode: boolean;
-
// Sets log level
function setLevel(level: number): void;
diff --git a/src/utils_log.js b/src/utils_log.js
index 2414ece14..8ee209d95 100644
--- a/src/utils_log.js
+++ b/src/utils_log.js
@@ -3,6 +3,19 @@ import log from 'apify-shared/log';
// Add docs for log separately, as it's imported from apify-shared.
// Adding them directly to the log object in utils in `index.js` breaks JSDoc.
+/**
+ * @typedef {Object} LoggerOptions
+ * @property {number} [level=4] Sets the log level to the given value, preventing messages from less important log levels
+ * from being printed to the console. Use in conjunction with the `log.LEVELS` constants.
+ * @property {number} [maxDepth=4] Max depth of data object that will be logged. Anything deeper than the limit will be stripped off.
+ * @property {number} [maxStringLength=2000] Max length of the string to be logged. Longer strings will be truncated.
+ * @property {string} [prefix] Prefix to be prepended the each logged line.
+ * @property {string} [suffix] Suffix that will be appended the each logged line.
+ * @property {Object} [logger] Logger implementation to be used. Default one is log.LoggerText to log messages as easily readable
+ * strings. Optionally you can use `log.LoggerJson` that formats each log line as a JSON.
+ * @property {Object} [data] Additional data to be added to each log line.
+ */
+
/**
* The log instance enables level aware logging of messages and we advise
* to use it instead of `console.log()` and its aliases in most development
@@ -43,6 +56,12 @@ import log from 'apify-shared/log';
* } catch (e) {
* log.exception(e, 'Exception occurred', { errorDetails: 'This is really bad!' }); // prints message
* }
+ *
+ * log.setOptions({ prefix: 'My actor' });
+ * log.info('I am running!'); // prints "My actor: I am running"
+ *
+ * const childLog = log.child({ prefix: 'Crawler' });
+ * log.info('I am crawling!'); // prints "My actor:Crawler: I am crawling"
* ```
*
* Another very useful way of setting the log level is by setting the `APIFY_LOG_LEVEL`
@@ -84,6 +103,30 @@ import log from 'apify-shared/log';
* @memberOf log
*/
+/**
+ * Configures logger.
+ * @name setOptions
+ * @param {LoggerOptions} options
+ * @method
+ * @memberOf log
+ */
+
+/**
+ * Creates a new instance of logger that inherits settings from a parent logger.
+ * @name child
+ * @param {LoggerOptions} [options] Supports the same options as the `setOptions()` method.
+ * @method
+ * @memberOf log
+ */
+
+/**
+ * Returns the logger configuration.
+ * @name getOptions
+ * @method
+ * @returns {LoggerOptions}
+ * @memberOf log
+ */
+
/**
* Logs a `DEBUG` message. By default, it will not be written to the console. To see `DEBUG`
* messages in the console, set the log level to `DEBUG` either using the `log.setLevel(log.LEVELS.DEBUG)`
diff --git a/test/request_list.test.js b/test/request_list.test.js
index 0fffe3bac..ed01e6d8a 100644
--- a/test/request_list.test.js
+++ b/test/request_list.test.js
@@ -685,7 +685,7 @@ describe('Apify.RequestList', () => {
expect(requestList.length()).toBe(4);
log.setLevel(log.LEVELS.INFO);
- const logStub = sinon.stub(console, 'log');
+ const logStub = sinon.stub(console, 'warn');
requestList = new Apify.RequestList({
sources: sources.concat([
diff --git a/types/index.esm.d.ts b/types/index.esm.d.ts
index f2ccf7fb8..08b03bd07 100644
--- a/types/index.esm.d.ts
+++ b/types/index.esm.d.ts
@@ -105,7 +105,7 @@ declare const exportedUtils: {
YOUTUBE_REGEX: RegExp;
YOUTUBE_REGEX_GLOBAL: RegExp;
};
- log: typeof log;
+ log: any;
enqueueLinks: typeof enqueueLinks;
requestAsBrowser: (options: import("./utils_request").RequestAsBrowserOptions) => Promise<any>;
};
diff --git a/website/i18n/en.json b/website/i18n/en.json
index 6374433bd..2eca8a47d 100644
--- a/website/i18n/en.json
+++ b/website/i18n/en.json
@@ -194,6 +194,9 @@
"typedefs/launch-puppeteer-options": {
"title": "LaunchPuppeteerOptions"
},
+ "typedefs/logger-options": {
+ "title": "LoggerOptions"
+ },
"typedefs/memory-info": {
"title": "MemoryInfo"
},
diff --git a/website/sidebars.json b/website/sidebars.json
index c7c1a00c2..1baa758d4 100644
--- a/website/sidebars.json
+++ b/website/sidebars.json
@@ -73,6 +73,7 @@
"typedefs/cheerio-crawler-options",
"typedefs/puppeteer-crawler-options",
"typedefs/puppeteer-pool-options",
+ "typedefs/logger-options",
"typedefs/launch-puppeteer-options",
"typedefs/request-options",
"typedefs/request-list-options",