Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

feature/Json-sink-keeps-rich · L-260918-83431f (#1392)

Louis Choquel2345dbca17d8
diff --git a/.drift/acks/cli-docs.toml b/.drift/acks/cli-docs.toml
index 34e934523..1e4b2da1b 100644
--- a/.drift/acks/cli-docs.toml
+++ b/.drift/acks/cli-docs.toml
@@ -1,8 +1,8 @@
contract = "cli-docs"
-digest = "sha256:7398755782f220e2cb9fa6ba5396251d5cd1dfd16fdfce691421472189c06279"
-reviewed_by = "Louis Choquel"
-reviewed_at = "2026-10-09T19:02:50Z"
-rationale = "clean-pass: the merge of dev into the log-pruning branch joins two trigger changes each reviewed on its own side, the deletion of VERBOSE lines in pipelex run's output writer and in pipelex which, and the agent CLI's hosted refusal envelope reading error_domain and retryable through sent_verdict. They touch different files and neither moves an option, an output file, a printed line or an exit behaviour the other's review relied on, so docs/tools/cli/ and the agent CLI CLAUDE.md still read as both reviews left them; no docs change."
+digest = "sha256:33069c67291228668d0f75b69195fb898918372b58dd416dc57d5f75cc9be789"
+reviewed_by = "Claude (Opus 5.5)"
+reviewed_at = "2026-10-09T20:06:34Z"
+rationale = "clean-pass: doctor_cmd.py now escapes the credentials advice at its print, and backend_credentials.py writes the table as [openai] instead of the Rich escape, so the doctor's Backend Credentials row prints the same [openai] it did before; graph_cmd.py's refused-spec branch keeps its console print and only turns its log line into a fixed message with file.path, error.type and error.message fields. Re-read doctor.md (the Backend Credentials row's advice, still accurate), run.md (no graph render failure wording) and agent_cli/CLAUDE.md (the agent doctor's envelope, which does not carry the comprehensive advice text, verified by grepping agent_cli for its factory): nothing stale."
[trigger_files]
"pipelex/cli/__init__.py" = "blob:8b137891791fe96927ad78e64b0aad7bded08bdc"
@@ -74,13 +74,13 @@ rationale = "clean-pass: the merge of dev into the log-pruning branch joins two
"pipelex/cli/commands/codegen/inputs_cmd.py" = "blob:28b22cfbca717d3f5ee75162b68e132439655deb"
"pipelex/cli/commands/codegen/types_cmd.py" = "blob:27d7a06d571ad832b449d21b6f9eb66c017fb197"
"pipelex/cli/commands/crate_loading.py" = "blob:3d7d651931ecf1a15a3e2d72411b3212b55d74ef"
-"pipelex/cli/commands/doctor_cmd.py" = "blob:84439f945eabf34917d73dc843d9513ed3daeb25"
+"pipelex/cli/commands/doctor_cmd.py" = "blob:ba2b1f3985b09aea8430410dad6c6bdbf0a2c599"
"pipelex/cli/commands/fix/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"
"pipelex/cli/commands/fix/_diff_sandbox.py" = "blob:d39c0df92b95567f74881deb01f8c1b3119fc8ef"
"pipelex/cli/commands/fix/_fix_core.py" = "blob:aa698305e6b60dfb7caf133698dc1d5c1f5f68f1"
"pipelex/cli/commands/fix/app.py" = "blob:6f019930e8e3e0043ee46d87493ba4602eb33a76"
"pipelex/cli/commands/fix/bundle_cmd.py" = "blob:427364cc453957dfe0cec879f019050329c13b3c"
-"pipelex/cli/commands/graph_cmd.py" = "blob:d54737af7efc9ab9bbf3f978050d017af0e27a41"
+"pipelex/cli/commands/graph_cmd.py" = "blob:69402efebdd624ccad432256718c468c868f93b0"
"pipelex/cli/commands/init/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"
"pipelex/cli/commands/init/backends.py" = "blob:04a1a06f1003ed0c7157ec4ca6c8659e4e09fe6b"
"pipelex/cli/commands/init/command.py" = "blob:f5cbca22f8ea19957f271af2d9844d9191263520"
@@ -108,7 +108,7 @@ rationale = "clean-pass: the merge of dev into the log-pruning branch joins two
"pipelex/cli/commands/run/_inputs_file_loader.py" = "blob:957e6eb00531d168b3b1e179a517717d069cb7fc"
"pipelex/cli/commands/run/_inputs_path_resolver.py" = "blob:c8ec89d98718e18af7aaaded44c9f545be60adab"
"pipelex/cli/commands/run/_main_stuff_file.py" = "blob:ca806e55085aa2e5748492751521296ce1f42960"
-"pipelex/cli/commands/run/_run_core.py" = "blob:90832f803424b691e30fab0677540ec54f98c66d"
+"pipelex/cli/commands/run/_run_core.py" = "blob:7e5b2fd0a44ad04b254c77384f24a370bc4e80b1"
"pipelex/cli/commands/run/_run_hosted.py" = "blob:dd5801877a636f71187e9b23c2fab18d4d25ba32"
"pipelex/cli/commands/run/app.py" = "blob:e312b159625e0a72bc9a18d3bfd2c73e7ac59f24"
"pipelex/cli/commands/run/bundle_cmd.py" = "blob:aba274dc8d6d0ac08baa3f21afbc06b2fdbec356"
diff --git a/.drift/acks/config-docs.toml b/.drift/acks/config-docs.toml
index e53a89150..ddb183aa9 100644
--- a/.drift/acks/config-docs.toml
+++ b/.drift/acks/config-docs.toml
@@ -1,8 +1,8 @@
contract = "config-docs"
-digest = "sha256:da981a7560d55779afc3d749b23978386605f9333bd8e9c2d25954b7cad15243"
+digest = "sha256:06083f22760900182f7c24332130ecebcde8874745159b69e6acf39e737f38eb"
reviewed_by = "Claude (Opus 5.5)"
-reviewed_at = "2026-10-09T18:11:07Z"
-rationale = "real-catch: log_levels.py joins the triggers because the LogLevel enum is what the config docs list. Re-derived from the enum its members VERBOSE, DEBUG, INFO, WARNING, ERROR, CRITICAL and OFF, which match the valid values logging-config.md gives for default_log_level and its gcp severity note naming VERBOSE as the only custom level; removing LogLevel.from_int touches no setting. The review caught logging-config.md's Migrating From the DEV Level section claiming a file naming DEV stops the boot: ConfigLoader.load_config_validated replays the ledger in memory on a validation failure and pipelex-config@6 is a safe remap, so such a file boots at DEBUG with a stale-configuration warning, now said there and pinned by a test through the loader. The ledger entry, configs.py and config_cogt.py name no level beyond that and are unchanged."
+reviewed_at = "2026-10-09T19:29:59Z"
+rationale = "real-catch: RichLogConfig loses is_markup_enabled and pipelex.toml its default. Grepped docs/configuration for the key and for rich_log: logging-config.md listed it under Syntax Highlighting and in the full example, both removed, and the section now says the console reads no message as markup and that a file still setting the key boots, deleted in memory by pipelex-config@7 with a stale-configuration warning, which tests/unit/pipelex/migration/test_markup_setting_retirement.py pins through ConfigLoader.load_config_validated. The ledger gains pipelex-config@7 (delete_key, safe) with defaults@7 and fingerprint@7; configs.py and config_cogt.py name no rich_log setting and are unchanged."
[trigger_files]
"pipelex/cogt/config_cogt.py" = "blob:02f71cb4a6b01aebda59c2e31f262ef9c865e17b"
@@ -10,7 +10,7 @@ rationale = "real-catch: log_levels.py joins the triggers because the LogLevel e
"pipelex/graph/graph_config.py" = "blob:dc0939a6d2730c0a10b88aa1a6e0b2dd583462a6"
"pipelex/language/mthds_config.py" = "blob:353009ad8c59dc5266bdd9dfa42b1ad18b67dca7"
"pipelex/methods/methods_config.py" = "blob:76ca9a0e64f8c10ef6a355aafd0d79d36b09f06a"
-"pipelex/pipelex.toml" = "blob:753da7e4cefacf1cc809ba9993c003b43e7e450e"
+"pipelex/pipelex.toml" = "blob:af0f9f1f117c404333866b8c60ae61d95afab962"
"pipelex/system/configuration/__init__.py" = "blob:8b137891791fe96927ad78e64b0aad7bded08bdc"
"pipelex/system/configuration/config_check.py" = "blob:2d5de29aba3d2ce205d3d82b4623f37b53108332"
"pipelex/system/configuration/config_loader.py" = "blob:7563f68051a568ba90d3209c1966d678078c6de2"
@@ -21,7 +21,7 @@ rationale = "real-catch: log_levels.py joins the triggers because the LogLevel e
"pipelex/system/configuration/pipe_func_config.py" = "blob:dc8890e914fbcdb611147e42eaa0cd87b32288a5"
"pipelex/system/telemetry/telemetry_config.py" = "blob:068a785d97a2d52ee43e82e723cd944b8b6601bc"
"pipelex/tools/aws/aws_config.py" = "blob:e27dc96bf0b2ef98a45354712421252fe0b893b5"
-"pipelex/tools/log/log_config.py" = "blob:63dd9f81fdbafc1a095aec6cde1b047d0ebdd2d4"
+"pipelex/tools/log/log_config.py" = "blob:4c2f9545758e9c04228aedd88adb68125adad7aa"
"pipelex/tools/log/log_levels.py" = "blob:12b94e4d4793a70156ac547ee20c9d483910ecae"
"pipelex/tools/network/network_config.py" = "blob:e12d1d0e4a40d04859f10a77addeaf8fe2b00414"
"pipelex/tools/secrets/secrets_config.py" = "blob:cf30aaec1c76c9cabc0531a5b9fe681e153d7024"
diff --git a/.drift/acks/pipelex-kernel-docs.toml b/.drift/acks/pipelex-kernel-docs.toml
index 528754aef..b10f9f4e0 100644
--- a/.drift/acks/pipelex-kernel-docs.toml
+++ b/.drift/acks/pipelex-kernel-docs.toml
@@ -1,8 +1,8 @@
contract = "pipelex-kernel-docs"
-digest = "sha256:a67b12d7b36c494bbc641d0ed14bd2c574e6ce49a8a3588b1032bf561e796287"
+digest = "sha256:f5bde33c053cdc8c339cad7478dace7beeb9f9a3323e22e1fb98992bdb05717a"
reviewed_by = "Claude (Opus 5.5)"
-reviewed_at = "2026-10-09T17:40:08Z"
-rationale = "clean-pass: the trigger changes delete VERBOSE lines in llm_ops.py, img_gen_prompt.py and llm_prompt_content.py (the per-image and per-document extraction lines, the prompting-style line and the object-direct generation lines) and the imports they used; no kernel function, parameter or behaviour moved. pipelex-kernel.md's log-context section was re-checked against the code: every kernel function taking a job_metadata still opens with job_metadata.log_context(), and the page quotes none of the deleted lines."
+reviewed_at = "2026-10-09T20:31:54Z"
+rationale = "clean-pass: llm_ops.py's VERBOSE line for a model whose backend SDK is not installed gains gen_ai.request.model (the deck handle) and gen_ai.response.model (the provider's model id), the keys and meanings of the LLM span; no kernel function, parameter or behaviour moved. Re-read pipelex-kernel.md's check_llm_setting_with_served_model paragraph: such a model is still left to the run, and the doc names no log field."
[trigger_files]
"pipelex/kernel/__init__.py" = "blob:c4f1b0fe2ccc17db9110733f71b612f4ac1ba4ce"
@@ -18,7 +18,7 @@ rationale = "clean-pass: the trigger changes delete VERBOSE lines in llm_ops.py,
"pipelex/kernel/img_gen_results.py" = "blob:46b4ca11c70d906ae6d58ce66461f9b3414f1bfa"
"pipelex/kernel/judgment_ops.py" = "blob:49ce7414eed9cfdf5595e877a64aacd29ada86ca"
"pipelex/kernel/judgment_results.py" = "blob:40ba1258cc81d34df3e427d5511ccd40c6edf789"
-"pipelex/kernel/llm_ops.py" = "blob:c106daf0905dd9548a2a3661d580901abff52faa"
+"pipelex/kernel/llm_ops.py" = "blob:4002e83ea886140a930d994a717d3ef329453d25"
"pipelex/kernel/llm_prompt_content.py" = "blob:eabf0ee760c3a26b8967973d511e712e9275e99b"
"pipelex/kernel/llm_results.py" = "blob:6e3b8a2878b5bb284ece5fac10262fb59ac2029d"
"pipelex/kernel/memory_ops.py" = "blob:7fbbfaaec99f9712d8e215520cb02e26aad3fb40"
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 418be957f..8c8f95d0a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,19 @@
- **The console shows a log call's fields and renders named layouts**: the `console` sink prints a record's fields after its message as a `key=value` suffix, `Scanned the inputs files=7`, each value on one line and cut short when long, leaving out the run identifiers, and colours a value by its field's name, a pipe code red and a concept bold green, with any other field dimmed. A call can name a console layout with `layout=`, a Rich template over its fields in which every value is escaped, starting with `LogLayout.PIPE_RUN` for the pipe-run tree; every other sink writes the plain message and the fields.
+### Changed
+
+- **The pipe announcement is a plain message with fields**: every live pipe run logs `Pipe run starts` with `pipe_type`, `pipe_code`, `output_concept` and `pipe_depth` as fields, where the `json`, `otlp` and `gcp` sinks used to receive Rich markup inside the message, and the console draws the same coloured pipe tree from those fields through `LogLayout.PIPE_RUN`. The warning that a model's fixed temperature replaced the requested one carries its values as fields in the same way: `model_handle`, `backend_name`, `sdk`, the model's handle under `gen_ai.request.model` and the provider's model id under `gen_ai.response.model` as on the LLM call's span, the requested temperature under `gen_ai.request.temperature`, and `fixed_temperature`; the verbose line saying a model's settings were not checked, since its backend's SDK is not installed, carries the same model fields.
+
+### Fixed
+
+- **Model descriptions and credential errors carry no backslashes**: a model's description in an error message or a log line, such as `gpt-5 → SDK[openai]•Backend[openai]•Model[gpt-5-2025]`, no longer has a backslash before each bracket, and the error naming a backend's missing credentials tells you to add `enabled = false` under `[openai]` rather than `\[openai]`. `pipelex run` prints a failed run's message exactly as written rather than reading it as markup.
+- **An empty list's output panel says so**: the title of the panel showing an operator pipe's output ends with `[empty list]` when the pipe produced an empty list, a label the console used to drop, as it already showed `[1 item]` or `[3 items]`.
+
+### Removed
+
+- **`is_markup_enabled` (Breaking)**: the console reads no log message as Rich markup, so the `[runtime.log.rich_log]` key that switched it is gone, along with `VERBATIM_MARK` in `pipelex.tools.log.log_fields`. A message now prints exactly as written, so a `list[int]`, a bracketed path, the `[myapp.jobs.nightly]: ` prefix of a logger named in lowercase and the `[cycle]` marker of a circular content no longer vanish, while a message of your own that relied on markup for its colour shows its tags and should put its values in `fields`. **Migration:** run `pipelex migrate`, which deletes the key from the file on disk (ledger entry `pipelex-config@7`).
+
## [v0.80.0] - 2026-10-09
### Changed
diff --git a/docs/configuration/config-practical/logging-config.md b/docs/configuration/config-practical/logging-config.md
index 3b83fadcd..db76a69a8 100644
--- a/docs/configuration/config-practical/logging-config.md
+++ b/docs/configuration/config-practical/logging-config.md
@@ -168,11 +168,11 @@ is_link_path_enabled = true
```toml
highlighter_name = "json" # or "repr"
-is_markup_enabled = true
```
- `highlighter_name`: Choose between JSON or repr highlighting
-- `is_markup_enabled`: Enable Rich markup syntax in log messages
+
+The console reads no log message as Rich markup: a message prints exactly as written, and colour comes from the fields and the layouts (see [Messages are plain text](../../tools/logging.md#messages-are-plain-text)). There is no setting to turn markup on. A file that still sets `is_markup_enabled` does not stop the boot: Pipelex deletes the key in memory with ledger entry `pipelex-config@7`, writes nothing, and warns that the configuration is out of date. Run `pipelex migrate` to delete it from the file on disk, which also ends the warning.
### Traceback Settings
@@ -302,7 +302,6 @@ is_show_time = false
is_show_level = true
is_link_path_enabled = true
highlighter_name = "json"
-is_markup_enabled = true
is_rich_tracebacks = true
is_tracebacks_word_wrap = true
is_tracebacks_show_locals = false
diff --git a/docs/tools/logging.md b/docs/tools/logging.md
index cc46c954c..ff364a6b2 100644
--- a/docs/tools/logging.md
+++ b/docs/tools/logging.md
@@ -41,7 +41,7 @@ from pipelex.tools.log.console_layouts import LogLayout
log.info(
"Pipe run starts",
- fields={"pipe_type": "PipeCompose", "pipe_code": "compose_company", "output_concept": "Company", "pipe_depth": 1, "is_dry_run": False},
+ fields={"pipe_type": "PipeCompose", "pipe_code": "compose_company", "output_concept": "Company", "pipe_depth": 1},
layout=LogLayout.PIPE_RUN,
)
@@ -85,7 +85,7 @@ The stdlib refuses an `extra` key that would overwrite one of the record's own a
Two kinds of name are reserved though nothing on a fresh record owns them yet, because both are stamped after the entries are attached and the stdlib's refusal therefore cannot cover them: what the formatter sets (`message`, `asctime`) and what Pipelex's own logging machinery sets — `_pipelex_forwarded`, the marker that tells the sink's handler a record reached it through the boot's holding handler already, `_pipelex_field_names`, the names the call's entries landed on, which is what the console renders, and `_pipelex_layout`, the [layout](#layouts) a call named. Both kinds take the same `field_` prefix, and no mark is ever handed to a sink as something the record carries. Without that reservation, `fields={"_pipelex_forwarded": True}` would have the sink's own filter read the record as one already delivered and drop it whole, and `fields={"_pipelex_layout": "pipe_run"}` would pick the line's layout.
-The names Rich's console handler reads off each record ahead of its own settings, `markup` and `highlighter`, are reserved the same way and for the same reason: a field landing on `markup` would decide whether the console reads that line's markup, and one landing on `highlighter` would be called in the highlighter's place, so `fields={"highlighter": "pygments"}` would raise inside the handler and lose the whole line. Such a field rides under `field_markup` or `field_highlighter`, and neither name is handed to a sink as something the record carries.
+The names Rich's console handler reads off each record ahead of its own settings, `markup` and `highlighter`, are reserved the same way and for the same reason: a field landing on `markup` would turn markup back on for that line, which the console otherwise never reads, and one landing on `highlighter` would be called in the highlighter's place, so `fields={"highlighter": "pygments"}` would raise inside the handler and lose the whole line. Such a field rides under `field_markup` or `field_highlighter`, and neither name is handed to a sink as something the record carries.
## The run-scoped context
@@ -210,7 +210,23 @@ Logging is configured in two steps at boot. `log.configure` sets the levels and
## Console rendering
-The default sink, `console`, renders through Rich with every `[runtime.log.rich_log]` setting. Rich is the `cli` extra (`pipelex[cli]`), which the command-line tools install. The sink imports Rich when it is built, so a process that selects `json` or `otlp` never loads it through the sink, and one that selects `console` without Rich installed stops at boot naming the extra to install and the `json` alternative. That refusal is not what keeps a server off the console: `typer` and `instructor`, both core dependencies, require Rich, so it is installed even where the extra is not, and a server that forgot to select `json` boots on the `console` sink. A server therefore selects `json` in its own configuration, and sets `pretty_print_mode` to `"poor"` or `"silent"` beside it, since the boot makes neither choice for it (see [Pretty-Print Mode](../configuration/config-practical/logging-config.md#pretty-print-mode)).
+The default sink, `console`, renders through Rich with every `[runtime.log.rich_log]` setting, and reads no message as markup (see [Messages are plain text](#messages-are-plain-text)). Rich is the `cli` extra (`pipelex[cli]`), which the command-line tools install. The sink imports Rich when it is built, so a process that selects `json` or `otlp` never loads it through the sink, and one that selects `console` without Rich installed stops at boot naming the extra to install and the `json` alternative. That refusal is not what keeps a server off the console: `typer` and `instructor`, both core dependencies, require Rich, so it is installed even where the extra is not, and a server that forgot to select `json` boots on the `console` sink. A server therefore selects `json` in its own configuration, and sets `pretty_print_mode` to `"poor"` or `"silent"` beside it, since the boot makes neither choice for it (see [Pretty-Print Mode](../configuration/config-practical/logging-config.md#pretty-print-mode)).
+
+### Messages are plain text
+
+The console reads no log message as Rich markup: a message is the caller's text, made of values nobody chose, and it prints exactly as written.
+
+```python
+log.error("Expected list[int], got [red]str[/red]")
+```
+
+```text
+ERROR 🧠: Expected list[int], got [red]str[/red]
+```
+
+A tag-shaped span in a message is text like any other: a `list[int]` in a type complaint, a bracketed path, the `[cycle]` marker in the rendering of a circular content, and the `[name]: ` prefix the console puts before a line from a logger with no emoji, such as `[myapp.jobs.nightly]: `. Colour is the console's own: it styles the [fields after the message](#fields-after-the-message) by their names and draws the few [layouts](#layouts), so a call is coloured by naming its fields, never by writing markup into its message. No sink but the console has ever read markup, so a message carrying a tag would reach the `json`, `otlp` and `gcp` sinks with the tag in it, and the rule for a Pipelex log call is to write none. A test checks the messages of one run against it: it runs the pipes of one test bundle live, a sequence nesting a structuring step, parallel summaries and a condition, with a stand-in worker answering every model call, and fails on any message the run logs, at any level, that holds a tag Rich would apply as styling: a closing tag, an `@` handler, or a tag naming a style, `[bold]` or `[link=https://pipelex.com]`. A bracketed word that names no style, `list[int]` or `[openai]`, passes, since it prints as written. A message logged only on a path that run does not take is not covered by it.
+
+Rich still honours its own per-record `markup` attribute, which a third-party library may set on a record it logs to ask for markup. No Pipelex call can set it: a field of that name is [reserved](#names-that-are-not-yours-to-give) and lands under `field_markup`.
### Fields after the message
@@ -226,7 +242,7 @@ INFO 🧠: Scanned the inputs files=7 source=inbox/march
- **What is shown.** The fields the call attached, in the order it gave them, under the name each landed on (`field_name` for a field called `name`). The run identifiers (`request_id`, `pipeline_run_id`, `pipe_run_id`) are left out, since they are the same on every line of a run and would drown the message, and so is `data`, the structured content the message already renders. Nothing else on the record is shown: neither the stdlib's own attributes, nor Pipelex's marks, nor what a record factory or a third-party library stamped on it.
- **How a value is written.** On one line: a string bare, or quoted when it is empty or holds a space, an `=`, a `"`, a `\` or a character a terminal would act on. Inside the quotes a backslash is written `\\`, a quote `\"` and any character a terminal would act on as its escape (`"line\nbreak"`), so `fields={"a": "b=c"}` prints `a="b=c"` rather than a second pair. A number, a boolean, `None`, a mapping, a list or a pydantic model is written as compact JSON (`true`, `null`, `{"key":"value"}`, `["a","b"]`), and anything else as its text. A value longer than 80 characters is cut short and ends with `…`. A key is written the same way, so a field whose name holds a line break, a space, an `=` or an escape sequence is quoted and escaped rather than forging a line or a second pair: `fields={"x=1": 2}` prints `"x=1"=2`. The [redaction](#redaction) has run before the console renders, so a secret is already `[REDACTED]`.
-- **Never markup.** The suffix is built as styled Rich `Text`, not as a markup string, so a value carrying `[red]x[/red]` prints exactly that, whatever `is_markup_enabled` says about the message.
+- **Never markup.** The suffix is built as styled Rich `Text`, not as a markup string, so a value carrying `[red]x[/red]` prints exactly that.
There is no setting to hide the suffix: it carries the values a message used to interpolate, so hiding it would hide what the line is about.
@@ -236,14 +252,14 @@ A field's value is coloured by the field's name, wherever it appears, from one t
### Layouts
-A few lines have a shape that matters on a terminal, the pipe-run tree above all. For those, a call names a layout, a Rich template over the record's fields registered in `pipelex/tools/log/console_layouts.py`:
+A few lines have a shape that matters on a terminal, the pipe-run tree above all. For those, a call names a layout, a Rich template over the record's fields registered in `pipelex/tools/log/console_layouts.py`. Every live pipe run announces itself this way, with the message `Pipe run starts` and the pipe in the fields, and the console draws the line byte for byte as it did when the announcement was markup in its message:
```python
from pipelex.tools.log.console_layouts import LogLayout
log.info(
"Pipe run starts",
- fields={"pipe_type": "PipeCompose", "pipe_code": "compose_company", "output_concept": "Company", "pipe_depth": 1, "is_dry_run": False},
+ fields={"pipe_type": "PipeCompose", "pipe_code": "compose_company", "output_concept": "Company", "pipe_depth": 1},
layout=LogLayout.PIPE_RUN,
)
```
@@ -263,7 +279,7 @@ The registry holds these layouts:
| Layout | Fields it presents | What it draws |
| --- | --- | --- |
-| `LogLayout.PIPE_RUN` | `pipe_type`, `pipe_code`, `output_concept`, `pipe_depth` (an integer, `0` at the top level and at most `100`), `is_dry_run` (a boolean) | `PipeCompose: compose_company → Company`, indented three spaces per level of depth and behind `↳` when nested, with `Dry run:` before the pipe type for a dry run, in the style map's colours. A depth or a flag of another type, or a depth out of range, falls back to the message |
+| `LogLayout.PIPE_RUN` | `pipe_type`, `pipe_code`, `output_concept`, `pipe_depth` (an integer, `0` at the top level and at most `100`) | `PipeCompose: compose_company → Company`, indented three spaces per level of depth and behind `↳` when nested, in the style map's colours. A depth of another type, or out of range, falls back to the message. The pipe announcement names it, and only a live run announces itself, so the line carries no run mode |
A layout is a `ConsoleLayout`: a `template` of Rich markup whose placeholders are bare field names (`{pipe_code}`, never an attribute, an index or a format spec, which registration refuses), the `presented_fields` the suffix leaves out, and, for a shape a template cannot express alone, a subclass whose `derived_values` computes presentation values such as an indentation from the fields.
@@ -309,6 +325,7 @@ Optional inclusion of caller information in logs, prefixed to the console line:
- Put a value a reader might filter or aggregate on in `fields`, and keep the message a sentence; the console shows it after the message, coloured by its name
- Name a field after the thing it carries, in `snake_case`, or after its OpenTelemetry key when one exists
- Leave the run identifiers to the context; bind them at the process entry, never per call
+ - Never write Rich markup in a message: the console prints it as written and every other sink carries the tags; colour comes from the fields and the layouts
3. **Structured Data**:
diff --git a/docs/under-the-hood/log-sink-plugins.md b/docs/under-the-hood/log-sink-plugins.md
index d6a48d69b..1c40a4a04 100644
--- a/docs/under-the-hood/log-sink-plugins.md
+++ b/docs/under-the-hood/log-sink-plugins.md
@@ -130,7 +130,7 @@ class LogSinkPlugin:
| Sink | What it does | Where it reads its settings |
|------|--------------|-----------------------------|
| `json` | One JSON object per line on the configured stream: `time`, `severity`, `logger`, `message`, `exception` when there is one, `trace_id`, `span_id` and `trace_flags` in hex when OpenTelemetry's current span names a trace, under the keys OpenTelemetry specifies for trace context in a JSON log that is not OTLP, `pipelex.trace_id` and `pipelex.span_id` in hex when a Pipelex span is active, then the fields, the context identifiers and `data` flat beside them, a field named like one of those keys under a `field_` prefix on every line, a non-finite float as the string `"NaN"`, `"Infinity"` or `"-Infinity"`. The key names are the ones the CloudWatch agent, the Google Cloud Logging agent and any OTLP collector ingest without a parser, and no ANSI ever. What the hosted plane's runner and worker select. | `console_log_target` |
-| `console` | The Rich handler with the emoji formatter and every `[runtime.log.rich_log]` setting, which shows a record's fields after its message, coloured by name, and renders the layouts a call names (see [Console rendering](../tools/logging.md#console-rendering)). Rich, the `cli` extra, is imported when the handler is built, and a process that selects another sink never loads it through this path. | `console_log_target`, `[runtime.log.rich_log]` |
+| `console` | The Rich handler with the emoji formatter and every `[runtime.log.rich_log]` setting, which reads no message as markup, shows a record's fields after its message, coloured by name, and renders the layouts a call names (see [Console rendering](../tools/logging.md#console-rendering)). Rich, the `cli` extra, is imported when the handler is built, and a process that selects another sink never loads it through this path. | `console_log_target`, `[runtime.log.rich_log]` |
| `otlp` | The OpenTelemetry logs signal: a `LoggerProvider` carrying the same service identity as the tracer, a `BatchLogRecordProcessor` and the OTLP HTTP log exporter. The message is the body, the level maps onto the OTel severity scale, the fields, identifiers and `data` ride as attributes (a mapping as JSON text), an exception lands under the `exception.*` semantic-convention keys, each record is filed under OpenTelemetry's current span, with the active Pipelex span as the `pipelex.trace_id` and `pipelex.span_id` attributes, and a collector receives the logs beside the spans the runtime already exports. The keys the sink writes itself, the `code.*` source location, the two `pipelex.*` keys and the `exception.*` set, are reserved on every record whether or not it carries an exception, so a field named like one of them rides under a `field_` prefix rather than being overwritten. A filter on the handler rejects the records of the sink's own export path, the SDK's by logger name and the transport's by the context value the SDK sets around an export, before the handler's lock is taken, so an export failure never re-enters the pipeline and a shutdown never waits on itself. | `[runtime.log.otlp]` |
| `gcp` | Google Cloud Logging through the `google-cloud-logging` client library, behind the `gcp-logging` extra. Each record becomes one struct entry: the level maps onto the Cloud Logging severity scale (which has nothing below `DEBUG`, so `VERBOSE` lands there), the message, the logger, the exception and the fields become the JSON payload, the run-scoped identifiers become the entry's labels, which is what Cloud Logging indexes, the entry's `trace`, `spanId` and `traceSampled` name OpenTelemetry's current span, the trace project-qualified, and the active Pipelex span rides in the payload as `pipelex.trace_id` and `pipelex.span_id`. The entries leave through the client library's background-thread transport, which batches them off the calling thread. The factory refreshes the credentials once before that transport starts, so credentials Google refuses stop the boot with `GcpLogSinkCredentialsError` rather than losing every record in silence. **Most processes on Google Cloud want `json` instead**: a platform whose logging agent reads the container's stdout, Cloud Run and GKE among them, ingests what that sink writes and needs no client at all; `gcp` is for a process with no such agent in front of it, or one writing to a log or a project that is not the ambient one. | `[runtime.log.gcp]` |
diff --git a/pipelex/cli/commands/doctor_cmd.py b/pipelex/cli/commands/doctor_cmd.py
index 84439f945..ba2b1f398 100644
--- a/pipelex/cli/commands/doctor_cmd.py
+++ b/pipelex/cli/commands/doctor_cmd.py
@@ -994,7 +994,8 @@ def display_health_report(
backend_credential_reports=bad_backend_credential_reports,
suggest_hosted_runs=True,
)
- console.print(error_msg)
+ # Plain text naming each backend's table as `[openai]`, which Rich would read as a style tag and drop
+ console.print(escape(error_msg))
console.print()
# Models section
diff --git a/pipelex/cli/commands/graph_cmd.py b/pipelex/cli/commands/graph_cmd.py
index d54737af7..69402efeb 100644
--- a/pipelex/cli/commands/graph_cmd.py
+++ b/pipelex/cli/commands/graph_cmd.py
@@ -193,12 +193,16 @@ def graph_render_cmd(
except GraphSpecValidationError as spec_error:
# A refused spec is a diagnosis, not a crash, so it gets its message and no traceback. The
# traceback below would print the pydantic error it was raised from, and that error quotes the
- # input it refused — which on a graph file is the run's own traced content. The message names the
- # user's path, which is escaped because both sinks read Rich markup and a path may hold brackets.
- escaped_message = escape(str(spec_error))
- log.error(escaped_message)
+ # input it refused — which on a graph file is the run's own traced content. For the same reason
+ # the log line carries the error as fields and not the exception itself, whose chain every sink
+ # would write. No sink reads a log message as markup, so the fields hold the text as written; the
+ # console print below does read markup, and a path may hold brackets, so it prints the text escaped.
+ log.error(
+ "The graph spec was refused, so no graph was rendered",
+ fields={"file.path": str(input_file), "error.type": type(spec_error).__name__, "error.message": str(spec_error)},
+ )
console = get_console()
- console.print(f"\n[bold red]Failed to render graph[/bold red]\n\n{escaped_message}\n")
+ console.print(f"\n[bold red]Failed to render graph[/bold red]\n\n{escape(str(spec_error))}\n")
raise typer.Exit(1) from spec_error
except Exception as exc:
diff --git a/pipelex/cli/commands/run/_run_core.py b/pipelex/cli/commands/run/_run_core.py
index 90832f803..7e5b2fd0a 100644
--- a/pipelex/cli/commands/run/_run_core.py
+++ b/pipelex/cli/commands/run/_run_core.py
@@ -560,7 +560,8 @@ def execute_run(
console = get_console()
print_traceback_if_requested(console=console)
console.print("\n[bold red]Failed to execute pipeline[/bold red]\n")
- console.print(f" {exc.message}\n")
+ # The message is plain text, a model's description such as `SDK[openai]` or a type such as `list[int]` included
+ console.print(f" {escape(exc.message)}\n")
raise typer.Exit(1) from exc
except Exception as exc:
diff --git a/pipelex/cli/dev_cli/commands/generate_projection_corpus_cmd.py b/pipelex/cli/dev_cli/commands/generate_projection_corpus_cmd.py
index 644b5b51d..f6567a233 100644
--- a/pipelex/cli/dev_cli/commands/generate_projection_corpus_cmd.py
+++ b/pipelex/cli/dev_cli/commands/generate_projection_corpus_cmd.py
@@ -37,6 +37,7 @@ from pathlib import Path
from typing import TYPE_CHECKING, Any, cast
from pydantic import BaseModel, ConfigDict, Field
+from rich.markup import escape
from pipelex.base_exceptions import PipelexError
from pipelex.cli.cli_factory import make_pipelex_for_cli
@@ -509,9 +510,7 @@ class ShapingGate:
failures = self.failures
undeclared = sorted(key for key in failures if key not in self.registry)
if undeclared:
- # Parenthesised, not bracketed: this message is printed through the Rich console, which
- # reads a `[compact]` as markup and eats it — leaving two shapes of one pipe as two
- # identical lines, each missing the half of the address that says which one failed.
+ # Parenthesised, so a refusal reads the same as the `unshapeable` lines a passing run prints.
listed = "\n ".join(
f"{pipe_ref} ({shape}): {type(failures[pipe_ref, shape]).__name__}: {str(failures[pipe_ref, shape]).splitlines()[0]}"
for pipe_ref, shape in undeclared
@@ -681,20 +680,20 @@ def generate_projection_corpus_cmd(*, bundle_paths: list[Path], output_dir: Path
console = get_console()
for bundle_path in bundle_paths:
if not bundle_path.is_file():
- console.print(f"[red]Bundle file not found: {bundle_path}[/red]")
+ console.print(f"[red]Bundle file not found: {escape(str(bundle_path))}[/red]")
sys.exit(2)
make_pipelex_for_cli(context=ErrorContext.VALIDATION, needs_inference=False)
try:
manifest = asyncio.run(generate_projection_corpus(bundle_paths=bundle_paths, output_dir=output_dir))
except ValidateBundleError as exc:
- console.print(f"[red]Bundle validation failed — the corpus requires valid bundles:[/red]\n{exc}")
+ console.print(f"[red]Bundle validation failed — the corpus requires valid bundles:[/red]\n{escape(str(exc))}")
sys.exit(1)
except ValueError as exc:
- console.print(f"[red]The corpus's own record is out of date:[/red]\n{exc}")
+ console.print(f"[red]The corpus's own record is out of date:[/red]\n{escape(str(exc))}")
sys.exit(1)
except PipelexError as exc:
- console.print(f"[red]Corpus generation failed:[/red]\n{exc}")
+ console.print(f"[red]Corpus generation failed:[/red]\n{escape(str(exc))}")
sys.exit(1)
finally:
Pipelex.teardown_if_needed()
diff --git a/pipelex/cli/dev_cli/commands/trace_input_semantics_cmd.py b/pipelex/cli/dev_cli/commands/trace_input_semantics_cmd.py
index c2eb7494a..a85694850 100644
--- a/pipelex/cli/dev_cli/commands/trace_input_semantics_cmd.py
+++ b/pipelex/cli/dev_cli/commands/trace_input_semantics_cmd.py
@@ -29,6 +29,8 @@ import sys
from pathlib import Path
from typing import Any
+from rich.markup import escape
+
from pipelex.base_exceptions import PipelexError
from pipelex.cli.cli_factory import make_pipelex_for_cli
from pipelex.cli.error_handlers import ErrorContext
@@ -349,7 +351,7 @@ def trace_input_semantics_cmd(*, bundle_paths: list[Path], output_dir: Path, all
console = get_console()
for bundle_path in bundle_paths:
if not bundle_path.is_file():
- console.print(f"[red]Bundle file not found: {bundle_path}[/red]")
+ console.print(f"[red]Bundle file not found: {escape(str(bundle_path))}[/red]")
sys.exit(2)
make_pipelex_for_cli(context=ErrorContext.VALIDATION, needs_inference=False)
@@ -362,10 +364,10 @@ def trace_input_semantics_cmd(*, bundle_paths: list[Path], output_dir: Path, all
)
)
except ValidateBundleError as exc:
- console.print(f"[red]Bundle validation failed — the trace requires a valid bundle:[/red]\n{exc}")
+ console.print(f"[red]Bundle validation failed — the trace requires a valid bundle:[/red]\n{escape(str(exc))}")
sys.exit(1)
except PipelexError as exc:
- console.print(f"[red]Trace failed:[/red]\n{exc}")
+ console.print(f"[red]Trace failed:[/red]\n{escape(str(exc))}")
sys.exit(1)
finally:
Pipelex.teardown_if_needed()
diff --git a/pipelex/cogt/llm/llm_worker_abstract.py b/pipelex/cogt/llm/llm_worker_abstract.py
index e262e794b..4c01b1afa 100644
--- a/pipelex/cogt/llm/llm_worker_abstract.py
+++ b/pipelex/cogt/llm/llm_worker_abstract.py
@@ -402,8 +402,17 @@ class LLMWorkerAbstract(InferenceWorkerAbstract, ABC):
requested_temperature = self._scaled_temperature(inference_model=self.inference_model, temperature=llm_job.job_params.temperature)
if fixed_temperature is not None and requested_temperature != fixed_temperature:
log.warning(
- f"Model {self.inference_model.desc} used with temperature {requested_temperature}, "
- f"but it must be {fixed_temperature} for this model so we forced it to {fixed_temperature}"
+ "The model takes a fixed temperature, so it is used in place of the requested one",
+ fields={
+ "model_handle": self.inference_model.name,
+ "backend_name": self.inference_model.backend_name,
+ "sdk": self.inference_model.sdk,
+ # The model keys mean what they mean on the LLM span: the handle requested, then the provider's id serving it
+ GenAISpanAttr.REQUEST_MODEL: self._get_request_model_name(),
+ GenAISpanAttr.RESPONSE_MODEL: self._get_response_model_name(),
+ GenAISpanAttr.REQUEST_TEMPERATURE: requested_temperature,
+ "fixed_temperature": fixed_temperature,
+ },
)
return self.constrained_job_params(inference_model=self.inference_model, job_params=llm_job.job_params)
diff --git a/pipelex/cogt/model_backends/backend_credentials.py b/pipelex/cogt/model_backends/backend_credentials.py
index 4cc4a4385..ee7dfa7ec 100644
--- a/pipelex/cogt/model_backends/backend_credentials.py
+++ b/pipelex/cogt/model_backends/backend_credentials.py
@@ -14,8 +14,7 @@ class BackendCredentialsReport(ConfigModel):
#: The way out a command-line user has besides provider keys: a run on the hosted Pipelex API needs none. Only a caller that
-#: knows it serves the `pipelex` or `pipelex-agent` CLI adds it, since a server operator cannot act on it. No square
-#: brackets: the message is printed through Rich markup and as plain text alike.
+#: knows it serves the `pipelex` or `pipelex-agent` CLI adds it, since a server operator cannot act on it.
HOSTED_RUNS_HINT = (
"\n☁ Or run on the hosted Pipelex API, which needs no provider key:\n"
" Run 'pipelex login' to save a Pipelex API key, then add --hosted to a run, or make hosted runs\n"
@@ -24,6 +23,13 @@ HOSTED_RUNS_HINT = (
class BackendCredentialsErrorMsgFactory:
+ """The messages that say which credentials a backend lacks and how to provide them or disable it.
+
+ A message is plain text, an exception's message or a report's, and names a backend's table in
+ `backends.toml` as it is written there, `[openai]`. It holds no Rich markup and no markup escape: a
+ surface that prints it through Rich markup escapes it where it prints it.
+ """
+
@classmethod
def make_one_variable_missing_error_msg(
cls,
@@ -51,7 +57,7 @@ class BackendCredentialsErrorMsgFactory:
f" Add the variable to your environment or .env file:\n"
f" - '{var_name}'=<your_api_key>\n"
"\n2. Disable this backend\n"
- rf" Add 'enabled = false' under '\[{backend_name}]' in '.pipelex/inference/backends.toml'" + "\n"
+ f" Add 'enabled = false' under '[{backend_name}]' in '.pipelex/inference/backends.toml'\n"
)
else:
error_msg = (
@@ -61,7 +67,7 @@ class BackendCredentialsErrorMsgFactory:
"1. Provide the missing secret\n"
f" Make sure '{var_name}' is available from your secrets provider.\n"
"\n2. Disable this backend\n"
- rf" Add 'enabled = false' under '\[{backend_name}]' in '.pipelex/inference/backends.toml'" + "\n"
+ f" Add 'enabled = false' under '[{backend_name}]' in '.pipelex/inference/backends.toml'\n"
)
# Point at the BYOK (Bring Your Own Keys) setup
@@ -131,7 +137,7 @@ class BackendCredentialsErrorMsgFactory:
error_msg += "\n2. Disable unused backends\n Disable backends you don't need in '.pipelex/inference/backends.toml':\n"
for backend_name in backend_names:
- error_msg += rf" - Add 'enabled = false' under '\[{backend_name}]'" + "\n"
+ error_msg += f" - Add 'enabled = false' under '[{backend_name}]'\n"
else:
error_msg = (
f"Could not get credentials for inference backend(s): {backends_list}\n\n"
@@ -146,7 +152,7 @@ class BackendCredentialsErrorMsgFactory:
error_msg += "\n2. Disable unused backends\n Disable backends you don't need in '.pipelex/inference/backends.toml':\n"
for backend_name in backend_names:
- error_msg += rf" - Add 'enabled = false' under '\[{backend_name}]'" + "\n"
+ error_msg += f" - Add 'enabled = false' under '[{backend_name}]'\n"
# Point at the BYOK (Bring Your Own Keys) setup
error_msg += (
diff --git a/pipelex/cogt/model_backends/model_spec.py b/pipelex/cogt/model_backends/model_spec.py
index 04b6fa531..230e4868e 100644
--- a/pipelex/cogt/model_backends/model_spec.py
+++ b/pipelex/cogt/model_backends/model_spec.py
@@ -78,11 +78,13 @@ class InferenceModelSpec(ConfigModel):
@property
def tag(self) -> str:
- return rf"{self.name} → \[{self.sdk}@{self.backend_name}]({self.model_id})"
+ """The model, its SDK, its backend and the provider's model id, as plain text for a message: never Rich markup."""
+ return f"{self.name} → [{self.sdk}@{self.backend_name}]({self.model_id})"
@property
def desc(self) -> str:
- return rf"{self.name} → SDK\[{self.sdk}]•Backend\[{self.backend_name}]•Model\[{self.model_id}]"
+ """The model, its SDK, its backend and the provider's model id, labelled, as plain text for a message: never Rich markup."""
+ return f"{self.name} → SDK[{self.sdk}]•Backend[{self.backend_name}]•Model[{self.model_id}]"
@property
def is_gen_object_supported(self) -> bool:
diff --git a/pipelex/kernel/llm_ops.py b/pipelex/kernel/llm_ops.py
index c106daf09..4002e83ea 100644
--- a/pipelex/kernel/llm_ops.py
+++ b/pipelex/kernel/llm_ops.py
@@ -39,6 +39,7 @@ from pipelex.kernel.memory_ops import store_result
from pipelex.runtime_hub import get_content_generator, get_inference_backend_registry, get_model_deck
from pipelex.system.exceptions import MissingDependencyError
from pipelex.system.job_metadata import JobMetadata
+from pipelex.system.telemetry.otel_constants import GenAISpanAttr
from pipelex.tools.jinja2.template_category import TemplateCategory
from pipelex.tools.templating.templating_style import TemplatingStyle
from pipelex.tools.typing.structure_printer import StructurePrinter
@@ -108,7 +109,17 @@ def check_llm_setting_with_served_model(*, llm_setting: LLMSetting, is_structure
check_request(inference_model=inference_model, job_params=job_params, is_structured=is_structured)
except MissingDependencyError:
# The backend's SDK is not installed here, so no worker for the model can be built: the run says so
- log.verbose(f"Model '{inference_model.desc}' was not checked: its backend's SDK is not installed")
+ log.verbose(
+ "The model's settings were not checked: its backend's SDK is not installed",
+ fields={
+ "model_handle": inference_model.name,
+ "backend_name": inference_model.backend_name,
+ "sdk": inference_model.sdk,
+ # The model keys of the LLM span and of the fixed-temperature warning, with the same meanings
+ GenAISpanAttr.REQUEST_MODEL: inference_model.name,
+ GenAISpanAttr.RESPONSE_MODEL: inference_model.model_id,
+ },
+ )
except LLMCapabilityError as refusal:
if registered_check and not registered_check.is_builtin:
# An external plugin's text may carry what it keeps private: it reaches the caller only if the plugin vouched for it
diff --git a/pipelex/migration/goldens/pipelex-config/[email protected] b/pipelex/migration/goldens/pipelex-config/[email protected]
new file mode 100644
index 000000000..af0f9f1f1
--- /dev/null
+++ b/pipelex/migration/goldens/pipelex-config/[email protected]
@@ -0,0 +1,496 @@
+####################################################################################################
+# Runtime — kernel-layer, process-scoped infrastructure (what runtime_hub brokers)
+####################################################################################################
+
+[runtime]
+
+[runtime.storage]
+is_fetch_remote_content_enabled = true
+is_upload_local_content_enabled = true
+method = "local" # local, in_memory, s3, gcp
+
+[runtime.storage.local]
+uri_format = "{hash}.{extension}"
+local_storage_path = ".pipelex/storage"
+
+[runtime.storage.in_memory]
+uri_format = "{hash}.{extension}"
+
+[runtime.storage.s3]
+uri_format = "{hash}.{extension}"
+bucket_name = ""
+region = ""
+signed_urls_lifespan_seconds = 3600
+
+[runtime.storage.gcp]
+uri_format = "{hash}.{extension}"
+bucket_name = ""
+project_id = ""
+signed_urls_lifespan_seconds = 3600
+
+[runtime.network]
+is_fetch_ssrf_guard_enabled = true
+
+[runtime.secrets]
+method = "env" # env (built-in); an external plugin may add e.g. vault
+
+####################################################################################################
+# Log config
+####################################################################################################
+
+[runtime.log]
+default_log_level = "INFO"
+# sink names the registered log sink boot installs on the root logger, the way runtime.storage.method
+# names a storage backend: "console" (the Rich handler, for a terminal), "json" (one JSON object per
+# line, for a server behind a log agent), "otlp" (the OpenTelemetry logs signal, for a collector),
+# "gcp" (Google Cloud Logging through the client library), or a token an installed plugin registers.
+# "console" is the default so the CLI keeps its rendering; a server selects "json". An unregistered
+# token stops the boot naming the registered ones.
+sink = "console"
+# console_log_target controls the stream the console and json sinks write to (log.debug/info/...).
+# Logs are diagnostics → stderr.
+console_log_target = "stderr"
+# console_print_target controls the Console returned by get_console(), used for banners,
+# deck notices, and the main `pipelex` CLI's data tables (`show backends`, `show models`,
+# `which`, `doctor`). Tables are data the user pipes to a file → stdout.
+# Note: the agent CLI (`pipelex-agent`) emits its actual JSON/markdown results via bare
+# builtin `print(...)` to sys.stdout, bypassing Rich entirely — so this knob does NOT
+# affect the agent CLI data channel. The agent CLI factory additionally forces both
+# targets to stderr internally to keep its stdout strictly reserved for the success
+# envelope (see `_AGENT_CLI_STDERR_CONSOLE_OVERRIDES` in `agent_cli_factory.py`).
+console_print_target = "stdout"
+
+json_logs_indent = 4
+presentation_line_width = 120
+silenced_problem_ids = ["azure_openai_no_stream_options"]
+caller_info_template = "file_line"
+is_caller_info_enabled = false
+
+# pretty_print_mode controls the panels `pretty_print(...)` renders, the "Output of pipe" panel after
+# every operator pipe among them: "rich" renders Rich tables, "poor" prints plain text, "silent" prints
+# nothing and builds no renderable at all. A host with no console, or one that runs pipes on a thread
+# that must not stall, sets "silent"; the agent CLI forces it.
+pretty_print_mode = "rich"
+
+[runtime.log.package_log_levels]
+# for logger names, the dots "." have been replaced by "-" to avoid toml parsing issues
+# TODO: use quotes to re-enable names with dots
+anthropic = "INFO"
+asyncio = "INFO"
+botocore = "INFO"
+botocore-credentials = "WARNING"
+google = "INFO"
+httpx = "WARNING"
+httpcore = "INFO"
+openai = "INFO"
+instructor = "INFO"
+urllib3-connectionpool = "INFO"
+urllib3-util-retry = "INFO"
+
+pipelex = "INFO"
+
+
+[runtime.log.rich_log]
+is_show_time = false
+is_show_level = true
+is_link_path_enabled = true
+highlighter_name = "json"
+is_rich_tracebacks = true
+is_tracebacks_word_wrap = true
+is_tracebacks_show_locals = false
+tracebacks_suppress = []
+keywords_to_hilight = []
+
+# The settings of the "otlp" sink. An endpoint key, when set, names the collector's logs URL
+# (for instance "http://localhost:4318/v1/logs"); left unset, the exporter follows the OpenTelemetry
+# environment conventions: OTEL_EXPORTER_OTLP_LOGS_ENDPOINT, then OTEL_EXPORTER_OTLP_ENDPOINT, then
+# the collector default on localhost. Empty headers likewise leave OTEL_EXPORTER_OTLP_HEADERS in charge.
+# A header value may name a secret rather than carry it, a collector's bearer token written
+# "Bearer ${OTLP_AUTH_TOKEN}" for one: it is resolved through the secrets provider [runtime.secrets]
+# selects, which boot builds before the sink, and a variable that does not resolve stops the boot.
+[runtime.log.otlp]
+headers = {}
+
+# The settings of the "gcp" sink, which writes to Google Cloud Logging through the client library.
+# Most processes want "json" instead: a platform that runs a logging agent over the container's
+# stdout, Cloud Run and GKE among them, ingests the JSON objects that sink writes and needs no client
+# at all. Select "gcp" for a process that must write to Cloud Logging directly. log_name is the
+# Cloud Logging log the entries land under. A project_id key names the project when it is not the
+# ambient one the credentials or the metadata server declare, and a credentials_file_path key names a
+# service-account JSON file when Application Default Credentials are not what the process should use.
+# The path may be a secret too, "${GCP_CREDENTIALS_FILE_PATH}" being the one the gcp storage provider
+# reads, resolved through the secrets provider the way the otlp headers are.
+[runtime.log.gcp]
+log_name = "pipelex"
+
+# [runtime.log.redaction] governs the one processor every sink is behind: it scrubs the secret shapes
+# the workspace already knows (an Authorization bearer token, an api-key or signature header, an OAuth
+# code parameter, a cookie, a JSON secret entry, and the sk_ / plx_sk_ / pk_ / bl_ key prefixes) out of
+# the message and out of every string a field carries, and neutralises the control characters in a
+# field value so a caller-supplied string cannot forge a line or a field separator. is_enabled = false
+# turns it off for a process that redacts downstream. extra_patterns are regular expressions added to
+# the shipped families for the secret shapes only this deployment knows; every match is replaced by
+# "[REDACTED]", and a pattern the re module refuses is a configuration error named at load.
+[runtime.log.redaction]
+is_enabled = true
+extra_patterns = []
+
+[runtime.aws]
+api_key_method = "env"
+bedrock_access_variant = "aws_access"
+
+[runtime.reporting]
+is_log_costs_to_console = true
+is_generate_cost_report_file_enabled = false
+cost_report_dir_path = "reports"
+cost_report_base_name = "cost_report"
+cost_report_extension = "csv"
+cost_report_unit_scale = 1.0
+
+####################################################################################################
+# Tracing config (event log for distributed tracing across workers)
+####################################################################################################
+
+[runtime.tracing]
+is_enabled = true
+backend = "ndjson"
+
+[runtime.tracing.ndjson]
+traces_dir = ".pipelex/traces"
+
+[runtime.tracing.dynamodb]
+table_name = ""
+region = ""
+
+[runtime.observer]
+observer_dir = "results/observer"
+
+[runtime.plugins]
+# Names of discovered plugins to skip at startup (a denylist). Discovery is the
+# source of truth for which plugins are present; this only turns one off.
+disabled = []
+
+####################################################################################################
+# Inference — the model-calling seam (was [cogt])
+####################################################################################################
+
+[inference]
+transport_max_retries = 2
+
+[inference.model_deck]
+is_model_fallback_enabled = true
+missing_presets_reaction = "log"
+
+[inference.llm]
+default_max_images = 100
+is_structure_prompt_enabled = true
+is_dump_text_prompts_enabled = false
+is_dump_response_text_enabled = false
+schema_reask_max_attempts = 3
+
+[inference.llm.instructor]
+is_dump_kwargs_enabled = false
+is_dump_response_enabled = false
+is_dump_error_enabled = false
+
+[inference.llm.openai.effort_to_level_map]
+none = "none"
+minimal = "minimal"
+low = "low"
+medium = "medium"
+high = "high"
+xhigh = "xhigh"
+max = "xhigh"
+
+[inference.llm.anthropic]
+structured_output_timeout_seconds = 1200
+
+[inference.llm.anthropic.effort_to_level_map]
+none = "disabled"
+minimal = "low"
+low = "low"
+medium = "medium"
+high = "high"
+xhigh = "xhigh"
+max = "max"
+
+[inference.llm.google.effort_to_level_map]
+none = "disabled"
+minimal = "minimal"
+low = "low"
+medium = "medium"
+high = "high"
+xhigh = "high"
+max = "high"
+
+[inference.llm.mistral.effort_to_level_map]
+none = "disabled"
+minimal = "reasoning"
+low = "reasoning"
+medium = "reasoning"
+high = "reasoning"
+xhigh = "reasoning"
+max = "reasoning"
+
+[inference.llm.effort_to_budget_maps.anthropic]
+none = 0 # Required by validator; unreachable at runtime (level map gates NONE as disabled before budget lookup)
+minimal = 512
+low = 1024
+medium = 5000
+high = 16384
+xhigh = 32768
+max = 65536
+
+[inference.llm.effort_to_budget_maps.gemini]
+none = 0 # Required by validator; unreachable at runtime (level map gates NONE as disabled before budget lookup)
+minimal = 512
+low = 1024
+medium = 5000
+high = 16384
+xhigh = 32768
+max = 65536
+
+[inference.llm.generic_templates]
+output_structure_prompt = """
+
+
+---
+The instance we want to generate will be for the following class:
+{{ class_structure_str }}
+
+DO NOT create information.
+If some information is not present for an attribute, output the default value or None according to the attribute definition.
+"""
+
+structuring_prompt = """
+Read the following text carefully and produce the requested structured output from it.
+
+---
+{{ text }}
+"""
+
+####################################################################################################
+# Image generation config
+####################################################################################################
+
+[inference.img_gen.img_gen_job]
+is_sync_mode = false
+
+[inference.img_gen.img_gen_param_defaults]
+nb_steps = 28 # 28 is a good default for Flux, possible values are [1,2,4,8] for SDXL Lightning
+quality = "low"
+aspect_ratio = "square" # "square", "landscape_4_3", "landscape_3_2", "landscape_16_9", "landscape_21_9", "landscape_4_1", "landscape_8_1", "portrait_3_4", "portrait_2_3", "portrait_9_16", "portrait_9_21", "portrait_1_4", "portrait_1_8"
+guidance_scale = 2.5
+background = "auto"
+is_moderated = true
+safety_tolerance = 5
+is_raw = false
+seed = "auto"
+
+[inference.img_gen.quality_to_steps_maps]
+flux = { "low" = 14, "medium" = 28, "high" = 56 }
+sdxl_lightning = { "low" = 2, "medium" = 4, "high" = 8 }
+qwen_image = { "low" = 10, "medium" = 20, "high" = 28 }
+
+####################################################################################################
+# Extract config
+####################################################################################################
+
+[inference.extract]
+default_page_views_dpi = 72
+
+####################################################################################################
+# Gateway test config
+####################################################################################################
+
+[inference.gateway_test.config_id_substitutions]
+# this is an empty dict, it's only used by Pipelex staff to override the ¨Pipelex Gateway¨ config
+# in order to test new model integrations or debug issues with the Gateway
+
+####################################################################################################
+# Templating config
+####################################################################################################
+
+[inference.templating]
+default_templating_style = { tag_style = "xml" }
+
+####################################################################################################
+# Dry run config
+####################################################################################################
+
+[inference.dry_run]
+text_gen_truncate_length = 256
+nb_list_items = 3
+nb_extract_pages = 4
+allowed_to_fail_pipes = [
+ "failing_pipelines.infinite_loop_1", # Loop but only for testing purposes — matched on the namespaced pipe_ref
+]
+image_urls = [
+ "https://storage.googleapis.com/public_test_files_7fa6_4277_9ab/fashion/fashion_photo_1.jpg",
+ "https://storage.googleapis.com/public_test_files_7fa6_4277_9ab/fashion/fashion_photo_2.png",
+]
+
+####################################################################################################
+# Interpreter — library-scoped method machinery (what interpreter_hub brokers)
+####################################################################################################
+
+[interpreter]
+
+####################################################################################################
+# MTHDS config
+####################################################################################################
+
+[interpreter.mthds.inline_tables]
+spaces_inside_curly_braces = true
+
+[interpreter.mthds.strings]
+prefer_literal = false
+force_multiline = false
+length_limit_to_multiline = 100
+ensure_trailing_newline = true
+ensure_leading_blank_line = true
+
+[interpreter.mthds.concepts]
+structure_field_ordering = [
+ "type",
+ "concept_ref",
+ "item_type",
+ "item_concept_ref",
+ "description",
+ "choices",
+ "required",
+]
+
+[interpreter.mthds.pipes]
+field_ordering = ["type", "description", "inputs", "output"]
+
+####################################################################################################
+# Methods config
+####################################################################################################
+
+[interpreter.methods]
+# When a bundle references another method by address (github.com/...->domain.pipe) and no installed
+# method matches, fetch the package by address and install it into ~/.mthds/methods/ so the load can
+# proceed. Set to false to forbid network fetches at load time — a miss then raises a diagnostic
+# telling you the address and how to install it manually. The PIPELEX_METHODS_FETCH_ON_MISS
+# environment variable (1/0, true/false) overrides this setting when set.
+fetch_on_miss = true
+
+####################################################################################################
+# Pipe run config
+####################################################################################################
+
+[interpreter.pipe_run]
+pipe_stack_limit = 20
+
+####################################################################################################
+# PipeFunc config
+####################################################################################################
+
+[interpreter.pipe_func]
+# PipeFunc execution mode, selected from the PipeFuncExecutorRegistry the plugins populate. "direct"
+# (core) imports and runs customer functions in this process; "daytona" (our Daytona sandbox plugin)
+# runs them in a remote box. Any non-"direct" mode carries customer PipeFunc code as source text and
+# executes it only inside an isolated sandbox, never imported here.
+execution_mode = "direct"
+
+####################################################################################################
+# Pipeline execution config
+####################################################################################################
+
+[interpreter.pipeline_execution]
+is_normalize_data_urls_to_storage = true
+is_mock_inputs = false
+is_generate_graph = true
+is_generate_usage = true
+
+# Bounded fan-out concurrency for PipeBatch: at most this many branches run at once.
+# A modest default keeps a large workload (one pipe over many items) from overwhelming
+# asyncio, memory, and provider rate limits. Set to "unbounded" to disable the bound.
+max_concurrency = 8
+
+[interpreter.pipeline_execution.graph]
+
+[interpreter.pipeline_execution.graph.data_inclusion]
+stuff_json_content = true
+error_stack_traces = true
+pipe_and_concept_registry = true
+
+[interpreter.pipeline_execution.graph.graphs_inclusion]
+graphspec_json = true
+mermaidflow_mmd = true
+mermaidflow_html = true
+reactflow_html = true
+
+[interpreter.pipeline_execution.graph.mermaid]
+direction = "left_to_right"
+is_include_data_edges = true
+is_include_contains_edges = false
+is_include_selected_outcome_edges = true
+is_show_stuff_codes = false
+
+[interpreter.pipeline_execution.graph.mermaid.style]
+theme = "dark"
+
+[interpreter.pipeline_execution.graph.reactflow]
+is_use_cdn = true
+layout_direction = "left_to_right"
+nodesep = 50
+ranksep = 30
+edge_type = "bezier"
+initial_zoom = 1.0
+pan_to_top = true
+default_title = "Pipelex Graph"
+# Forwarded to mthds-ui as `showControllers`: when true, controller pipes (PipeSequence,
+# PipeParallel, PipeCondition, PipeBatch) render as grouping containers around their children;
+# when false, they collapse out of the graph and edges connect stuff nodes directly.
+show_batch_controller = true
+
+[interpreter.pipeline_execution.graph.reactflow.style]
+theme = "dark"
+palette = "dracula"
+
+[interpreter.scan]
+excluded_dirs = [
+ ".venv",
+ "venv",
+ "env",
+ ".env",
+ "virtualenv",
+ ".virtualenv",
+ ".git",
+ "__pycache__",
+ ".pytest_cache",
+ ".mypy_cache",
+ ".ruff_cache",
+ "node_modules",
+ "results",
+]
+
+[interpreter.builder]
+fix_loop_max_attempts = 10
+default_output_dir = "."
+default_bundle_file_name = "bundle"
+default_directory_base_name = "pipeline"
+
+####################################################################################################
+# Run — where `pipelex run` and `pipelex-agent run` execute a method
+####################################################################################################
+
+[run]
+# Where a run executes when the command passes neither --hosted nor --local:
+# - "local" runs on this machine, on the backends configured in .pipelex/inference/ (your own provider keys)
+# - "hosted" runs on the hosted Pipelex API, with the key in PIPELEX_API_KEY (the environment or ~/.pipelex/.env),
+# at the origin in PIPELEX_BASE_URL (https://api.pipelex.com when unset)
+execution = "local"
+
+####################################################################################################
+# Kit Config
+####################################################################################################
+
+[kit]
+# Agent rule files to install (used by `pipelex-dev kit rules`)
+# Each entry is one of: "cursor", "agents", "claude"
+# - "cursor" installs .cursor/rules/*.mdc files (cannot be combined with others)
+# - "agents" / "claude" install AGENTS.md / CLAUDE.md respectively — both can be listed together
+preferred_agent_targets = ["claude", "agents"]
diff --git a/pipelex/migration/goldens/pipelex-config/[email protected] b/pipelex/migration/goldens/pipelex-config/[email protected]
new file mode 100644
index 000000000..df37e96a7
--- /dev/null
+++ b/pipelex/migration/goldens/pipelex-config/[email protected]
@@ -0,0 +1,1512 @@
+{
+ "surface_id": "pipelex-config",
+ "schema_version": 7,
+ "document_root_is_open": false,
+ "paths": {
+ "inference": {
+ "value_type": "table",
+ "required": true,
+ "open_node": false
+ },
+ "inference.dry_run": {
+ "value_type": "table",
+ "required": true,
+ "open_node": false
+ },
+ "inference.dry_run.allowed_to_fail_pipes": {
+ "value_type": "list[str]",
+ "required": false,
+ "default": [
+ "failing_pipelines.infinite_loop_1"
+ ],
+ "open_node": false
+ },
+ "inference.dry_run.image_urls": {
+ "value_type": "list[str]",
+ "required": true,
+ "default": [
+ "https://storage.googleapis.com/public_test_files_7fa6_4277_9ab/fashion/fashion_photo_1.jpg",
+ "https://storage.googleapis.com/public_test_files_7fa6_4277_9ab/fashion/fashion_photo_2.png"
+ ],
+ "open_node": false
+ },
+ "inference.dry_run.nb_extract_pages": {
+ "value_type": "int",
+ "required": true,
+ "default": 4,
+ "open_node": false
+ },
+ "inference.dry_run.nb_list_items": {
+ "value_type": "int",
+ "required": true,
+ "default": 3,
+ "open_node": false
+ },
+ "inference.dry_run.text_gen_truncate_length": {
+ "value_type": "int",
+ "required": true,
+ "default": 256,
+ "open_node": false
+ },
+ "inference.extract": {
+ "value_type": "table",
+ "required": true,
+ "open_node": false
+ },
+ "inference.extract.default_page_views_dpi": {
+ "value_type": "int",
+ "required": true,
+ "default": 72,
+ "open_node": false
+ },
+ "inference.gateway_test": {
+ "value_type": "table",
+ "required": true,
+ "open_node": false
+ },
+ "inference.gateway_test.config_id_substitutions": {
+ "value_type": "dict[str, str]",
+ "required": true,
+ "default": {},
+ "open_node": true
+ },
+ "inference.gateway_test.config_id_substitutions.*": {
+ "value_type": "str",
+ "required": true,
+ "open_node": false
+ },
+ "inference.img_gen": {
+ "value_type": "table",
+ "required": true,
+ "open_node": false
+ },
+ "inference.img_gen.img_gen_job": {
+ "value_type": "table",
+ "required": true,
+ "open_node": false
+ },
+ "inference.img_gen.img_gen_job.is_sync_mode": {
+ "value_type": "bool",
+ "required": true,
+ "default": false,
+ "open_node": false
+ },
+ "inference.img_gen.img_gen_param_defaults": {
+ "value_type": "table",
+ "required": true,
+ "open_node": false
+ },
+ "inference.img_gen.img_gen_param_defaults.aspect_ratio": {
+ "value_type": "enum",
+ "required": true,
+ "enum_members": [
+ "landscape_16_9",
+ "landscape_21_9",
+ "landscape_3_2",
+ "landscape_4_1",
+ "landscape_4_3",
+ "landscape_8_1",
+ "portrait_1_4",
+ "portrait_1_8",
+ "portrait_2_3",
+ "portrait_3_4",
+ "portrait_9_16",
+ "portrait_9_21",
+ "square"
+ ],
+ "default": "square",
+ "open_node": false
+ },
+ "inference.img_gen.img_gen_param_defaults.background": {
+ "value_type": "enum",
+ "required": true,
+ "enum_members": [
+ "auto",
+ "opaque",
+ "transparent"
+ ],
+ "default": "auto",
+ "open_node": false
+ },
+ "inference.img_gen.img_gen_param_defaults.guidance_scale": {
+ "value_type": "float",
+ "required": true,
+ "default": 2.5,
+ "open_node": false,
+ "constraints": {
+ "gt": 0
+ }
+ },
+ "inference.img_gen.img_gen_param_defaults.is_moderated": {
+ "value_type": "bool",
+ "required": false,
+ "default": true,
+ "open_node": false
+ },
+ "inference.img_gen.img_gen_param_defaults.is_raw": {
+ "value_type": "bool",
+ "required": false,
+ "default": false,
+ "open_node": false
+ },
+ "inference.img_gen.img_gen_param_defaults.nb_steps": {
+ "value_type": "int",
+ "required": false,
+ "default": 28,
+ "open_node": false,
+ "constraints": {
+ "gt": 0
+ }
+ },
+ "inference.img_gen.img_gen_param_defaults.quality": {
+ "value_type": "enum",
+ "required": false,
+ "enum_members": [
+ "high",
+ "low",
+ "medium"
+ ],
+ "default": "low",
+ "open_node": false
+ },
+ "inference.img_gen.img_gen_param_defaults.safety_tolerance": {
+ "value_type": "int",
+ "required": true,
+ "default": 5,
+ "open_node": false,
+ "constraints": {
+ "ge": 1,
+ "le": 6
+ }
+ },
+ "inference.img_gen.img_gen_param_defaults.seed": {
+ "value_type": "int | literal",
+ "required": true,
+ "enum_members": [
+ "auto"
+ ],
+ "default": "auto",
+ "open_node": false
+ },
+ "inference.img_gen.img_gen_param_defaults.size": {
+ "value_type": "enum | table",
+ "required": false,
+ "enum_members": [
+ "0.5k",
+ "1k",
+ "2k",
+ "4k"
+ ],
+ "open_node": false
+ },
+ "inference.img_gen.quality_to_steps_maps": {
+ "value_type": "dict[str, dict[str, int]]",
+ "required": true,
+ "default": {
+ "flux": {
+ "low": 14,
+ "medium": 28,
+ "high": 56
+ },
+ "sdxl_lightning": {
+ "low": 2,
+ "medium": 4,
+ "high": 8
+ },
+ "qwen_image": {
+ "low": 10,
+ "medium": 20,
+ "high": 28
+ }
+ },
+ "open_node": true
+ },
+ "inference.img_gen.quality_to_steps_maps.*": {
+ "value_type": "dict[str, int]",
+ "required": true,
+ "open_node": true
+ },
+ "inference.img_gen.quality_to_steps_maps.*.*": {
+ "value_type": "int",
+ "required": true,
+ "open_node": false
+ },
+ "inference.llm": {
+ "value_type": "table",
+ "required": true,
+ "open_node": false
+ },
+ "inference.llm.anthropic": {
+ "value_type": "table",
+ "required": true,
+ "open_node": false
+ },
+ "inference.llm.anthropic.effort_to_level_map": {
+ "value_type": "dict[str, str]",
+ "required": true,
+ "default": {
+ "none": "disabled",
+ "minimal": "low",
+ "low": "low",
+ "medium": "medium",
+ "high": "high",
+ "xhigh": "xhigh",
+ "max": "max"
+ },
+ "open_node": true
+ },
+ "inference.llm.anthropic.effort_to_level_map.*": {
+ "value_type": "str",
+ "required": true,
+ "open_node": false
+ },
+ "inference.llm.anthropic.structured_output_timeout_seconds": {
+ "value_type": "int",
+ "required": true,
+ "default": 1200,
+ "open_node": false
+ },
+ "inference.llm.default_max_images": {
+ "value_type": "int",
+ "required": true,
+ "default": 100,
+ "open_node": false
+ },
+ "inference.llm.effort_to_budget_maps": {
+ "value_type": "dict[str, dict[str, int]]",
+ "required": true,
+ "default": {
+ "anthropic": {
+ "none": 0,
+ "minimal": 512,
+ "low": 1024,
+ "medium": 5000,
+ "high": 16384,
+ "xhigh": 32768,
+ "max": 65536
+ },
+ "gemini": {
+ "none": 0,
+ "minimal": 512,
+ "low": 1024,
+ "medium": 5000,
+ "high": 16384,
+ "xhigh": 32768,
+ "max": 65536
+ }
+ },
+ "open_node": true
+ },
+ "inference.llm.effort_to_budget_maps.*": {
+ "value_type": "dict[str, int]",
+ "required": true,
+ "open_node": true
+ },
+ "inference.llm.effort_to_budget_maps.*.*": {
+ "value_type": "int",
+ "required": true,
+ "open_node": false
+ },
+ "inference.llm.generic_templates": {
+ "value_type": "dict[str, str]",
+ "required": true,
+ "default": {
+ "output_structure_prompt": "\n\n---\nThe instance we want to generate will be for the following class:\n{{ class_structure_str }}\n\nDO NOT create information.\nIf some information is not present for an attribute, output the default value or None according to the attribute definition.\n",
+ "structuring_prompt": "Read the following text carefully and produce the requested structured output from it.\n\n---\n{{ text }}\n"
+ },
+ "open_node": true
+ },
+ "inference.llm.generic_templates.*": {
+ "value_type": "str",
+ "required": true,
+ "open_node": false
+ },
+ "inference.llm.google": {
+ "value_type": "table",
+ "required": true,
+ "open_node": false
+ },
+ "inference.llm.google.effort_to_level_map": {
+ "value_type": "dict[str, str]",
+ "required": true,
+ "default": {
+ "none": "disabled",
+ "minimal": "minimal",
+ "low": "low",
+ "medium": "medium",
+ "high": "high",
+ "xhigh": "high",
+ "max": "high"
+ },
+ "open_node": true
+ },
+ "inference.llm.google.effort_to_level_map.*": {
+ "value_type": "str",
+ "required": true,
+ "open_node": false
+ },
+ "inference.llm.instructor": {
+ "value_type": "table",
+ "required": true,
+ "open_node": false
+ },
+ "inference.llm.instructor.is_dump_error_enabled": {
+ "value_type": "bool",
+ "required": true,
+ "default": false,
+ "open_node": false
+ },
+ "inference.llm.instructor.is_dump_kwargs_enabled": {
+ "value_type": "bool",
+ "required": true,
+ "default": false,
+ "open_node": false
+ },
+ "inference.llm.instructor.is_dump_response_enabled": {
+ "value_type": "bool",
+ "required": true,
+ "default": false,
+ "open_node": false
+ },
+ "inference.llm.is_dump_response_text_enabled": {
+ "value_type": "bool",
+ "required": true,
+ "default": false,
+ "open_node": false
+ },
+ "inference.llm.is_dump_text_prompts_enabled": {
+ "value_type": "bool",
+ "required": true,
+ "default": false,
+ "open_node": false
+ },
+ "inference.llm.is_structure_prompt_enabled": {
+ "value_type": "bool",
+ "required": true,
+ "default": true,
+ "open_node": false
+ },
+ "inference.llm.mistral": {
+ "value_type": "table",
+ "required": true,
+ "open_node": false
+ },
+ "inference.llm.mistral.effort_to_level_map": {
+ "value_type": "dict[str, str]",
+ "required": true,
+ "default": {
+ "none": "disabled",
+ "minimal": "reasoning",
+ "low": "reasoning",
+ "medium": "reasoning",
+ "high": "reasoning",
+ "xhigh": "reasoning",
+ "max": "reasoning"
+ },
+ "open_node": true
+ },
+ "inference.llm.mistral.effort_to_level_map.*": {
+ "value_type": "str",
+ "required": true,
+ "open_node": false
+ },
+ "inference.llm.openai": {
+ "value_type": "table",
+ "required": true,
+ "open_node": false
+ },
+ "inference.llm.openai.effort_to_level_map": {
+ "value_type": "dict[str, str]",
+ "required": true,
+ "default": {
+ "none": "none",
+ "minimal": "minimal",
+ "low": "low",
+ "medium": "medium",
+ "high": "high",
+ "xhigh": "xhigh",
+ "max": "xhigh"
+ },
+ "open_node": true
+ },
+ "inference.llm.openai.effort_to_level_map.*": {
+ "value_type": "str",
+ "required": true,
+ "open_node": false
+ },
+ "inference.llm.schema_reask_max_attempts": {
+ "value_type": "int",
+ "required": true,
+ "default": 3,
+ "open_node": false,
+ "constraints": {
+ "ge": 1,
+ "le": 10
+ }
+ },
+ "inference.model_deck": {
+ "value_type": "table",
+ "required": true,
+ "open_node": false
+ },
+ "inference.model_deck.is_model_fallback_enabled": {
+ "value_type": "bool",
+ "required": true,
+ "default": true,
+ "open_node": false
+ },
+ "inference.model_deck.missing_presets_reaction": {
+ "value_type": "enum",
+ "required": true,
+ "enum_members": [
+ "log",
+ "none",
+ "raise"
+ ],
+ "default": "log",
+ "open_node": false
+ },
+ "inference.templating": {
+ "value_type": "table",
+ "required": true,
+ "open_node": false
+ },
+ "inference.templating.default_templating_style": {
+ "value_type": "table",
+ "required": true,
+ "open_node": false
+ },
+ "inference.templating.default_templating_style.tag_style": {
+ "value_type": "enum",
+ "required": true,
+ "enum_members": [
+ "no_tag",
+ "square_brackets",
+ "ticks",
+ "xml"
+ ],
+ "default": "xml",
+ "open_node": false
+ },
+ "inference.templating.default_templating_style.text_format": {
+ "value_type": "enum",
+ "required": false,
+ "enum_members": [
+ "html",
+ "json",
+ "markdown",
+ "plain"
+ ],
+ "open_node": false
+ },
+ "inference.transport_max_retries": {
+ "value_type": "int",
+ "required": true,
+ "default": 2,
+ "open_node": false,
+ "constraints": {
+ "ge": 0,
+ "le": 10
+ }
+ },
+ "interpreter": {
+ "value_type": "table",
+ "required": true,
+ "open_node": false
+ },
+ "interpreter.builder": {
+ "value_type": "table",
+ "required": true,
+ "open_node": false
+ },
+ "interpreter.builder.default_bundle_file_name": {
+ "value_type": "str",
+ "required": true,
+ "default": "bundle",
+ "open_node": false
+ },
+ "interpreter.builder.default_directory_base_name": {
+ "value_type": "str",
+ "required": true,
+ "default": "pipeline",
+ "open_node": false
+ },
+ "interpreter.builder.default_output_dir": {
+ "value_type": "str",
+ "required": true,
+ "default": ".",
+ "open_node": false
+ },
+ "interpreter.builder.fix_loop_max_attempts": {
+ "value_type": "int",
+ "required": true,
+ "default": 10,
+ "open_node": false
+ },
+ "interpreter.methods": {
+ "value_type": "table",
+ "required": true,
+ "open_node": false
+ },
+ "interpreter.methods.fetch_on_miss": {
+ "value_type": "bool",
+ "required": true,
+ "default": true,
+ "open_node": false
+ },
+ "interpreter.mthds": {
+ "value_type": "table",
+ "required": true,
+ "open_node": false
+ },
+ "interpreter.mthds.concepts": {
+ "value_type": "table",
+ "required": true,
+ "open_node": false
+ },
+ "interpreter.mthds.concepts.structure_field_ordering": {
+ "value_type": "list[str]",
+ "required": true,
+ "default": [
+ "type",
+ "concept_ref",
+ "item_type",
+ "item_concept_ref",
+ "description",
+ "choices",
+ "required"
+ ],
+ "open_node": false
+ },
+ "interpreter.mthds.inline_tables": {
+ "value_type": "table",
+ "required": true,
+ "open_node": false
+ },
+ "interpreter.mthds.inline_tables.spaces_inside_curly_braces": {
+ "value_type": "bool",
+ "required": true,
+ "default": true,
+ "open_node": false
+ },
+ "interpreter.mthds.pipes": {
+ "value_type": "table",
+ "required": true,
+ "open_node": false
+ },
+ "interpreter.mthds.pipes.field_ordering": {
+ "value_type": "list[str]",
+ "required": true,
+ "default": [
+ "type",
+ "description",
+ "inputs",
+ "output"
+ ],
+ "open_node": false
+ },
+ "interpreter.mthds.strings": {
+ "value_type": "table",
+ "required": true,
+ "open_node": false
+ },
+ "interpreter.mthds.strings.ensure_leading_blank_line": {
+ "value_type": "bool",
+ "required": true,
+ "default": true,
+ "open_node": false
+ },
+ "interpreter.mthds.strings.ensure_trailing_newline": {
+ "value_type": "bool",
+ "required": true,
+ "default": true,
+ "open_node": false
+ },
+ "interpreter.mthds.strings.force_multiline": {
+ "value_type": "bool",
+ "required": true,
+ "default": false,
+ "open_node": false
+ },
+ "interpreter.mthds.strings.length_limit_to_multiline": {
+ "value_type": "int",
+ "required": true,
+ "default": 100,
+ "open_node": false
+ },
+ "interpreter.mthds.strings.prefer_literal": {
+ "value_type": "bool",
+ "required": true,
+ "default": false,
+ "open_node": false
+ },
+ "interpreter.pipe_func": {
+ "value_type": "table",
+ "required": true,
+ "open_node": false
+ },
+ "interpreter.pipe_func.execution_mode": {
+ "value_type": "str",
+ "required": true,
+ "default": "direct",
+ "open_node": false
+ },
+ "interpreter.pipe_run": {
+ "value_type": "table",
+ "required": true,
+ "open_node": false
+ },
+ "interpreter.pipe_run.pipe_stack_limit": {
+ "value_type": "int",
+ "required": true,
+ "default": 20,
+ "open_node": false
+ },
+ "interpreter.pipeline_execution": {
+ "value_type": "table",
+ "required": true,
+ "open_node": false
+ },
+ "interpreter.pipeline_execution.graph": {
+ "value_type": "table",
+ "required": true,
+ "open_node": false
+ },
+ "interpreter.pipeline_execution.graph.data_inclusion": {
+ "value_type": "table",
+ "required": true,
+ "open_node": false
+ },
+ "interpreter.pipeline_execution.graph.data_inclusion.error_stack_traces": {
+ "value_type": "bool",
+ "required": true,
+ "default": true,
+ "open_node": false
+ },
+ "interpreter.pipeline_execution.graph.data_inclusion.pipe_and_concept_registry": {
+ "value_type": "bool",
+ "required": true,
+ "default": true,
+ "open_node": false
+ },
+ "interpreter.pipeline_execution.graph.data_inclusion.stuff_json_content": {
+ "value_type": "bool",
+ "required": true,
+ "default": true,
+ "open_node": false
+ },
+ "interpreter.pipeline_execution.graph.graphs_inclusion": {
+ "value_type": "table",
+ "required": true,
+ "open_node": false
+ },
+ "interpreter.pipeline_execution.graph.graphs_inclusion.graphspec_json": {
+ "value_type": "bool",
+ "required": true,
+ "default": true,
+ "open_node": false
+ },
+ "interpreter.pipeline_execution.graph.graphs_inclusion.mermaidflow_html": {
+ "value_type": "bool",
+ "required": true,
+ "default": true,
+ "open_node": false
+ },
+ "interpreter.pipeline_execution.graph.graphs_inclusion.mermaidflow_mmd": {
+ "value_type": "bool",
+ "required": true,
+ "default": true,
+ "open_node": false
+ },
+ "interpreter.pipeline_execution.graph.graphs_inclusion.reactflow_html": {
+ "value_type": "bool",
+ "required": true,
+ "default": true,
+ "open_node": false
+ },
+ "interpreter.pipeline_execution.graph.mermaid": {
+ "value_type": "table",
+ "required": true,
+ "open_node": false
+ },
+ "interpreter.pipeline_execution.graph.mermaid.direction": {
+ "value_type": "enum",
+ "required": true,
+ "enum_members": [
+ "left_to_right",
+ "top_down"
+ ],
+ "default": "left_to_right",
+ "open_node": false
+ },
+ "interpreter.pipeline_execution.graph.mermaid.is_include_contains_edges": {
+ "value_type": "bool",
+ "required": true,
+ "default": false,
+ "open_node": false
+ },
+ "interpreter.pipeline_execution.graph.mermaid.is_include_data_edges": {
+ "value_type": "bool",
+ "required": true,
+ "default": true,
+ "open_node": false
+ },
+ "interpreter.pipeline_execution.graph.mermaid.is_include_selected_outcome_edges": {
+ "value_type": "bool",
+ "required": true,
+ "default": true,
+ "open_node": false
+ },
+ "interpreter.pipeline_execution.graph.mermaid.is_show_stuff_codes": {
+ "value_type": "bool",
+ "required": true,
+ "default": false,
+ "open_node": false
+ },
+ "interpreter.pipeline_execution.graph.mermaid.style": {
+ "value_type": "table",
+ "required": true,
+ "open_node": false
+ },
+ "interpreter.pipeline_execution.graph.mermaid.style.theme": {
+ "value_type": "enum",
+ "required": true,
+ "enum_members": [
+ "base",
+ "dark",
+ "default",
+ "forest",
+ "neutral"
+ ],
+ "default": "dark",
+ "open_node": false
+ },
+ "interpreter.pipeline_execution.graph.reactflow": {
+ "value_type": "table",
+ "required": true,
+ "open_node": false
+ },
+ "interpreter.pipeline_execution.graph.reactflow.default_title": {
+ "value_type": "str",
+ "required": true,
+ "default": "Pipelex Graph",
+ "open_node": false
+ },
+ "interpreter.pipeline_execution.graph.reactflow.edge_type": {
+ "value_type": "enum",
+ "required": true,
+ "enum_members": [
+ "bezier",
+ "smoothstep",
+ "step",
+ "straight"
+ ],
+ "default": "bezier",
+ "open_node": false
+ },
+ "interpreter.pipeline_execution.graph.reactflow.initial_zoom": {
+ "value_type": "float",
+ "required": true,
+ "default": 1.0,
+ "open_node": false
+ },
+ "interpreter.pipeline_execution.graph.reactflow.is_use_cdn": {
+ "value_type": "bool",
+ "required": true,
+ "default": true,
+ "open_node": false
+ },
+ "interpreter.pipeline_execution.graph.reactflow.layout_direction": {
+ "value_type": "enum",
+ "required": true,
+ "enum_members": [
+ "left_to_right",
+ "top_down"
+ ],
+ "default": "left_to_right",
+ "open_node": false
+ },
+ "interpreter.pipeline_execution.graph.reactflow.nodesep": {
+ "value_type": "int",
+ "required": true,
+ "default": 50,
+ "open_node": false
+ },
+ "interpreter.pipeline_execution.graph.reactflow.pan_to_top": {
+ "value_type": "bool",
+ "required": true,
+ "default": true,
+ "open_node": false
+ },
+ "interpreter.pipeline_execution.graph.reactflow.ranksep": {
+ "value_type": "int",
+ "required": true,
+ "default": 30,
+ "open_node": false
+ },
+ "interpreter.pipeline_execution.graph.reactflow.show_batch_controller": {
+ "value_type": "bool",
+ "required": true,
+ "default": true,
+ "open_node": false
+ },
+ "interpreter.pipeline_execution.graph.reactflow.style": {
+ "value_type": "table",
+ "required": true,
+ "open_node": false
+ },
+ "interpreter.pipeline_execution.graph.reactflow.style.palette": {
+ "value_type": "enum",
+ "required": true,
+ "enum_members": [
+ "dracula",
+ "yellow_blue"
+ ],
+ "default": "dracula",
+ "open_node": false
+ },
+ "interpreter.pipeline_execution.graph.reactflow.style.theme": {
+ "value_type": "enum",
+ "required": true,
+ "enum_members": [
+ "dark",
+ "light",
+ "system"
+ ],
+ "default": "dark",
+ "open_node": false
+ },
+ "interpreter.pipeline_execution.is_generate_graph": {
+ "value_type": "bool",
+ "required": true,
+ "default": true,
+ "open_node": false
+ },
+ "interpreter.pipeline_execution.is_generate_usage": {
+ "value_type": "bool",
+ "required": true,
+ "default": true,
+ "open_node": false
+ },
+ "interpreter.pipeline_execution.is_mock_inputs": {
+ "value_type": "bool",
+ "required": true,
+ "default": false,
+ "open_node": false
+ },
+ "interpreter.pipeline_execution.is_normalize_data_urls_to_storage": {
+ "value_type": "bool",
+ "required": true,
+ "default": true,
+ "open_node": false
+ },
+ "interpreter.pipeline_execution.max_concurrency": {
+ "value_type": "int | literal",
+ "required": true,
+ "enum_members": [
+ "unbounded"
+ ],
+ "default": 8,
+ "open_node": false,
+ "constraints": {
+ "ge": 1
+ }
+ },
+ "interpreter.scan": {
+ "value_type": "table",
+ "required": true,
+ "open_node": false
+ },
+ "interpreter.scan.excluded_dirs": {
+ "value_type": "frozenset[str]",
+ "required": true,
+ "default": [
+ ".venv",
+ "venv",
+ "env",
+ ".env",
+ "virtualenv",
+ ".virtualenv",
+ ".git",
+ "__pycache__",
+ ".pytest_cache",
+ ".mypy_cache",
+ ".ruff_cache",
+ "node_modules",
+ "results"
+ ],
+ "open_node": false
+ },
+ "kit": {
+ "value_type": "table",
+ "required": true,
+ "open_node": false
+ },
+ "kit.preferred_agent_targets": {
+ "value_type": "list[enum]",
+ "required": true,
+ "enum_members": [
+ "agents",
+ "claude",
+ "cursor"
+ ],
+ "default": [
+ "claude",
+ "agents"
+ ],
+ "open_node": false
+ },
+ "run": {
+ "value_type": "table",
+ "required": true,
+ "open_node": false
+ },
+ "run.execution": {
+ "value_type": "enum",
+ "required": true,
+ "enum_members": [
+ "hosted",
+ "local"
+ ],
+ "default": "local",
+ "open_node": false
+ },
+ "runtime": {
+ "value_type": "table",
+ "required": true,
+ "open_node": false
+ },
+ "runtime.aws": {
+ "value_type": "table",
+ "required": true,
+ "open_node": false
+ },
+ "runtime.aws.api_key_method": {
+ "value_type": "enum",
+ "required": true,
+ "enum_members": [
+ "env",
+ "secret_provider"
+ ],
+ "default": "env",
+ "open_node": false
+ },
+ "runtime.aws.bedrock_access_variant": {
+ "value_type": "enum",
+ "required": true,
+ "enum_members": [
+ "aws_access",
+ "bedrock_token"
+ ],
+ "default": "aws_access",
+ "open_node": false
+ },
+ "runtime.log": {
+ "value_type": "table",
+ "required": true,
+ "open_node": false
+ },
+ "runtime.log.caller_info_template": {
+ "value_type": "enum",
+ "required": true,
+ "enum_members": [
+ "file_func",
+ "file_line",
+ "file_line_func",
+ "func",
+ "func_line",
+ "func_module",
+ "func_module_line"
+ ],
+ "default": "file_line",
+ "open_node": false
+ },
+ "runtime.log.console_log_target": {
+ "value_type": "enum",
+ "required": true,
+ "enum_members": [
+ "file",
+ "stderr",
+ "stdout"
+ ],
+ "default": "stderr",
+ "open_node": false
+ },
+ "runtime.log.console_print_target": {
+ "value_type": "enum",
+ "required": true,
+ "enum_members": [
+ "file",
+ "stderr",
+ "stdout"
+ ],
+ "default": "stdout",
+ "open_node": false
+ },
+ "runtime.log.default_log_level": {
+ "value_type": "enum",
+ "required": true,
+ "enum_members": [
+ "CRITICAL",
+ "DEBUG",
+ "ERROR",
+ "INFO",
+ "OFF",
+ "VERBOSE",
+ "WARNING"
+ ],
+ "default": "INFO",
+ "open_node": false
+ },
+ "runtime.log.gcp": {
+ "value_type": "table",
+ "required": true,
+ "open_node": false
+ },
+ "runtime.log.gcp.credentials_file_path": {
+ "value_type": "str",
+ "required": false,
+ "open_node": false
+ },
+ "runtime.log.gcp.log_name": {
+ "value_type": "str",
+ "required": true,
+ "default": "pipelex",
+ "open_node": false
+ },
+ "runtime.log.gcp.project_id": {
+ "value_type": "str",
+ "required": false,
+ "open_node": false
+ },
+ "runtime.log.is_caller_info_enabled": {
+ "value_type": "bool",
+ "required": true,
+ "default": false,
+ "open_node": false
+ },
+ "runtime.log.json_logs_indent": {
+ "value_type": "int",
+ "required": true,
+ "default": 4,
+ "open_node": false
+ },
+ "runtime.log.otlp": {
+ "value_type": "table",
+ "required": true,
+ "open_node": false
+ },
+ "runtime.log.otlp.endpoint": {
+ "value_type": "str",
+ "required": false,
+ "open_node": false
+ },
+ "runtime.log.otlp.headers": {
+ "value_type": "dict[str, str]",
+ "required": true,
+ "default": {},
+ "open_node": true
+ },
+ "runtime.log.otlp.headers.*": {
+ "value_type": "str",
+ "required": true,
+ "open_node": false
+ },
+ "runtime.log.package_log_levels": {
+ "value_type": "dict[str, enum]",
+ "required": true,
+ "default": {
+ "anthropic": "INFO",
+ "asyncio": "INFO",
+ "botocore": "INFO",
+ "botocore-credentials": "WARNING",
+ "google": "INFO",
+ "httpx": "WARNING",
+ "httpcore": "INFO",
+ "openai": "INFO",
+ "instructor": "INFO",
+ "urllib3-connectionpool": "INFO",
+ "urllib3-util-retry": "INFO",
+ "pipelex": "INFO"
+ },
+ "open_node": true
+ },
+ "runtime.log.package_log_levels.*": {
+ "value_type": "enum",
+ "required": true,
+ "enum_members": [
+ "CRITICAL",
+ "DEBUG",
+ "ERROR",
+ "INFO",
+ "OFF",
+ "VERBOSE",
+ "WARNING"
+ ],
+ "open_node": false
+ },
+ "runtime.log.presentation_line_width": {
+ "value_type": "int",
+ "required": true,
+ "default": 120,
+ "open_node": false
+ },
+ "runtime.log.pretty_print_mode": {
+ "value_type": "enum",
+ "required": true,
+ "enum_members": [
+ "poor",
+ "rich",
+ "silent"
+ ],
+ "default": "rich",
+ "open_node": false
+ },
+ "runtime.log.redaction": {
+ "value_type": "table",
+ "required": true,
+ "open_node": false
+ },
+ "runtime.log.redaction.extra_patterns": {
+ "value_type": "list[str]",
+ "required": true,
+ "default": [],
+ "open_node": false
+ },
+ "runtime.log.redaction.is_enabled": {
+ "value_type": "bool",
+ "required": true,
+ "default": true,
+ "open_node": false
+ },
+ "runtime.log.rich_log": {
+ "value_type": "table",
+ "required": true,
+ "open_node": false
+ },
+ "runtime.log.rich_log.highlighter_name": {
+ "value_type": "enum",
+ "required": true,
+ "enum_members": [
+ "json",
+ "repr"
+ ],
+ "default": "json",
+ "open_node": false
+ },
+ "runtime.log.rich_log.is_link_path_enabled": {
+ "value_type": "bool",
+ "required": true,
+ "default": true,
+ "open_node": false
+ },
+ "runtime.log.rich_log.is_rich_tracebacks": {
+ "value_type": "bool",
+ "required": true,
+ "default": true,
+ "open_node": false
+ },
+ "runtime.log.rich_log.is_show_level": {
+ "value_type": "bool",
+ "required": true,
+ "default": true,
+ "open_node": false
+ },
+ "runtime.log.rich_log.is_show_time": {
+ "value_type": "bool",
+ "required": true,
+ "default": false,
+ "open_node": false
+ },
+ "runtime.log.rich_log.is_tracebacks_show_locals": {
+ "value_type": "bool",
+ "required": true,
+ "default": false,
+ "open_node": false
+ },
+ "runtime.log.rich_log.is_tracebacks_word_wrap": {
+ "value_type": "bool",
+ "required": true,
+ "default": true,
+ "open_node": false
+ },
+ "runtime.log.rich_log.keywords_to_hilight": {
+ "value_type": "list[str]",
+ "required": true,
+ "default": [],
+ "open_node": false
+ },
+ "runtime.log.rich_log.tracebacks_suppress": {
+ "value_type": "list[str]",
+ "required": true,
+ "default": [],
+ "open_node": false
+ },
+ "runtime.log.silenced_problem_ids": {
+ "value_type": "list[str]",
+ "required": true,
+ "default": [
+ "azure_openai_no_stream_options"
+ ],
+ "open_node": false
+ },
+ "runtime.log.sink": {
+ "value_type": "str",
+ "required": true,
+ "default": "console",
+ "open_node": false
+ },
+ "runtime.network": {
+ "value_type": "table",
+ "required": true,
+ "open_node": false
+ },
+ "runtime.network.is_fetch_ssrf_guard_enabled": {
+ "value_type": "bool",
+ "required": true,
+ "default": true,
+ "open_node": false
+ },
+ "runtime.observer": {
+ "value_type": "table",
+ "required": true,
+ "open_node": false
+ },
+ "runtime.observer.observer_dir": {
+ "value_type": "str",
+ "required": true,
+ "default": "results/observer",
+ "open_node": false
+ },
+ "runtime.plugins": {
+ "value_type": "table",
+ "required": true,
+ "open_node": false
+ },
+ "runtime.plugins.disabled": {
+ "value_type": "list[str]",
+ "required": true,
+ "default": [],
+ "open_node": false
+ },
+ "runtime.reporting": {
+ "value_type": "table",
+ "required": true,
+ "open_node": false
+ },
+ "runtime.reporting.cost_report_base_name": {
+ "value_type": "str",
+ "required": true,
+ "default": "cost_report",
+ "open_node": false
+ },
+ "runtime.reporting.cost_report_dir_path": {
+ "value_type": "str",
+ "required": true,
+ "default": "reports",
+ "open_node": false
+ },
+ "runtime.reporting.cost_report_extension": {
+ "value_type": "str",
+ "required": true,
+ "default": "csv",
+ "open_node": false
+ },
+ "runtime.reporting.cost_report_unit_scale": {
+ "value_type": "float",
+ "required": true,
+ "default": 1.0,
+ "open_node": false,
+ "constraints": {
+ "gt": 0
+ }
+ },
+ "runtime.reporting.is_generate_cost_report_file_enabled": {
+ "value_type": "bool",
+ "required": true,
+ "default": false,
+ "open_node": false
+ },
+ "runtime.reporting.is_log_costs_to_console": {
+ "value_type": "bool",
+ "required": true,
+ "default": true,
+ "open_node": false
+ },
+ "runtime.secrets": {
+ "value_type": "table",
+ "required": true,
+ "open_node": false
+ },
+ "runtime.secrets.method": {
+ "value_type": "str",
+ "required": true,
+ "default": "env",
+ "open_node": false
+ },
+ "runtime.storage": {
+ "value_type": "table",
+ "required": true,
+ "open_node": false
+ },
+ "runtime.storage.gcp": {
+ "value_type": "table",
+ "required": false,
+ "open_node": false
+ },
+ "runtime.storage.gcp.bucket_name": {
+ "value_type": "str",
+ "required": true,
+ "default": "",
+ "open_node": false
+ },
+ "runtime.storage.gcp.project_id": {
+ "value_type": "str",
+ "required": true,
+ "default": "",
+ "open_node": false
+ },
+ "runtime.storage.gcp.signed_urls_lifespan_seconds": {
+ "value_type": "int | literal",
+ "required": true,
+ "enum_members": [
+ "disabled"
+ ],
+ "default": 3600,
+ "open_node": false
+ },
+ "runtime.storage.gcp.uri_format": {
+ "value_type": "str",
+ "required": true,
+ "default": "{hash}.{extension}",
+ "open_node": false
+ },
+ "runtime.storage.in_memory": {
+ "value_type": "table",
+ "required": false,
+ "open_node": false
+ },
+ "runtime.storage.in_memory.uri_format": {
+ "value_type": "str",
+ "required": true,
+ "default": "{hash}.{extension}",
+ "open_node": false
+ },
+ "runtime.storage.is_fetch_remote_content_enabled": {
+ "value_type": "bool",
+ "required": true,
+ "default": true,
+ "open_node": false
+ },
+ "runtime.storage.is_upload_local_content_enabled": {
+ "value_type": "bool",
+ "required": true,
+ "default": true,
+ "open_node": false
+ },
+ "runtime.storage.local": {
+ "value_type": "table",
+ "required": false,
+ "open_node": false
+ },
+ "runtime.storage.local.local_storage_path": {
+ "value_type": "str",
+ "required": true,
+ "default": ".pipelex/storage",
+ "open_node": false
+ },
+ "runtime.storage.local.uri_format": {
+ "value_type": "str",
+ "required": true,
+ "default": "{hash}.{extension}",
+ "open_node": false
+ },
+ "runtime.storage.method": {
+ "value_type": "str",
+ "required": true,
+ "default": "local",
+ "open_node": false
+ },
+ "runtime.storage.s3": {
+ "value_type": "table",
+ "required": false,
+ "open_node": false
+ },
+ "runtime.storage.s3.bucket_name": {
+ "value_type": "str",
+ "required": true,
+ "default": "",
+ "open_node": false
+ },
+ "runtime.storage.s3.region": {
+ "value_type": "str",
+ "required": true,
+ "default": "",
+ "open_node": false
+ },
+ "runtime.storage.s3.signed_urls_lifespan_seconds": {
+ "value_type": "int | literal",
+ "required": true,
+ "enum_members": [
+ "disabled"
+ ],
+ "default": 3600,
+ "open_node": false
+ },
+ "runtime.storage.s3.uri_format": {
+ "value_type": "str",
+ "required": true,
+ "default": "{hash}.{extension}",
+ "open_node": false
+ },
+ "runtime.tracing": {
+ "value_type": "table",
+ "required": true,
+ "open_node": false
+ },
+ "runtime.tracing.backend": {
+ "value_type": "enum",
+ "required": true,
+ "enum_members": [
+ "dynamodb",
+ "ndjson"
+ ],
+ "default": "ndjson",
+ "open_node": false
+ },
+ "runtime.tracing.dynamodb": {
+ "value_type": "table",
+ "required": false,
+ "open_node": false
+ },
+ "runtime.tracing.dynamodb.region": {
+ "value_type": "str",
+ "required": true,
+ "default": "",
+ "open_node": false
+ },
+ "runtime.tracing.dynamodb.table_name": {
+ "value_type": "str",
+ "required": true,
+ "default": "",
+ "open_node": false
+ },
+ "runtime.tracing.is_enabled": {
+ "value_type": "bool",
+ "required": true,
+ "default": true,
+ "open_node": false
+ },
+ "runtime.tracing.ndjson": {
+ "value_type": "table",
+ "required": false,
+ "open_node": false
+ },
+ "runtime.tracing.ndjson.traces_dir": {
+ "value_type": "str",
+ "required": true,
+ "default": ".pipelex/traces",
+ "open_node": false
+ }
+ }
+}
diff --git a/pipelex/migration/ledgers/pipelex-config.toml b/pipelex/migration/ledgers/pipelex-config.toml
index 268ac54af..2d1c66d28 100644
--- a/pipelex/migration/ledgers/pipelex-config.toml
+++ b/pipelex/migration/ledgers/pipelex-config.toml
@@ -12,7 +12,7 @@ id = "pipelex-config"
title = "The main Pipelex configuration"
base_file = "pipelex.toml"
tier_glob = "pipelex_*.toml"
-current_schema_version = 6
+current_schema_version = 7
min_supported_schema_version = 0
# This entry is inserted *below* the reshape rather than appended above it, and the order is what
@@ -482,3 +482,27 @@ kind = "remap_value"
table_path = ["runtime", "log", "package_log_levels"]
key = "*"
mapping = { DEV = "INFO" }
+
+[[migration]]
+id = "pipelex-config@7"
+to_schema_version = 7
+introduced_in = "0.81.0"
+breaking = true
+safety = "safe"
+title = "The console reads no log message as markup"
+description = """
+`is_markup_enabled` under `[runtime.log.rich_log]` is gone: the `console` sink prints every log message as \
+written and reads none of it as Rich markup, the colour coming from the fields and the layouts instead. The \
+key is deleted from files that still set it.
+"""
+guidance = """
+Nothing to decide. No Pipelex log message carries markup any more, and a message read as markup lost \
+whatever in it was shaped like a tag, a `list[int]` or a lowercase logger's `[name]: ` prefix. Text you \
+log through Pipelex now prints exactly as written, so a message that relied on markup for its colour shows \
+its tags: put the values in `fields`, which the console colours by name.
+"""
+
+[[migration.ops]]
+kind = "delete_key"
+table_path = ["runtime", "log", "rich_log"]
+key = "is_markup_enabled"
diff --git a/pipelex/pipe_machinery/pipe_abstract.py b/pipelex/pipe_machinery/pipe_abstract.py
index ff5fb5dda..16d3c289c 100644
--- a/pipelex/pipe_machinery/pipe_abstract.py
+++ b/pipelex/pipe_machinery/pipe_abstract.py
@@ -46,6 +46,7 @@ from pipelex.system.telemetry.otel_constants import (
from pipelex.system.telemetry.otel_factory import OtelFactory
from pipelex.system.telemetry.telemetry_identity import make_run_identity_span_attributes
from pipelex.system.telemetry.telemetry_manager_abstract import TelemetryManagerAbstract
+from pipelex.tools.log.console_layouts import LogLayout
from pipelex.tools.misc.package_utils import get_package_version
from pipelex.tools.misc.string_utils import get_root_from_dotted_path, is_snake_case
from pipelex.validation_error_types import PipeValidationErrorType
@@ -55,6 +56,9 @@ if TYPE_CHECKING:
PipeAbstractType = type["PipeAbstract"]
+#: The message every live pipe run announces itself with; the pipe is in the fields, and the console draws it as the pipe tree.
+PIPE_RUN_STARTS_MESSAGE = "Pipe run starts"
+
class AbsentInput(NamedTuple):
"""One needed input that is absent from working memory, with its ledger record when one exists."""
@@ -636,18 +640,19 @@ class PipeAbstract(ABC, BaseModel):
"""
return {}
- def _format_pipe_run_info(self, pipe_run_params: PipeRunParams) -> str:
- indent_level = len(pipe_run_params.pipe_stack) - 1
- indent = " " * indent_level
- if indent_level > 0:
- indent = f"{indent}[yellow]↳[/yellow] "
- pipe_type_label = f"[white]{self.pipe_type}:[/white]"
- if pipe_run_params.run_mode.is_dry:
- pipe_type_label = f"[dim]Dry run:[/dim] {pipe_type_label}"
- pipe_code_label = f"[red]{self.code}[/red]"
- concept_code_label = f"[bold green]{self.output.concept.code}[/bold green]"
- arrow = "[yellow]→[/yellow]"
- return f"{indent}{pipe_type_label} {pipe_code_label} {arrow} {concept_code_label}"
+ def _pipe_run_fields(self, *, pipe_run_params: PipeRunParams) -> dict[str, Any]:
+ """The fields of the pipe announcement, the ones the console's pipe-run layout draws as the pipe tree.
+
+ The depth counts the pipes this one runs under: the top-level pipe is already on the stack when it
+ announces itself, so it stands at depth 0, and a pipe run with no stack at all is drawn there too.
+ Only a live run announces itself, so the fields carry no run mode.
+ """
+ return {
+ "pipe_type": self.pipe_type,
+ "pipe_code": self.code,
+ "output_concept": self.output.concept.code,
+ "pipe_depth": max(len(pipe_run_params.pipe_stack) - 1, 0),
+ }
@final
async def run_pipe(
@@ -983,6 +988,7 @@ class PipeAbstract(ABC, BaseModel):
output_name: str | None = None,
library_crate: LibraryCrate | None = None,
) -> PipeOutput:
+ assert not pipe_run_params.run_mode.is_dry, f"Live run of {self.type} '{self.code}' called with run_mode = {pipe_run_params.run_mode}"
# Generate pipe_run_id (business ID, always set)
this_pipe_run_id = PipelineFactory.make_pipe_run_id()
@@ -992,18 +998,21 @@ class PipeAbstract(ABC, BaseModel):
# comes back when the pipe returns, however it returns. A pipe lifted for absent optional
# inputs never gets here: it has no run and no id, and its skip line carries the enclosing binding.
with log.context(pipe_run_id=this_pipe_run_id):
- log.info(self._format_pipe_run_info(pipe_run_params=pipe_run_params))
+ # A fixed message, the pipe in the fields: the console draws the pipe tree from them through
+ # the pipe-run layout, and every other sink writes them as keys a query can select on.
+ log.info(PIPE_RUN_STARTS_MESSAGE, fields=self._pipe_run_fields(pipe_run_params=pipe_run_params), layout=LogLayout.PIPE_RUN)
# Handle telemetry ------------------------------------------------------------
- # Derive OtelContext if telemetry is enabled (not dry mode and tracer available)
+ # Derive OtelContext if telemetry is enabled (a tracer is available). The assertion above
+ # holds this method to a live run, so no run-mode check is needed.
# The trace_id comes from parent's otel_context (already computed at pipeline start)
this_otel_context: OtelContext | None = None
span: Span | None = None
is_root_span: bool = False
parent_otel_context = job_metadata.otel_context
- if not pipe_run_params.run_mode.is_dry and parent_otel_context is not None:
+ if parent_otel_context is not None:
# Start OTel span first
span, is_root_span = self._start_pipe_span(
parent_otel_context=parent_otel_context,
@@ -1023,7 +1032,7 @@ class PipeAbstract(ABC, BaseModel):
# Create child metadata with updated pipe_code and pipe_run_id
# This passes down a modified copy rather than mutating the original
# otel_context is passed separately because it must always be set explicitly
- # (even when None in dry mode) to avoid inheriting stale parent context
+ # (even when None, with no tracer) to avoid inheriting stale parent context
child_metadata = job_metadata.copy_with_update(
otel_context=this_otel_context,
pipe_code=self.code,
@@ -1076,9 +1085,8 @@ class PipeAbstract(ABC, BaseModel):
# unset — so everything downstream that identifies a step by it (leaf-activity labelling in a
# distributed backend, log correlation) sees an anonymous step in DRY and a named one in LIVE.
# Telemetry stays live-only on purpose: `pipe_run_id` and `otel_context` belong to a real run.
- # `otel_context=None` matches what `live_run_pipe` itself computes in dry mode, and clearing
- # it explicitly is the point of that parameter being required — inheriting the parent's would
- # attach a dry step to a live span.
+ # A dry run opens no span, so `otel_context` is cleared, and explicitly, which is the point of
+ # that parameter being required: inheriting the parent's would attach a dry step to a live span.
child_metadata = job_metadata.copy_with_update(otel_context=None, pipe_code=self.code)
return await self._dry_run_pipe(
job_metadata=child_metadata,
diff --git a/pipelex/pipe_operators/pipe_operator.py b/pipelex/pipe_operators/pipe_operator.py
index 9d37febf9..7174d0910 100644
--- a/pipelex/pipe_operators/pipe_operator.py
+++ b/pipelex/pipe_operators/pipe_operator.py
@@ -77,7 +77,9 @@ class PipeOperator(PipeAbstract, Generic[PipeOperatorOutputType]):
list_content: ListContent[StuffContent] = main_stuff.as_list_content() # pyright: ignore[reportUnknownVariableType, reportUnknownMemberType]
nb_items = len(list_content.items)
if nb_items == 0:
- output_concept_with_multiplicity += " [empty list]"
+ # Escaped, since the title is markup and Rich would read `[empty list]` as a tag and drop it; a count
+ # opens with a digit, which no tag does, so the two suffixes below print as written unescaped
+ output_concept_with_multiplicity += " \\[empty list]"
elif nb_items == 1:
output_concept_with_multiplicity += " [1 item]"
else:
diff --git a/pipelex/pipelex.toml b/pipelex/pipelex.toml
index 753da7e4c..af0f9f1f1 100644
--- a/pipelex/pipelex.toml
+++ b/pipelex/pipelex.toml
@@ -94,7 +94,6 @@ pipelex = "INFO"
is_show_time = false
is_show_level = true
is_link_path_enabled = true
-is_markup_enabled = true
highlighter_name = "json"
is_rich_tracebacks = true
is_tracebacks_word_wrap = true
diff --git a/pipelex/system/exceptions.py b/pipelex/system/exceptions.py
index 592b48356..a8fddaa58 100644
--- a/pipelex/system/exceptions.py
+++ b/pipelex/system/exceptions.py
@@ -3,7 +3,6 @@ from enum import StrEnum
from typing import ClassVar
from pipelex.base_exceptions import ErrorDomain, PipelexError, PipelexUnexpectedError
-from pipelex.tools.log.log_fields import VERBATIM_MARK
class ToolError(PipelexError):
@@ -53,17 +52,12 @@ class TracebackMessageError(PipelexError):
def __init__(self, message: str):
super().__init__(message)
logger_name = __name__
- # The message is assembled from whatever raised, so it carries text nobody chose: `[Errno 2]`, a
- # `list[int]` in a type complaint, a fragment of a traceback. On a console that interprets markup
- # that is a tag, and Rich either swallows the bracketed span or raises on an unbalanced one — an
- # error message is the last line that may be silently altered. The mark asks for it verbatim.
- verbatim = {VERBATIM_MARK: False}
match self.__class__.error_mode:
case TracebackMessageErrorMode.ERROR:
- logging.getLogger(logger_name).error(message, extra=verbatim)
+ logging.getLogger(logger_name).error(message)
case TracebackMessageErrorMode.EXCEPTION:
self.logger = logging.getLogger(logger_name)
- self.logger.exception(message, extra=verbatim)
+ self.logger.exception(message)
class FatalError(TracebackMessageError):
diff --git a/pipelex/tools/log/console_fields.py b/pipelex/tools/log/console_fields.py
index 832962e2a..f4be2c8c6 100644
--- a/pipelex/tools/log/console_fields.py
+++ b/pipelex/tools/log/console_fields.py
@@ -19,7 +19,7 @@ Colour follows the name the field was given, from ``FIELD_STYLES``, wherever the
collision prefix it landed under; a field outside the map renders dimmed. The map is one table in code, the
colours the pipe announcement has always used. Nothing here imports Rich: the suffix is a list of
``(text, style)`` segments the sink turns into a Rich ``Text``, never a markup string, so a value carrying
-``[red]`` prints as written whatever the handler's markup setting.
+``[red]`` prints as written.
"""
from __future__ import annotations
diff --git a/pipelex/tools/log/console_layouts.py b/pipelex/tools/log/console_layouts.py
index 2de52291a..6044c9fea 100644
--- a/pipelex/tools/log/console_layouts.py
+++ b/pipelex/tools/log/console_layouts.py
@@ -118,9 +118,11 @@ def _styled_placeholder(*, field: str, suffix: str = "") -> str:
#: How far a nested pipe run is indented per level of depth.
PIPE_RUN_INDENT = " "
#: What marks a nested pipe run, after its indentation.
-PIPE_RUN_BRANCH = "↳ "
-#: What precedes a dry run's pipe type.
-PIPE_RUN_DRY_RUN_LABEL = "Dry run: "
+PIPE_RUN_BRANCH = "↳"
+#: What separates the branch mark from the pipe type. It is a placeholder of its own, outside the mark's style, so
+#: the line is the very one the announcement printed when it was written as markup in its message, colour codes
+#: included.
+PIPE_RUN_GAP = " "
#: The deepest nesting the pipe-run layout indents. The indentation is built from the depth, so an unbounded
#: one builds a string as long as the caller likes, or raises ``OverflowError``; a deeper run than this,
#: which is far past any pipe stack a run reaches, falls back to its message.
@@ -128,23 +130,22 @@ PIPE_RUN_MAX_DEPTH = 100
class PipeRunLayout(ConsoleLayout):
- """The pipe-run tree: a nested run indented under its parent, behind a branch mark, a dry run labelled.
+ """The pipe-run tree: a nested run indented under its parent, behind a branch mark.
``PipeCompose: compose_company → Company`` at the top level, the same line indented and behind ``↳`` for
- a nested run, ``Dry run:`` before the pipe type for a dry one. It reads ``pipe_depth``, an integer from
- ``0`` for a top-level run up to ``PIPE_RUN_MAX_DEPTH``, and ``is_dry_run``, a boolean.
+ a nested run. It reads ``pipe_depth``, an integer from ``0`` for a top-level run up to
+ ``PIPE_RUN_MAX_DEPTH``. Only a live run announces itself, so the layout draws no run mode.
"""
@override
def derived_values(self, *, fields: Mapping[str, Any]) -> dict[str, Any]:
- """The indentation, the branch mark and the dry-run label, from a depth and a flag this layout checks first.
+ """The indentation and the branch mark, from a depth this layout checks first.
Raises:
- TypeError: If the depth is not an integer, or the flag not a boolean.
+ TypeError: If the depth is not an integer.
ValueError: If the depth is negative or deeper than ``PIPE_RUN_MAX_DEPTH``.
"""
depth = fields["pipe_depth"]
- is_dry_run = fields["is_dry_run"]
# A boolean is an integer to Python, and ``True`` would indent one level.
if isinstance(depth, bool) or not isinstance(depth, int):
msg = f"The pipe-run layout's depth must be an integer, not {type(depth).__name__}"
@@ -152,24 +153,21 @@ class PipeRunLayout(ConsoleLayout):
if not 0 <= depth <= PIPE_RUN_MAX_DEPTH:
msg = f"The pipe-run layout's depth must be between 0 and {PIPE_RUN_MAX_DEPTH}"
raise ValueError(msg)
- # Judged by truthiness, the string ``"false"`` would label a live run a dry one.
- if not isinstance(is_dry_run, bool):
- msg = f"The pipe-run layout's dry-run flag must be a boolean, not {type(is_dry_run).__name__}"
- raise TypeError(msg)
+ is_nested = depth > 0
return {
"indent": PIPE_RUN_INDENT * depth,
- "branch": PIPE_RUN_BRANCH if depth > 0 else "",
- "dry_run_label": PIPE_RUN_DRY_RUN_LABEL if is_dry_run else "",
+ "branch": PIPE_RUN_BRANCH if is_nested else "",
+ "branch_gap": PIPE_RUN_GAP if is_nested else "",
}
PIPE_RUN_LAYOUT = PipeRunLayout(
template=(
- "{indent}[yellow]{branch}[/yellow][dim]{dry_run_label}[/dim]"
+ "{indent}[yellow]{branch}[/yellow]{branch_gap}"
f"{_styled_placeholder(field='pipe_type', suffix=':')} {_styled_placeholder(field='pipe_code')} "
f"[yellow]→[/yellow] {_styled_placeholder(field='output_concept')}"
),
- presented_fields=frozenset({"pipe_type", "pipe_code", "output_concept", "pipe_depth", "is_dry_run"}),
+ presented_fields=frozenset({"pipe_type", "pipe_code", "output_concept", "pipe_depth"}),
)
#: Every layout a call can name, by its name.
diff --git a/pipelex/tools/log/console_log_sink.py b/pipelex/tools/log/console_log_sink.py
index ab44babe2..c7bc7bd0d 100644
--- a/pipelex/tools/log/console_log_sink.py
+++ b/pipelex/tools/log/console_log_sink.py
@@ -1,9 +1,11 @@
"""The ``console`` sink: the Rich handler with the emoji formatter and every ``[runtime.log.rich_log]`` setting.
-The handler renders a record's fields after its message as a styled ``key=value`` suffix, and a record that
-names a layout through that layout's template; what is shown and how it is coloured is in
-``console_fields`` and ``console_layouts``. A traceback printed as text, with Rich tracebacks off, goes
-under the line, after the suffix.
+The handler reads no message as Rich markup: a message is the caller's text, made of values nobody chose,
+and prints as written, ``list[int]`` and ``[red]`` included. The colour is the sink's own: it renders a
+record's fields after its message as a styled ``key=value`` suffix, and a record that names a layout
+through that layout's template; what is shown and how it is coloured is in ``console_fields`` and
+``console_layouts``. A traceback printed as text, with Rich tracebacks off, goes under the line, after the
+suffix.
Rich is the ``cli`` extra. It is imported when the handler is built and nowhere else in this module, so
this module asks for Rich only where this sink is the one selected; a process that selects another sink
@@ -66,11 +68,11 @@ class ConsoleLogSink(LogSink):
# Declared here because ``RichHandler`` is imported here, which is what keeps Rich off the import
# path of a process that selected another sink. ``RichHandler`` overrides ``emit`` and does not
# restore the ``try``/``handleError`` the stdlib's own handlers put around theirs, so anything
- # raised while rendering leaves the log call: a message carrying a tag-shaped span — `list[int]` in
- # a type complaint, or a bracketed path, both of which an error message is made of — raises
- # ``MarkupError`` out of `log.error` and replaces whatever was being reported with itself. A log
- # call never raises, so the guard goes back on and a line Rich cannot render gets the stdlib's own
- # recovery instead.
+ # raised while rendering leaves the log call: a record a third-party library stamped with Rich's own
+ # ``markup`` attribute, whose message Rich then reads as markup and cannot balance, raises
+ # ``MarkupError`` out of the call that logged it and replaces whatever was being reported with
+ # itself. A log call never raises, so the guard goes back on and a line Rich cannot render gets the
+ # stdlib's own recovery instead.
class GuardedRichHandler(RichHandler):
@override
def emit(self, record: logging.LogRecord) -> None:
@@ -83,8 +85,9 @@ class ConsoleLogSink(LogSink):
def render_message(self, record: logging.LogRecord, message: str) -> ConsoleRenderable:
"""The message, or the layout the call named, then the fields as a styled ``key=value`` suffix, then any traceback text.
- The suffix is assembled as ``Text`` from styled segments and appended after the highlighter has
- run on the message, so neither the markup setting nor the highlighter ever reads a field's value.
+ The message is plain text, read as no markup, and the suffix is assembled as ``Text`` from styled
+ segments and appended after the highlighter has run on the message, so the highlighter never reads
+ a field's value.
A layout that cannot be filled, whatever it raises, falls back to the message, and every field
then goes to the suffix. The dispatch stamps a layout only on a call whose content is a string,
so a structured content, which only its message renders, never meets one.
@@ -143,7 +146,13 @@ class ConsoleLogSink(LogSink):
show_level=config.is_show_level,
enable_link_path=config.is_link_path_enabled,
highlighter=highlighter,
- markup=config.is_markup_enabled,
+ # A message is the caller's text, and none of Pipelex's carries markup: colour comes from the
+ # fields and the layouts, which style their own text. Read as markup, a message loses whatever
+ # is shaped like a tag, the `[name]: ` channel prefix of a lowercase logger, a `list[int]`, the
+ # `[cycle]` marker of a circular content, or raises on a bracketed path. Rich still honours its
+ # own per-record `markup` attribute, which no Pipelex call can set: a field of that name is
+ # reserved and lands under the `field_` prefix.
+ markup=False,
rich_tracebacks=config.is_rich_tracebacks,
tracebacks_word_wrap=config.is_tracebacks_word_wrap,
tracebacks_show_locals=config.is_tracebacks_show_locals,
diff --git a/pipelex/tools/log/log_config.py b/pipelex/tools/log/log_config.py
index 63dd9f81f..4c2f95457 100644
--- a/pipelex/tools/log/log_config.py
+++ b/pipelex/tools/log/log_config.py
@@ -56,7 +56,6 @@ class RichLogConfig(ConfigModel):
is_show_level: bool
is_link_path_enabled: bool
highlighter_name: HighlighterName = Field(strict=False)
- is_markup_enabled: bool
is_rich_tracebacks: bool
is_tracebacks_word_wrap: bool
is_tracebacks_show_locals: bool
diff --git a/pipelex/tools/log/log_fields.py b/pipelex/tools/log/log_fields.py
index 9e1247a87..5fbcbe96a 100644
--- a/pipelex/tools/log/log_fields.py
+++ b/pipelex/tools/log/log_fields.py
@@ -39,13 +39,11 @@ FORMATTER_OWNED_ATTRIBUTES = frozenset({"message", "asctime"})
# the marker and cost the whole record its delivery.
FORWARDED_MARK = "_pipelex_forwarded"
-# Rich's own per-record override of its handler's markup setting, which it reads as
-# ``getattr(record, "markup", self.markup)``, and so the one way to put one record on the console verbatim
-# while the handler goes on interpreting markup for every other. A message assembled from an exception is
-# what needs it: text like ``[Errno 2]`` is a markup tag to Rich, which swallows the span, or raises on an
-# unbalanced one. The name is Rich's rather than ours, but stamping it is a console concern that rides no
-# wire, and reserving it is what stops a caller steering the console through a field of that name.
-VERBATIM_MARK = "markup"
+# Rich's per-record override of its handler's markup setting, which ``RichHandler.render_message`` reads as
+# ``getattr(record, "markup", self.markup)``. The console sink reads no message as markup, and a caller's
+# field landing on this name would turn markup back on for its line, where a tag-shaped span of the message,
+# a ``list[int]`` or a bracketed path, is swallowed or raises.
+RICH_MARKUP_ATTRIBUTE = "markup"
# Rich's per-record override of its handler's highlighter, which ``RichHandler.render_message`` reads as
# ``getattr(record, "highlighter", self.highlighter)`` and then calls on the line. A caller's field landing
@@ -55,8 +53,8 @@ RICH_HIGHLIGHTER_ATTRIBUTE = "highlighter"
# Every attribute Rich's handler reads off a record ahead of its own setting: ``render_message`` reads these
# two, and nothing else ``emit`` or ``render`` reads is outside the stdlib's own attributes. Each steers the
# console alone and means nothing on a wire, so neither is a field a caller can land on nor one a sink is
-# handed as something the record carries.
-RICH_RECORD_OVERRIDES = frozenset({VERBATIM_MARK, RICH_HIGHLIGHTER_ATTRIBUTE})
+# handed as something the record carries, whoever stamped it.
+RICH_RECORD_OVERRIDES = frozenset({RICH_MARKUP_ATTRIBUTE, RICH_HIGHLIGHTER_ATTRIBUTE})
# The attribute the redaction stamps on a record it could not strip, and takes back off the moment it
# has. It is the fail-closed half of the scrub: a record still carrying it reaches no sink, because what
@@ -82,7 +80,7 @@ LAYOUT_MARK = "_pipelex_layout"
# and so a fresh record owns none of them: a caller's field spelling ``FORWARDED_MARK`` would be read as
# the forwarding marker and cost the whole record its delivery, and one spelling ``UNSCRUBBED_MARK`` would
# have a perfectly ordinary record read as one the scrub could not strip, and dropped.
-PIPELEX_OWNED_ATTRIBUTES = frozenset({FORWARDED_MARK, VERBATIM_MARK, UNSCRUBBED_MARK, FIELD_NAMES_MARK, LAYOUT_MARK})
+PIPELEX_OWNED_ATTRIBUTES = frozenset({FORWARDED_MARK, UNSCRUBBED_MARK, FIELD_NAMES_MARK, LAYOUT_MARK})
# Reserved whether or not the record carries the name yet, which is exactly what the stdlib's own refusal
# cannot cover: the formatter's and Pipelex's are stamped after the entries are attached, and Rich's are
diff --git a/pipelex/tools/log/log_holding.py b/pipelex/tools/log/log_holding.py
index 597d77907..1c76c253b 100644
--- a/pipelex/tools/log/log_holding.py
+++ b/pipelex/tools/log/log_holding.py
@@ -102,8 +102,8 @@ class HoldingLogHandler(logging.Handler):
def release_to(self, *, handler: logging.Handler) -> None:
"""Hand every held record to the handler, in the order they were emitted, and forward to it whatever arrives after.
- One record the handler cannot render, a line Rich reads as unbalanced markup for one, gets the
- stdlib's own recovery, ``handleError``, and costs none of the records after it.
+ One record the handler cannot render, a third-party line Rich is asked to read as markup and cannot
+ balance for one, gets the stdlib's own recovery, ``handleError``, and costs none of the records after it.
A drained record is marked after its delivery exactly as a forwarded one is, and for the same
reason: a thread that read the root logger's handler list before the handoff can still reach the
diff --git a/subject_grants.toml b/subject_grants.toml
index 623f0edda..369054e86 100644
--- a/subject_grants.toml
+++ b/subject_grants.toml
@@ -4028,10 +4028,6 @@ rationale = "Verb-object: ends the span; result fields stay keyword."
param = "needed_inputs"
rationale = "Noun-named derivation from the needed inputs; single operand."
-["pipelex/pipe_machinery/pipe_abstract.py::PipeAbstract._format_pipe_run_info"]
-param = "pipe_run_params"
-rationale = "Derivation from the run params; single operand."
-
["pipelex/pipe_machinery/pipe_abstract.py::PipeAbstract._make_single_concept_data_for_registry"]
param = "concept"
rationale = "Derivation from the concept; single operand."
diff --git a/tests/helpers/console_log_rendering.py b/tests/helpers/console_log_rendering.py
index be7911488..423f345bb 100644
--- a/tests/helpers/console_log_rendering.py
+++ b/tests/helpers/console_log_rendering.py
@@ -27,13 +27,12 @@ if TYPE_CHECKING:
CONSOLE_WIDTH = 200
-# The fields the pipe-run layout presents, for a top-level run that is not a dry run.
+# The fields the pipe-run layout presents, for a top-level run.
PIPE_RUN_FIELDS: dict[str, Any] = {
"pipe_type": "PipeCompose",
"pipe_code": "compose_company",
"output_concept": "Company",
"pipe_depth": 0,
- "is_dry_run": False,
}
diff --git a/tests/helpers/rich_markup.py b/tests/helpers/rich_markup.py
new file mode 100644
index 000000000..8251ad65e
--- /dev/null
+++ b/tests/helpers/rich_markup.py
@@ -0,0 +1,49 @@
+"""What Rich would read as console markup in a piece of text: the tags it applies as styling rather than prints.
+
+A tag-shaped span is not markup by its shape alone. Rich's tag pattern also matches the ``[int]`` of ``list[int]``,
+a ``[cycle]`` marker and a backend's table, ``[openai]``, yet none of them names a style, and a log message holding
+one is plain text that prints as written. So a tag counts as markup only when Rich would style with it: a closing
+tag, ``[/]`` or ``[/red]``; an ``@`` handler, ``[@click=app.bell]``; or an opening tag whose text Rich parses as a
+style, ``[bold]``, ``[on blue]``, ``[link=https://pipelex.com]``, or that names a style of Rich's default theme,
+``[repr.number]``. A tag escaped with a backslash is text, since Rich prints it as written.
+"""
+
+from __future__ import annotations
+
+from rich.default_styles import DEFAULT_STYLES
+from rich.errors import StyleSyntaxError
+from rich.markup import RE_TAGS
+from rich.style import Style
+
+
+def _reads_as_style(*, tag_text: str) -> bool:
+ """Whether Rich applies a tag's text as styling, rather than leaving the reader a bracketed word.
+
+ A closing tag and an ``@`` handler are markup whatever they name. An opening tag is markup when it names a style
+ of the default theme, or when Rich parses its text as a style, ``name=parameters`` read the way Rich's own
+ ``Tag`` reads it.
+ """
+ if tag_text.startswith(("/", "@")):
+ return True
+ if tag_text in DEFAULT_STYLES:
+ return True
+ name, separator, parameters = tag_text.partition("=")
+ style_text = f"{name} {parameters}" if separator else name
+ try:
+ Style.parse(style_text)
+ except StyleSyntaxError:
+ return False
+ return True
+
+
+def find_markup_tags(*, text: str) -> list[str]:
+ """The unescaped tags in the text that Rich would apply as markup, in order, each as written."""
+ tags: list[str] = []
+ for match in RE_TAGS.finditer(text):
+ full_text, escapes, tag_text = match.groups()
+ # An odd run of backslashes escapes the tag, which then prints as written.
+ if len(escapes) % 2 == 1:
+ continue
+ if _reads_as_style(tag_text=tag_text):
+ tags.append(full_text[len(escapes) :])
+ return tags
diff --git a/tests/helpers/test_rich_markup.py b/tests/helpers/test_rich_markup.py
new file mode 100644
index 000000000..cc2bd9883
--- /dev/null
+++ b/tests/helpers/test_rich_markup.py
@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+import pytest
+
+from tests.helpers.rich_markup import find_markup_tags
+
+
+class TestFindMarkupTags:
+ @pytest.mark.parametrize(
+ "text",
+ [
+ "Expected list[int], got str",
+ "The content refers to itself: [cycle]",
+ "Add 'enabled = false' under '[openai]' in '.pipelex/inference/backends.toml'",
+ "[Errno 2] No such file or directory: 'graph.json'",
+ "Read /work/[draft]/graph.json",
+ "An escaped tag prints as written: \\[bold]",
+ "Pipe run starts",
+ ],
+ ids=["a type", "the cycle marker", "a backend table", "an errno", "a bracketed path", "an escaped tag", "no bracket at all"],
+ )
+ def test_a_bracketed_word_rich_applies_no_style_for_is_not_markup(self, text: str) -> None:
+ assert find_markup_tags(text=text) == []
+
+ @pytest.mark.parametrize(
+ ("text", "expected_tags"),
+ [
+ ("[bold]x[/bold]", ["[bold]", "[/bold]"]),
+ ("[red]x[/]", ["[red]", "[/]"]),
+ ("[link=https://x]y[/link]", ["[link=https://x]", "[/link]"]),
+ ("Click [@click=app.bell]here", ["[@click=app.bell]"]),
+ ("A number in [repr.number]the theme's style", ["[repr.number]"]),
+ ("Expected list[int], got [on blue]str", ["[on blue]"]),
+ ],
+ ids=["a style and its closing tag", "a colour closed by the bare tag", "a link", "a handler", "a theme style", "beside a type"],
+ )
+ def test_a_tag_rich_would_style_with_is_markup(self, text: str, expected_tags: list[str]) -> None:
+ assert find_markup_tags(text=text) == expected_tags
diff --git a/tests/integration/pipelex/pipeline/test_data.py b/tests/integration/pipelex/pipeline/test_data.py
index a621f819c..b3ca9c472 100644
--- a/tests/integration/pipelex/pipeline/test_data.py
+++ b/tests/integration/pipelex/pipeline/test_data.py
@@ -1,5 +1,7 @@
from typing import ClassVar
+from mthds.protocol.pipeline_inputs import PipelineInputs
+
class LocatedRunFailureTestData:
"""Bundles whose runs fail at a nested pipe, one per way a run failure is located."""
@@ -411,3 +413,101 @@ inputs = { topic = "Text" }
output = "Text"
template = "About {{ topic }}"
"""
+
+
+class LogMarkupTestData:
+ """A nested method run live, every model call answered by a stand-in worker, to read what its log messages say."""
+
+ MAIN_PIPE: ClassVar[str] = "profile_company"
+
+ INPUTS: ClassVar[PipelineInputs] = {"brief": "Acme builds reusable rockets in Toulouse and sells launches to research labs."}
+
+ MTHDS: ClassVar[str] = """
+domain = "log_markup_check"
+description = "Describe a company, structure it, summarize it two ways at once, route and compose a report"
+main_pipe = "profile_company"
+
+[concept.Company]
+description = "A company"
+
+[concept.Company.structure]
+name = { type = "text", description = "The company's name" }
+sector = { type = "text", description = "The company's sector" }
+
+[concept.ShortSummary]
+description = "A one-sentence summary"
+refines = "Text"
+
+[concept.DetailedSummary]
+description = "A detailed summary"
+refines = "Text"
+
+[pipe.profile_company]
+type = "PipeSequence"
+description = "Describe the company, structure it, summarize it and compose the report"
+inputs = { brief = "Text" }
+output = "Text"
+steps = [
+ { pipe = "describe_company", result = "description" },
+ { pipe = "structure_company", result = "company" },
+ { pipe = "summarize_company", result = "summaries" },
+ { pipe = "route_report", result = "report" },
+]
+
+[pipe.describe_company]
+type = "PipeLLM"
+description = "Describe the company the brief is about"
+inputs = { brief = "Text" }
+output = "Text"
+prompt = "Describe the company this brief is about: $brief"
+
+[pipe.structure_company]
+type = "PipeLLM"
+description = "Structure the description as a company"
+inputs = { description = "Text" }
+output = "Company"
+prompt = "Read the company out of this description: $description"
+
+[pipe.summarize_company]
+type = "PipeParallel"
+description = "Summarize the description in one sentence and in detail, at once"
+inputs = { description = "Text" }
+output = "Composite"
+add_each_output = true
+branches = [
+ { pipe = "summarize_short", result = "short_summary" },
+ { pipe = "summarize_detailed", result = "detailed_summary" },
+]
+
+[pipe.summarize_short]
+type = "PipeLLM"
+description = "Summarize in one sentence"
+inputs = { description = "Text" }
+output = "ShortSummary"
+prompt = "Summarize in one sentence: $description"
+
+[pipe.summarize_detailed]
+type = "PipeLLM"
+description = "Summarize in detail"
+inputs = { description = "Text" }
+output = "DetailedSummary"
+prompt = "Summarize in detail: $description"
+
+[pipe.route_report]
+type = "PipeCondition"
+description = "Pick the report's shape"
+inputs = { company = "Company", short_summary = "ShortSummary" }
+output = "Text"
+expression_template = "{{ 'compact' }}"
+default_outcome = "compose_report"
+
+[pipe.route_report.outcomes]
+compact = "compose_report"
+
+[pipe.compose_report]
+type = "PipeCompose"
+description = "Compose the report"
+inputs = { company = "Company", short_summary = "ShortSummary" }
+output = "Text"
+template = "{{ company.name }} works in {{ company.sector }}: {{ short_summary.text }}"
+"""
diff --git a/tests/integration/pipelex/pipeline/test_log_messages_carry_no_markup.py b/tests/integration/pipelex/pipeline/test_log_messages_carry_no_markup.py
new file mode 100644
index 000000000..f65c5e53d
--- /dev/null
+++ b/tests/integration/pipelex/pipeline/test_log_messages_carry_no_markup.py
@@ -0,0 +1,124 @@
+"""A live run writes no log message carrying Rich markup, at any level, as the ``json`` sink receives it.
+
+Colour is the console sink's job: it styles a record's fields by their names and draws the few named layouts,
+and reads no message as markup. A message carrying a tag would reach every other sink with the tag in it, as
+the pipe announcement did when a helper assembled its markup, which is exactly what a search of the log calls
+cannot see. So this runs a method and reads every message the ``json`` sink was handed, at the lowest level,
+for a tag Rich would apply as styling. A bracketed word that names no style, a ``list[int]``, a ``[cycle]``
+marker or a backend's ``[openai]`` table, is plain text that prints as written, and passes.
+
+The run is live, because only a live run announces its pipes, a dry one taking another path. Every model call
+is answered by a stand-in worker, so nothing reaches a provider and the run costs nothing.
+"""
+
+from __future__ import annotations
+
+import io
+import json
+import logging
+from contextlib import contextmanager
+from typing import TYPE_CHECKING, Any
+
+import pytest
+from typing_extensions import override
+
+from pipelex.cogt.llm.llm_worker_abstract import LLMWorkerAbstract
+from pipelex.cogt.llm.thinking_mode import ThinkingMode
+from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
+from pipelex.cogt.model_backends.model_type import ModelType
+from pipelex.pipe_machinery.pipe_abstract import PIPE_RUN_STARTS_MESSAGE
+from pipelex.pipeline.runner import PipelexMTHDSProtocol
+from pipelex.system.pipe_run_mode import PipeRunMode
+from pipelex.tools.log.json_log_sink import LOGGER_KEY, MESSAGE_KEY, SEVERITY_KEY, JsonLogSink
+from pipelex.tools.log.log_levels import LOGGING_LEVEL_VERBOSE, LOGGING_LEVEL_VERBOSE_NAME
+from tests.helpers.rich_markup import find_markup_tags
+from tests.integration.pipelex.pipeline.test_data import LogMarkupTestData
+
+if TYPE_CHECKING:
+ from collections.abc import Generator
+
+ from pytest_mock import MockerFixture
+
+ from pipelex.cogt.llm.llm_job import LLMJob
+ from pipelex.tools.typing.pydantic_utils import BaseModelTypeVar
+
+STAND_IN_ANSWER = "Acme builds reusable rockets for research labs."
+
+
+class StandInLLMWorker(LLMWorkerAbstract):
+ """Answers every generation without a provider: a fixed text, and an object whose every field is a fixed text."""
+
+ @override
+ async def _gen_text(self, llm_job: LLMJob) -> str:
+ return STAND_IN_ANSWER
+
+ @override
+ async def _gen_object(self, llm_job: LLMJob, *, schema: type[BaseModelTypeVar]) -> BaseModelTypeVar:
+ return schema.model_validate({field_name: f"Stand-in {field_name}" for field_name in schema.model_fields})
+
+
+def _stand_in_worker() -> StandInLLMWorker:
+ inference_model = InferenceModelSpec(
+ backend_name="stand_in",
+ name="stand-in-llm",
+ sdk="stand_in",
+ model_type=ModelType.LLM,
+ model_id="stand-in-llm-1",
+ inputs=["text"],
+ outputs=["text", "structured"],
+ costs={},
+ thinking_mode=ThinkingMode.NONE,
+ max_tokens=None,
+ max_prompt_images=None,
+ )
+ return StandInLLMWorker(inference_model=inference_model, reporting_delegate=None)
+
+
+@contextmanager
+def _json_sink_on_root(*, buffer: io.StringIO) -> Generator[None]:
+ """The ``json`` sink's handler on the root logger, beside whatever the session installed, for as long as the block runs."""
+ handler = JsonLogSink(stream=buffer).handler
+ root_logger = logging.getLogger()
+ root_logger.addHandler(handler)
+ try:
+ yield
+ finally:
+ root_logger.removeHandler(handler)
+ handler.close()
+
+
[email protected](loop_scope="class")
+class TestLogMessagesCarryNoMarkup:
+ async def test_a_live_run_logs_no_message_carrying_markup(self, mocker: MockerFixture, caplog: pytest.LogCaptureFixture) -> None:
+ mocker.patch("pipelex.cogt.content_generation.llm_generate.get_llm_worker", return_value=_stand_in_worker())
+ # Every Pipelex logger down to the lowest level; `caplog` puts the level back afterwards.
+ caplog.set_level(LOGGING_LEVEL_VERBOSE, logger="pipelex")
+ runner = PipelexMTHDSProtocol(pipe_run_mode=PipeRunMode.LIVE)
+ buffer = io.StringIO()
+
+ with _json_sink_on_root(buffer=buffer):
+ response = await runner.execute(
+ pipe_code=LogMarkupTestData.MAIN_PIPE, mthds_contents=[LogMarkupTestData.MTHDS], inputs=LogMarkupTestData.INPUTS
+ )
+
+ lines: list[dict[str, Any]] = [json.loads(line) for line in buffer.getvalue().splitlines() if line]
+ assert response.pipe_output.main_stuff_as_text.text == f"Stand-in name works in Stand-in sector: {STAND_IN_ANSWER}"
+ # The run was read at every level, and live: each pipe announced itself, the nested ones below the method's.
+ assert any(line[SEVERITY_KEY] == LOGGING_LEVEL_VERBOSE_NAME for line in lines)
+ announced = {(line["pipe_code"], line["pipe_depth"]) for line in lines if line[MESSAGE_KEY] == PIPE_RUN_STARTS_MESSAGE}
+ assert announced == {
+ ("profile_company", 0),
+ ("describe_company", 1),
+ ("structure_company", 1),
+ ("summarize_company", 1),
+ ("summarize_short", 2),
+ ("summarize_detailed", 2),
+ ("route_report", 1),
+ ("compose_report", 2),
+ }
+ carrying_markup = [
+ f"{line[LOGGER_KEY]}: {line[MESSAGE_KEY]!r} holds {find_markup_tags(text=line[MESSAGE_KEY])}"
+ for line in lines
+ if find_markup_tags(text=line[MESSAGE_KEY])
+ ]
+ assert not carrying_markup, "These log messages carry what Rich would apply as markup:\n" + "\n".join(carrying_markup)
diff --git a/tests/unit/pipelex/cli/dev_cli/test_dev_cli_failure_prints.py b/tests/unit/pipelex/cli/dev_cli/test_dev_cli_failure_prints.py
new file mode 100644
index 000000000..c8ccb48a5
--- /dev/null
+++ b/tests/unit/pipelex/cli/dev_cli/test_dev_cli_failure_prints.py
@@ -0,0 +1,124 @@
+from __future__ import annotations
+
+import io
+from typing import TYPE_CHECKING
+
+import pytest
+from rich.console import Console
+
+from pipelex.base_exceptions import PipelexError
+from pipelex.cli.dev_cli.commands import generate_projection_corpus_cmd as corpus_mod
+from pipelex.cli.dev_cli.commands import trace_input_semantics_cmd as trace_mod
+from pipelex.pipeline.exceptions import ValidateBundleError
+
+if TYPE_CHECKING:
+ from collections.abc import Callable
+ from pathlib import Path
+ from types import ModuleType
+
+ from pytest_mock import MockerFixture
+
+#: Wide enough that no assertion below depends on where Rich decides to wrap.
+CONSOLE_WIDTH = 400
+
+#: A style tag Rich would apply and drop, and a closing tag its markup parser would refuse with a MarkupError.
+BRACKETED_TEXT = "the [bold]compact[/bold] shape of list[int] ends at [/closing]"
+
+
+def _corpus_cmd(*, bundle_paths: list[Path], output_dir: Path) -> None:
+ corpus_mod.generate_projection_corpus_cmd(bundle_paths=bundle_paths, output_dir=output_dir)
+
+
+def _trace_cmd(*, bundle_paths: list[Path], output_dir: Path) -> None:
+ trace_mod.trace_input_semantics_cmd(bundle_paths=bundle_paths, output_dir=output_dir)
+
+
+class TestDevCliFailurePrints:
+ @pytest.fixture
+ def console_buffer(self, mocker: MockerFixture) -> io.StringIO:
+ """Route both command modules' consoles to one StringIO-backed console, and keep Pipelex from booting."""
+ buffer = io.StringIO()
+ console = Console(file=buffer, force_terminal=False, width=CONSOLE_WIDTH)
+ for module in (corpus_mod, trace_mod):
+ mocker.patch.object(module, "get_console", return_value=console)
+ mocker.patch.object(module, "make_pipelex_for_cli")
+ mocker.patch.object(module.Pipelex, "teardown_if_needed")
+ return buffer
+
+ @pytest.mark.parametrize(
+ ("module", "run_command", "work_function_name", "failure", "heading"),
+ [
+ pytest.param(
+ corpus_mod,
+ _corpus_cmd,
+ "generate_projection_corpus",
+ ValidateBundleError(message=BRACKETED_TEXT),
+ "Bundle validation failed — the corpus requires valid bundles:",
+ id="corpus-validation",
+ ),
+ pytest.param(
+ corpus_mod,
+ _corpus_cmd,
+ "generate_projection_corpus",
+ ValueError(BRACKETED_TEXT),
+ "The corpus's own record is out of date:",
+ id="corpus-record",
+ ),
+ pytest.param(
+ corpus_mod,
+ _corpus_cmd,
+ "generate_projection_corpus",
+ PipelexError(BRACKETED_TEXT),
+ "Corpus generation failed:",
+ id="corpus-generation",
+ ),
+ pytest.param(
+ trace_mod,
+ _trace_cmd,
+ "trace_input_semantics",
+ ValidateBundleError(message=BRACKETED_TEXT),
+ "Bundle validation failed — the trace requires a valid bundle:",
+ id="trace-validation",
+ ),
+ pytest.param(
+ trace_mod,
+ _trace_cmd,
+ "trace_input_semantics",
+ PipelexError(BRACKETED_TEXT),
+ "Trace failed:",
+ id="trace-failure",
+ ),
+ ],
+ )
+ def test_a_failure_prints_its_bracketed_text_as_written(
+ self,
+ mocker: MockerFixture,
+ console_buffer: io.StringIO,
+ tmp_path: Path,
+ module: ModuleType,
+ run_command: Callable[..., None],
+ work_function_name: str,
+ failure: Exception,
+ heading: str,
+ ) -> None:
+ """The failure's text follows its markup heading as written: no tag in it is applied, dropped or refused."""
+ bundle_path = tmp_path / "bundle.mthds"
+ bundle_path.write_text("", encoding="utf-8")
+ mocker.patch.object(module, work_function_name, side_effect=failure)
+
+ with pytest.raises(SystemExit) as exit_info:
+ run_command(bundle_paths=[bundle_path], output_dir=tmp_path / "out")
+
+ assert exit_info.value.code == 1
+ assert console_buffer.getvalue() == f"{heading}\n{BRACKETED_TEXT}\n"
+
+ @pytest.mark.parametrize("run_command", [pytest.param(_corpus_cmd, id="corpus"), pytest.param(_trace_cmd, id="trace")])
+ def test_a_missing_bundle_is_named_with_its_brackets(self, console_buffer: io.StringIO, tmp_path: Path, run_command: Callable[..., None]) -> None:
+ """A bundle path holding brackets is named as it stands when the file is missing."""
+ missing_bundle = tmp_path / "[bold]draft[/bold]" / "bundle.mthds"
+
+ with pytest.raises(SystemExit) as exit_info:
+ run_command(bundle_paths=[missing_bundle], output_dir=tmp_path / "out")
+
+ assert exit_info.value.code == 2
+ assert console_buffer.getvalue() == f"Bundle file not found: {missing_bundle}\n"
diff --git a/tests/unit/pipelex/cli/test_doctor_display_report.py b/tests/unit/pipelex/cli/test_doctor_display_report.py
index c49e42b69..5cda354ba 100644
--- a/tests/unit/pipelex/cli/test_doctor_display_report.py
+++ b/tests/unit/pipelex/cli/test_doctor_display_report.py
@@ -508,6 +508,27 @@ class TestDoctorDisplayReport:
assert "my[test]backend" in output
assert "MY[dev]_API_KEY" in output
+ def test_the_credentials_advice_names_each_backend_table_as_written(self, console: Console) -> None:
+ """The advice names a backend's table as `[openai]`, which the console would read as a style tag and drop."""
+ kwargs = _healthy_report_kwargs()
+ kwargs["backends_healthy"] = False
+ kwargs["backends_message"] = "1 backend(s) have missing or invalid credentials"
+ kwargs["backend_credential_reports"] = {
+ "openai": BackendCredentialsReport(
+ backend_name="openai",
+ required_vars=["OPENAI_API_KEY"],
+ missing_vars=["OPENAI_API_KEY"],
+ placeholder_vars=[],
+ all_credentials_valid=False,
+ )
+ }
+
+ display_health_report(**kwargs)
+
+ output = console.export_text()
+ assert " - Add 'enabled = false' under '[openai]'" in output
+ assert "\\[" not in output
+
def test_a_bracketed_deck_filename_survives_the_per_file_detail(self, console: Console) -> None:
kwargs = _healthy_report_kwargs()
kwargs["deck_healthy"] = False
diff --git a/tests/unit/pipelex/cli/test_graph_render_cmd.py b/tests/unit/pipelex/cli/test_graph_render_cmd.py
index 8c159677a..cba43047b 100644
--- a/tests/unit/pipelex/cli/test_graph_render_cmd.py
+++ b/tests/unit/pipelex/cli/test_graph_render_cmd.py
@@ -37,6 +37,7 @@ class TestGraphRenderRefusedSpec:
telemetry_manager.telemetry_context.return_value = contextlib.nullcontext()
printed = io.StringIO()
mocker.patch(f"{GRAPH_CMD}.get_console", return_value=Console(file=printed, width=1000))
+ log_spy = mocker.patch(f"{GRAPH_CMD}.log")
spec_dir = tmp_path / bracketed_dir
spec_dir.mkdir(parents=True)
@@ -49,3 +50,12 @@ class TestGraphRenderRefusedSpec:
assert exit_info.value.exit_code == 1
assert "Failed to render graph" in printed.getvalue()
assert str(spec_file) in printed.getvalue()
+ # The log line is a fixed message and carries the path and the diagnosis as written, with no escape:
+ # only the console print reads markup, and no sink reads a log message as markup.
+ log_spy.error.assert_called_once()
+ assert log_spy.error.call_args.args == ("The graph spec was refused, so no graph was rendered",)
+ fields = log_spy.error.call_args.kwargs["fields"]
+ assert fields["file.path"] == str(spec_file)
+ assert fields["error.type"] == "GraphSpecValidationError"
+ assert str(spec_file) in fields["error.message"]
+ assert "\\[" not in fields["error.message"]
diff --git a/tests/unit/pipelex/cli/test_run_core_wrapper.py b/tests/unit/pipelex/cli/test_run_core_wrapper.py
index bf5fac105..21c6aa744 100644
--- a/tests/unit/pipelex/cli/test_run_core_wrapper.py
+++ b/tests/unit/pipelex/cli/test_run_core_wrapper.py
@@ -11,13 +11,15 @@ import pytest
import typer
from rich.console import Console
-from pipelex.base_exceptions import PipelexError
+from pipelex.base_exceptions import PipelexError, PipelexSetupError
from pipelex.cli.commands.run._run_core import execute_run
from pipelex.cli.error_handlers import ErrorContext
+from pipelex.cogt.model_backends.backend_credentials import BackendCredentialsErrorMsgFactory
from pipelex.cogt.model_backends.model_type import ModelType
from pipelex.core.pipes.exceptions import PipeOperatorModelChoiceError
from pipelex.pipe_operators.exceptions import PipeOperatorModelAvailabilityError
from pipelex.system.pipe_run_mode import PipeRunMode
+from pipelex.tools.secrets.env_secrets_provider import EnvSecretsProvider
if TYPE_CHECKING:
from pytest_mock import MockerFixture
@@ -134,9 +136,17 @@ class TestExecuteRunWrapper:
assert exc_info.value.exit_code == 1
wrapper_mocks["teardown"].assert_called_once()
- def test_pipelex_error_prints_message_and_exits(self, wrapper_mocks: dict[str, Any]) -> None:
- """A PipelexError prints the friendly failure message and exits 1."""
- wrapper_mocks["core"].side_effect = PipelexError("the pipe blew up")
+ @pytest.mark.parametrize(
+ "message",
+ [
+ "the pipe blew up",
+ "Model 'gpt-5 → SDK[openai]•Backend[openai]•Model[gpt-5-2025]' does not support documents",
+ "Expected list[int], got [red]str[/red]",
+ ],
+ )
+ def test_pipelex_error_prints_message_and_exits(self, wrapper_mocks: dict[str, Any], message: str) -> None:
+ """A PipelexError prints the friendly failure message, its own message exactly as written, and exits 1."""
+ wrapper_mocks["core"].side_effect = PipelexError(message)
with pytest.raises(typer.Exit) as exc_info:
_call_execute_run()
@@ -144,7 +154,25 @@ class TestExecuteRunWrapper:
assert exc_info.value.exit_code == 1
output = wrapper_mocks["console"].export_text()
assert "Failed to execute pipeline" in output
- assert "the pipe blew up" in output
+ assert message in output
+
+ def test_a_missing_credential_message_prints_the_backend_table_as_written(self, wrapper_mocks: dict[str, Any]) -> None:
+ r"""The message names the table as `[openai]`; it used to carry a Rich escape, which the escape here printed as `\[openai]`."""
+ message = BackendCredentialsErrorMsgFactory.make_one_variable_missing_error_msg(
+ secrets_provider=EnvSecretsProvider(),
+ backend_name="openai",
+ var_name="OPENAI_API_KEY",
+ suggest_hosted_runs=True,
+ )
+ wrapper_mocks["core"].side_effect = PipelexSetupError(message)
+
+ with pytest.raises(typer.Exit) as exc_info:
+ _call_execute_run()
+
+ assert exc_info.value.exit_code == 1
+ output = wrapper_mocks["console"].export_text()
+ assert "Add 'enabled = false' under '[openai]' in '.pipelex/inference/backends.toml'" in output
+ assert "\\[" not in output
def test_unexpected_error_prints_exception_and_exits(self, wrapper_mocks: dict[str, Any]) -> None:
"""An unexpected exception prints the rich traceback and exits 1."""
diff --git a/tests/unit/pipelex/cogt/llm/test_llm_worker_constrained_job_params.py b/tests/unit/pipelex/cogt/llm/test_llm_worker_constrained_job_params.py
index ae26d8f53..0e1b274e0 100644
--- a/tests/unit/pipelex/cogt/llm/test_llm_worker_constrained_job_params.py
+++ b/tests/unit/pipelex/cogt/llm/test_llm_worker_constrained_job_params.py
@@ -1,4 +1,11 @@
-from typing import Any
+from __future__ import annotations
+
+import io
+import json
+import logging
+from typing import TYPE_CHECKING, Any
+
+from typing_extensions import override
from pipelex.cogt.llm.llm_job_components import LLMJobParams, ReasoningEffort
from pipelex.cogt.llm.llm_worker_abstract import LLMWorkerAbstract
@@ -7,6 +14,16 @@ from pipelex.cogt.model_backends.constraints import ListedConstraint, ValuedCons
from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
from pipelex.cogt.model_backends.model_type import ModelType
from pipelex.cogt.usage.cost_category import CostCategory
+from pipelex.tools.log.json_log_sink import MESSAGE_KEY, JsonLogSink
+
+if TYPE_CHECKING:
+ import pytest
+ from pytest_mock import MockerFixture
+
+ from pipelex.cogt.llm.llm_job import LLMJob
+ from pipelex.tools.typing.pydantic_utils import BaseModelTypeVar
+
+FIXED_TEMPERATURE_WARNING = "The model takes a fixed temperature, so it is used in place of the requested one"
def _make_model(
@@ -33,6 +50,18 @@ def _make_model(
)
+class _ProviderlessLLMWorker(LLMWorkerAbstract):
+ """A worker that never reaches a provider: only its handling of the model's constraints is exercised."""
+
+ @override
+ async def _gen_text(self, llm_job: LLMJob) -> str:
+ raise NotImplementedError
+
+ @override
+ async def _gen_object(self, llm_job: LLMJob, *, schema: type[BaseModelTypeVar]) -> BaseModelTypeVar:
+ raise NotImplementedError
+
+
class TestConstrainedJobParams:
def test_a_model_without_constraints_changes_nothing(self) -> None:
job_params = LLMJobParams(temperature=0.5, max_tokens=None)
@@ -60,3 +89,42 @@ class TestConstrainedJobParams:
def test_a_fixed_temperature_already_asked_for_changes_nothing(self) -> None:
inference_model = _make_model(thinking_mode=ThinkingMode.MANUAL, valued_constraints={ValuedConstraint.FIXED_TEMPERATURE: 1})
assert LLMWorkerAbstract.constrained_job_params(inference_model=inference_model, job_params=LLMJobParams(temperature=1)) is None
+
+ def test_a_replaced_temperature_is_warned_with_the_model_and_both_temperatures_as_fields(
+ self, mocker: MockerFixture, caplog: pytest.LogCaptureFixture
+ ) -> None:
+ """The warning names the model as its description used to, with the model keys meaning what they mean on the LLM span:
+ the handle requested under `gen_ai.request.model` and the provider's model id under `gen_ai.response.model`. The json sink
+ writes these OpenTelemetry keys, and the requested temperature's, as plain keys.
+ """
+ inference_model = _make_model(thinking_mode=ThinkingMode.MANUAL, valued_constraints={ValuedConstraint.FIXED_TEMPERATURE: 1})
+ worker = _ProviderlessLLMWorker(inference_model=inference_model)
+ llm_job = mocker.MagicMock(job_params=LLMJobParams(temperature=0.2))
+
+ with caplog.at_level(logging.WARNING, logger=LLMWorkerAbstract.__module__):
+ worker._apply_constraints(llm_job) # ruff: ignore[private-member-access] # pyright: ignore[reportPrivateUsage]
+
+ (record,) = [record for record in caplog.records if record.getMessage() == FIXED_TEMPERATURE_WARNING]
+ buffer = io.StringIO()
+ JsonLogSink(stream=buffer).handler.handle(record)
+ line: dict[str, Any] = json.loads(buffer.getvalue())
+ assert line[MESSAGE_KEY] == FIXED_TEMPERATURE_WARNING
+ field_names = (
+ "model_handle",
+ "backend_name",
+ "sdk",
+ "gen_ai.request.model",
+ "gen_ai.response.model",
+ "gen_ai.request.temperature",
+ "fixed_temperature",
+ )
+ assert {name: line[name] for name in field_names} == {
+ "model_handle": "gpt-test",
+ "backend_name": "openai",
+ "sdk": "openai",
+ "gen_ai.request.model": "gpt-test",
+ "gen_ai.response.model": "gpt-test-id",
+ "gen_ai.request.temperature": 0.2,
+ "fixed_temperature": 1,
+ }
+ assert "backend" not in line
diff --git a/tests/unit/pipelex/cogt/model_backends/test_backend_credentials.py b/tests/unit/pipelex/cogt/model_backends/test_backend_credentials.py
index 3eb451c59..99c4e27cc 100644
--- a/tests/unit/pipelex/cogt/model_backends/test_backend_credentials.py
+++ b/tests/unit/pipelex/cogt/model_backends/test_backend_credentials.py
@@ -62,7 +62,7 @@ class TestBackendCredentials:
" Add the variable to your environment or .env file:\n"
" - 'OPENAI_API_KEY'=<your_api_key>\n"
"\n2. Disable this backend\n"
- " Add 'enabled = false' under '\\[openai]' in '.pipelex/inference/backends.toml'\n" + BYOK_HINT
+ " Add 'enabled = false' under '[openai]' in '.pipelex/inference/backends.toml'\n" + BYOK_HINT
)
assert error_msg == expected_msg
@@ -81,7 +81,7 @@ class TestBackendCredentials:
"1. Provide the missing secret\n"
" Make sure 'ANTHROPIC_API_KEY' is available from your secrets provider.\n"
"\n2. Disable this backend\n"
- " Add 'enabled = false' under '\\[anthropic]' in '.pipelex/inference/backends.toml'\n" + BYOK_HINT
+ " Add 'enabled = false' under '[anthropic]' in '.pipelex/inference/backends.toml'\n" + BYOK_HINT
)
assert error_msg == expected_msg
assert ".env file" not in error_msg
@@ -103,7 +103,7 @@ class TestBackendCredentials:
" - 'OPENAI_API_KEY'=<your_api_key>\n"
"\n2. Disable unused backends\n"
" Disable backends you don't need in '.pipelex/inference/backends.toml':\n"
- " - Add 'enabled = false' under '\\[openai]'\n" + BYOK_HINT
+ " - Add 'enabled = false' under '[openai]'\n" + BYOK_HINT
)
assert error_msg == expected_msg
@@ -117,7 +117,7 @@ class TestBackendCredentials:
assert " • 'mistral': unresolved placeholders: 'MISTRAL_API_KEY'" in error_msg
assert " (Also replace placeholder values like '${VAR}' with actual keys)\n" in error_msg
assert "=<your_api_key>" not in error_msg
- assert " - Add 'enabled = false' under '\\[mistral]'\n" in error_msg
+ assert " - Add 'enabled = false' under '[mistral]'\n" in error_msg
def test_comprehensive_env_both_kinds_one_backend(self):
"""Missing and placeholder issues on one backend are joined with a semicolon on its detail line."""
@@ -146,8 +146,8 @@ class TestBackendCredentials:
openai_var_pos = error_msg.index(" - 'OPENAI_API_KEY'=<your_api_key>\n")
shared_var_pos = error_msg.index(" - 'SHARED_KEY'=<your_api_key>\n")
assert azure_var_pos < openai_var_pos < shared_var_pos
- assert " - Add 'enabled = false' under '\\[openai]'\n" in error_msg
- assert " - Add 'enabled = false' under '\\[azure]'\n" in error_msg
+ assert " - Add 'enabled = false' under '[openai]'\n" in error_msg
+ assert " - Add 'enabled = false' under '[azure]'\n" in error_msg
@pytest.mark.parametrize("provider_kind", ["none", "generic_mock"])
def test_comprehensive_non_env_provider(self, mocker: MockerFixture, provider_kind: str):
@@ -177,8 +177,8 @@ class TestBackendCredentials:
" - 'AWS_REGION'\n"
"\n2. Disable unused backends\n"
" Disable backends you don't need in '.pipelex/inference/backends.toml':\n"
- " - Add 'enabled = false' under '\\[anthropic]'\n"
- " - Add 'enabled = false' under '\\[bedrock]'\n" + BYOK_HINT
+ " - Add 'enabled = false' under '[anthropic]'\n"
+ " - Add 'enabled = false' under '[bedrock]'\n" + BYOK_HINT
)
assert error_msg == expected_msg
@@ -206,8 +206,27 @@ class TestBackendCredentials:
assert error_msg.endswith(BYOK_HINT + HOSTED_RUNS_HINT) is suggest_hosted_runs
assert ("--hosted" in error_msg) is suggest_hosted_runs
- def test_hosted_runs_hint_holds_no_square_bracket(self):
- """The hint is printed through Rich markup and as plain text, so it names the setting without a bracketed table."""
- assert "[" not in HOSTED_RUNS_HINT
+ def test_hosted_runs_hint_names_the_login_and_the_setting(self):
assert "pipelex login" in HOSTED_RUNS_HINT
assert 'execution = "hosted"' in HOSTED_RUNS_HINT
+
+ @pytest.mark.parametrize("secrets_provider_kind", ["env", "generic_mock"])
+ def test_every_message_names_the_table_as_written_with_no_markup_escape(self, mocker: MockerFixture, secrets_provider_kind: str):
+ r"""The messages are plain text: a Rich escape in them reached the user as `\[openai]` wherever nothing read them as markup."""
+ secrets_provider: SecretsProviderAbstract
+ if secrets_provider_kind == "env":
+ secrets_provider = EnvSecretsProvider()
+ else:
+ secrets_provider = mocker.MagicMock(spec=SecretsProviderAbstract)
+ reports = {"openai": make_report(backend_name="openai", missing_vars=["OPENAI_API_KEY"], placeholder_vars=[])}
+ messages = [
+ BackendCredentialsErrorMsgFactory.make_one_variable_missing_error_msg(
+ secrets_provider=secrets_provider, backend_name="openai", var_name="OPENAI_API_KEY", suggest_hosted_runs=True
+ ),
+ BackendCredentialsErrorMsgFactory.make_comprehensive_error_msg(
+ backend_credential_reports=reports, secrets_provider=secrets_provider, suggest_hosted_runs=True
+ ),
+ ]
+ for message in messages:
+ assert "under '[openai]'" in message
+ assert "\\" not in message
diff --git a/tests/unit/pipelex/cogt/model_backends/test_model_spec_description.py b/tests/unit/pipelex/cogt/model_backends/test_model_spec_description.py
new file mode 100644
index 000000000..23dadf75a
--- /dev/null
+++ b/tests/unit/pipelex/cogt/model_backends/test_model_spec_description.py
@@ -0,0 +1,33 @@
+"""A model spec's `tag` and `desc` are plain text: they name the SDK, the backend and the model id in brackets, never escaped for Rich.
+
+They reach error messages, which a log sink or an API response carries as written, so a backslash put there for a
+console reading markup would show up everywhere else.
+"""
+
+from pipelex.cogt.llm.thinking_mode import ThinkingMode
+from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
+from pipelex.cogt.model_backends.model_type import ModelType
+from pipelex.cogt.usage.cost_category import CostCategory
+
+
+def _make_spec() -> InferenceModelSpec:
+ return InferenceModelSpec(
+ backend_name="openai",
+ name="gpt-5",
+ sdk="openai_responses",
+ model_type=ModelType.LLM,
+ model_id="gpt-5-2025",
+ outputs=["text"],
+ costs={CostCategory.INPUT: 1.0, CostCategory.OUTPUT: 1.0},
+ thinking_mode=ThinkingMode.MANUAL,
+ max_tokens=None,
+ max_prompt_images=None,
+ )
+
+
+class TestModelSpecDescription:
+ def test_the_tag_and_the_description_are_plain_text(self) -> None:
+ spec = _make_spec()
+
+ assert spec.tag == "gpt-5 → [openai_responses@openai](gpt-5-2025)"
+ assert spec.desc == "gpt-5 → SDK[openai_responses]•Backend[openai]•Model[gpt-5-2025]"
diff --git a/tests/unit/pipelex/kernel/test_check_llm_setting_with_served_model.py b/tests/unit/pipelex/kernel/test_check_llm_setting_with_served_model.py
index 642945b3d..1ebcd6d24 100644
--- a/tests/unit/pipelex/kernel/test_check_llm_setting_with_served_model.py
+++ b/tests/unit/pipelex/kernel/test_check_llm_setting_with_served_model.py
@@ -16,6 +16,7 @@ from pipelex.cogt.usage.cost_category import CostCategory
from pipelex.kernel.llm_ops import check_llm_setting_with_served_model
from pipelex.plugins.inference_backend_registry import LLMRequestCheck
from pipelex.system.exceptions import MissingDependencyError
+from pipelex.system.telemetry.otel_constants import GenAISpanAttr
if TYPE_CHECKING:
from pytest_mock import MockerFixture
@@ -120,5 +121,16 @@ class TestCheckLLMSettingWithServedModel:
def test_a_backend_whose_sdk_is_missing_is_left_to_the_run(self, mocker: MockerFixture) -> None:
check = mocker.MagicMock(side_effect=MissingDependencyError("some-sdk", "some-extra"))
_serve(mocker, served=_make_model(thinking_mode=ThinkingMode.MANUAL), check=check)
+ log_spy = mocker.patch("pipelex.kernel.llm_ops.log")
check_llm_setting_with_served_model(llm_setting=_SETTING, is_structured=True)
check.assert_called_once()
+ log_spy.verbose.assert_called_once_with(
+ "The model's settings were not checked: its backend's SDK is not installed",
+ fields={
+ "model_handle": "some-model",
+ "backend_name": "some_backend",
+ "sdk": "some_sdk",
+ GenAISpanAttr.REQUEST_MODEL: "some-model",
+ GenAISpanAttr.RESPONSE_MODEL: "some-model-id",
+ },
+ )
diff --git a/tests/unit/pipelex/migration/test_markup_setting_retirement.py b/tests/unit/pipelex/migration/test_markup_setting_retirement.py
new file mode 100644
index 000000000..7f707980d
--- /dev/null
+++ b/tests/unit/pipelex/migration/test_markup_setting_retirement.py
@@ -0,0 +1,65 @@
+"""The entry that retires `is_markup_enabled`, exercised on the files it is about.
+
+The console reads no log message as Rich markup, so the `[runtime.log.rich_log]` key that switched it is gone and
+the entry deletes it, whichever value the file gave. It is `safe`, so it is also what a boot replays in memory over
+a file still setting the key: such a file boots with a stale-configuration warning rather than stopping the boot.
+"""
+
+from pathlib import Path
+from typing import Any
+
+import pytest
+from pytest_mock import MockerFixture
+
+from pipelex.migration.engine import replay_ledger_over_text
+from pipelex.migration.ledger import load_ledger, packaged_migration_dir
+from pipelex.system.configuration.config_loader import ConfigLoader
+from pipelex.system.configuration.config_surface import PIPELEX_CONFIG_SURFACE_ID
+from pipelex.system.configuration.configs import PipelexConfig
+from pipelex.tools.misc.toml_utils import load_toml_from_content
+
+ENTRY_ID = "pipelex-config@7"
+
+FILE_SETTING_MARKUP = """\
+[runtime.log.rich_log]
+is_show_time = true
+is_markup_enabled = false
+"""
+
+
+class TestTheMarkupSettingRetirement:
+ @pytest.mark.parametrize("value", ["true", "false"])
+ def test_the_key_is_deleted_and_its_neighbours_kept(self, value: str) -> None:
+ ledger = load_ledger(migration_dir=packaged_migration_dir(), surface_id=PIPELEX_CONFIG_SURFACE_ID)
+ replay = replay_ledger_over_text(ledger=ledger, text=f"[runtime.log.rich_log]\nis_show_time = true\nis_markup_enabled = {value}\n")
+
+ assert replay.blocked == []
+ assert [step.entry_id for step in replay.steps] == [ENTRY_ID]
+ migrated: dict[str, Any] = load_toml_from_content(replay.text)
+ assert migrated == {"runtime": {"log": {"rich_log": {"is_show_time": True}}}}
+
+ @pytest.mark.usefixtures("no_pipelex_home")
+ def test_a_project_file_setting_the_key_boots_with_a_warning(self, tmp_path: Path, mocker: MockerFixture) -> None:
+ """Through the boot's own loader: the file is migrated in memory, left as it is on disk, and the warning names the remedy."""
+ fake_home = tmp_path / "home"
+ (fake_home / ".pipelex").mkdir(parents=True)
+ project_root = tmp_path / "project"
+ (project_root / ".git").mkdir(parents=True)
+ project_dir = project_root / ".pipelex"
+ project_dir.mkdir()
+ project_file = project_dir / "pipelex.toml"
+ project_file.write_text(FILE_SETTING_MARKUP, encoding="utf-8")
+ mocker.patch.object(Path, "home", return_value=fake_home)
+ mocker.patch.object(Path, "cwd", return_value=project_root)
+ loader = ConfigLoader()
+
+ config = loader.load_config_validated(config_cls=PipelexConfig)
+
+ assert config.runtime.log.rich_log.is_show_time is True
+ assert "is_markup_enabled" not in type(config.runtime.log.rich_log).model_fields
+ assert project_file.read_text(encoding="utf-8") == FILE_SETTING_MARKUP, "a boot writes nothing"
+ parked = loader.take_stale_configuration_warning()
+ assert parked is not None
+ assert str(project_file) in parked
+ assert "The console reads no log message as markup" in parked
+ assert "Run `pipelex migrate`" in parked
diff --git a/tests/unit/pipelex/pipe_machinery/test_live_run_pipe_announcement.py b/tests/unit/pipelex/pipe_machinery/test_live_run_pipe_announcement.py
new file mode 100644
index 000000000..f107ff97b
--- /dev/null
+++ b/tests/unit/pipelex/pipe_machinery/test_live_run_pipe_announcement.py
@@ -0,0 +1,272 @@
+"""The line every live pipe run starts with: a fixed message, the pipe in its fields, and the console's pipe-run layout.
+
+The announcement used to be Rich markup assembled into the message, which every sink but the console wrote
+with its tags. It is now ``Pipe run starts`` with ``pipe_type``, ``pipe_code``, ``output_concept`` and
+``pipe_depth`` as fields, and the console draws the pipe tree from them through ``LogLayout.PIPE_RUN``.
+The reference for that drawing is the markup the message used to carry, rendered by a Rich handler that
+read it, so the console line is checked byte for byte, colour codes included. Only a live run announces
+itself, and ``live_run_pipe`` refuses a dry run mode.
+"""
+
+from __future__ import annotations
+
+import io
+import json
+import logging
+from typing import TYPE_CHECKING, Any
+
+import pytest
+from rich.console import Console
+from rich.highlighter import Highlighter, JSONHighlighter, ReprHighlighter
+from rich.logging import RichHandler
+from rich.text import Text
+from typing_extensions import override
+
+from pipelex.core.concepts.concept_factory import ConceptFactory
+from pipelex.core.concepts.native.concept_native import NativeConceptCode
+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.pipes.pipe_output import PipeOutput
+from pipelex.core.pipes.stuff_spec.stuff_spec import StuffSpec
+from pipelex.pipe_machinery.pipe_abstract import PIPE_RUN_STARTS_MESSAGE, PipeAbstract
+from pipelex.pipe_run.pipe_run_params import PipeRunParams
+from pipelex.system.console_target import ConsoleTarget
+from pipelex.system.job_metadata import JobMetadata, RunMetadata
+from pipelex.system.pipe_run_mode import PipeRunMode
+from pipelex.tools.log.console_layouts import LogLayout
+from pipelex.tools.log.console_log_sink import ConsoleLogSink
+from pipelex.tools.log.json_log_sink import MESSAGE_KEY, JsonLogSink
+from pipelex.tools.log.log_config import HighlighterName, RichLogConfig
+from pipelex.tools.log.log_fields import LAYOUT_MARK
+from pipelex.tools.log.log_formatter import EmojiLogFormatter
+from tests.helpers.console_log_rendering import package_log_config
+from tests.helpers.rich_markup import find_markup_tags
+
+if TYPE_CHECKING:
+ from pipelex.core.memory.working_memory import WorkingMemory
+ from pipelex.libraries.library_crate import LibraryCrate
+
+PIPE_CODE = "compose_company"
+CONSOLE_WIDTH = 120
+# Every field the announcement carries, all of which the console's pipe-run layout draws.
+ANNOUNCEMENT_FIELDS = ("pipe_type", "pipe_code", "output_concept", "pipe_depth")
+
+# The pipes a run is nested under, the announcing pipe last, as `run_pipe` leaves the stack when it calls
+# `live_run_pipe`, with the depth the announcement reports for each.
+PIPE_STACKS: list[tuple[list[str], int]] = [
+ ([], 0),
+ ([PIPE_CODE], 0),
+ (["build_profile", PIPE_CODE], 1),
+ (["run_batch", "build_profile", PIPE_CODE], 2),
+]
+PIPE_STACK_IDS = ["no stack at all", "a top-level run", "nested once", "nested twice"]
+
+
+class QuietPipe(PipeAbstract):
+ """A pipe whose live run does nothing but return, so the only line it writes is its announcement."""
+
+ pipe_category: Any = "PipeOperator"
+ type: Any = "PipeFunc"
+
+ @override
+ def validate_inputs_with_library(self) -> None: ...
+
+ @override
+ def validate_inputs_static(self) -> None: ...
+
+ @override
+ def validate_output_with_library(self) -> None: ...
+
+ @override
+ def validate_output_static(self) -> None: ...
+
+ @override
+ async def _validate_before_run(self, **kwargs: Any) -> None: ...
+
+ @override
+ async def _validate_after_run(self, **kwargs: Any) -> None: ...
+
+ @override
+ def required_variables(self) -> set[str]:
+ return set()
+
+ @override
+ def needed_inputs(self, *, visited_pipes: set[str] | None = None) -> Any:
+ return self.inputs
+
+ @override
+ async def _live_run_pipe(
+ self,
+ *,
+ job_metadata: JobMetadata,
+ working_memory: WorkingMemory,
+ pipe_run_params: PipeRunParams,
+ output_name: str | None = None,
+ library_crate: LibraryCrate | None = None,
+ ) -> PipeOutput:
+ return PipeOutput(working_memory=working_memory, pipeline_run_id=job_metadata.run_metadata.pipeline_run_id)
+
+ @override
+ async def _dry_run_pipe(
+ self,
+ *,
+ job_metadata: JobMetadata,
+ working_memory: WorkingMemory,
+ pipe_run_params: PipeRunParams,
+ output_name: str | None = None,
+ library_crate: LibraryCrate | None = None,
+ ) -> PipeOutput:
+ raise NotImplementedError
+
+
+def _make_pipe() -> QuietPipe:
+ return QuietPipe(
+ code=PIPE_CODE,
+ domain_code="test_announcement",
+ description="A pipe that announces itself",
+ output=StuffSpec(concept=ConceptFactory.make_native_concept(NativeConceptCode.TEXT)),
+ )
+
+
+def _job_metadata() -> JobMetadata:
+ return JobMetadata(run_metadata=RunMetadata(user_id="pytest", storage_scope="test/scope", read_scope=None, pipeline_run_id="plr-announce"))
+
+
+def _pipe_run_params(*, run_mode: PipeRunMode, pipe_stack: list[str]) -> PipeRunParams:
+ return PipeRunParams(run_mode=run_mode, batch_max_concurrency=1, pipe_stack_limit=10, pipe_stack=list(pipe_stack))
+
+
+async def _announcement(*, caplog: pytest.LogCaptureFixture, pipe_stack: list[str]) -> logging.LogRecord:
+ """The record the pipe's live run announced itself with, the pipe stack left as `run_pipe` leaves it."""
+ pipe = _make_pipe()
+ pipe_run_params = _pipe_run_params(run_mode=PipeRunMode.LIVE, pipe_stack=pipe_stack)
+ with caplog.at_level(logging.INFO, logger=PipeAbstract.__module__):
+ await pipe.live_run_pipe(job_metadata=_job_metadata(), working_memory=WorkingMemoryFactory.make_empty(), pipe_run_params=pipe_run_params)
+ (record,) = [record for record in caplog.records if record.name == PipeAbstract.__module__ and record.getMessage() == PIPE_RUN_STARTS_MESSAGE]
+ return record
+
+
+def _markup_announcement(*, pipe: PipeAbstract, pipe_stack: list[str]) -> str:
+ """The announcement as `PipeAbstract._format_pipe_run_info` wrote it, Rich markup in the message, for a live run."""
+ indent_level = len(pipe_stack) - 1
+ indent = " " * indent_level
+ if indent_level > 0:
+ indent = f"{indent}[yellow]↳[/yellow] "
+ pipe_type_label = f"[white]{pipe.pipe_type}:[/white]"
+ pipe_code_label = f"[red]{pipe.code}[/red]"
+ concept_code_label = f"[bold green]{pipe.output.concept.code}[/bold green]"
+ arrow = "[yellow]→[/yellow]"
+ return f"{indent}{pipe_type_label} {pipe_code_label} {arrow} {concept_code_label}"
+
+
+def _colour_console(*, buffer: io.StringIO) -> Console:
+ return Console(file=buffer, width=CONSOLE_WIDTH, force_terminal=True, color_system="truecolor", legacy_windows=False)
+
+
+def _console_rich_log_config() -> RichLogConfig:
+ """The shipped console settings with file links off: Rich mints a fresh random id for every link it prints."""
+ return package_log_config().rich_log.model_copy(update={"is_link_path_enabled": False})
+
+
+def _markup_reading_handler(*, config: RichLogConfig, buffer: io.StringIO) -> RichHandler:
+ """The console handler as it was when it read every message as markup, setting for setting."""
+ highlighter: Highlighter
+ match config.highlighter_name:
+ case HighlighterName.JSON:
+ highlighter = JSONHighlighter()
+ case HighlighterName.REPR:
+ highlighter = ReprHighlighter()
+ handler = RichHandler(
+ console=_colour_console(buffer=buffer),
+ show_time=config.is_show_time,
+ show_level=config.is_show_level,
+ enable_link_path=config.is_link_path_enabled,
+ highlighter=highlighter,
+ markup=True,
+ rich_tracebacks=config.is_rich_tracebacks,
+ tracebacks_word_wrap=config.is_tracebacks_word_wrap,
+ tracebacks_show_locals=config.is_tracebacks_show_locals,
+ tracebacks_suppress=config.tracebacks_suppress,
+ keywords=config.keywords_to_hilight,
+ )
+ handler.setFormatter(EmojiLogFormatter())
+ return handler
+
+
+class TestLiveRunPipeAnnouncement:
+ @pytest.mark.parametrize(("pipe_stack", "pipe_depth"), PIPE_STACKS, ids=PIPE_STACK_IDS)
+ async def test_the_announcement_is_a_fixed_message_with_the_pipe_in_its_fields(
+ self, caplog: pytest.LogCaptureFixture, pipe_stack: list[str], pipe_depth: int
+ ) -> None:
+ record = await _announcement(caplog=caplog, pipe_stack=pipe_stack)
+
+ assert find_markup_tags(text=record.getMessage()) == []
+ assert {name: getattr(record, name) for name in ANNOUNCEMENT_FIELDS} == {
+ "pipe_type": "QuietPipe",
+ "pipe_code": PIPE_CODE,
+ "output_concept": "Text",
+ "pipe_depth": pipe_depth,
+ }
+ assert not hasattr(record, "is_dry_run"), "only a live run announces itself, so the announcement carries no run mode"
+ assert record.__dict__[LAYOUT_MARK] == LogLayout.PIPE_RUN
+
+ @pytest.mark.parametrize(("pipe_stack", "pipe_depth"), PIPE_STACKS, ids=PIPE_STACK_IDS)
+ async def test_the_console_prints_the_line_the_markup_announcement_printed_byte_for_byte(
+ self, caplog: pytest.LogCaptureFixture, pipe_stack: list[str], pipe_depth: int
+ ) -> None:
+ """The indentation, the `↳`, `QuietPipe: compose_company → Text` and every colour code, as the markup message printed them."""
+ record = await _announcement(caplog=caplog, pipe_stack=pipe_stack)
+ config = _console_rich_log_config()
+ markup_record = logging.LogRecord(
+ name=record.name,
+ level=record.levelno,
+ pathname=record.pathname,
+ lineno=record.lineno,
+ msg=_markup_announcement(pipe=_make_pipe(), pipe_stack=pipe_stack),
+ args=(),
+ exc_info=None,
+ )
+ reference_buffer = io.StringIO()
+ _markup_reading_handler(config=config, buffer=reference_buffer).handle(markup_record)
+ buffer = io.StringIO()
+ handler = ConsoleLogSink(rich_log_config=config, target=ConsoleTarget.STDERR).handler
+ assert isinstance(handler, RichHandler)
+ handler.console = _colour_console(buffer=buffer)
+
+ handler.handle(record)
+
+ rendered = buffer.getvalue()
+ assert "\x1b[" in rendered, "the comparison is only worth something with the colour codes in it"
+ assert f"{' ' * pipe_depth}{'↳ ' if pipe_depth else ''}QuietPipe: {PIPE_CODE} → Text" in Text.from_ansi(rendered).plain
+ assert rendered == reference_buffer.getvalue()
+
+ async def test_the_json_sink_writes_the_plain_message_and_the_pipe_as_keys(self, caplog: pytest.LogCaptureFixture) -> None:
+ record = await _announcement(caplog=caplog, pipe_stack=["build_profile", PIPE_CODE])
+ buffer = io.StringIO()
+
+ JsonLogSink(stream=buffer).handler.handle(record)
+
+ line: dict[str, Any] = json.loads(buffer.getvalue())
+ assert line[MESSAGE_KEY] == PIPE_RUN_STARTS_MESSAGE
+ assert {name: line[name] for name in ANNOUNCEMENT_FIELDS} == {
+ "pipe_type": "QuietPipe",
+ "pipe_code": PIPE_CODE,
+ "output_concept": "Text",
+ "pipe_depth": 1,
+ }
+ assert "is_dry_run" not in line
+ assert LogLayout.PIPE_RUN not in line.values()
+
+ async def test_a_dry_run_mode_is_refused_before_anything_is_announced(self, caplog: pytest.LogCaptureFixture) -> None:
+ """Only `run_pipe`'s live case calls it, but it is public: a dry run handed to it would announce itself and open a span."""
+ pipe_run_params = _pipe_run_params(run_mode=PipeRunMode.DRY, pipe_stack=[PIPE_CODE])
+
+ with (
+ caplog.at_level(logging.INFO, logger=PipeAbstract.__module__),
+ pytest.raises(AssertionError, match=rf"^Live run of PipeFunc '{PIPE_CODE}' called with run_mode = dry$"),
+ ):
+ await _make_pipe().live_run_pipe(
+ job_metadata=_job_metadata(), working_memory=WorkingMemoryFactory.make_empty(), pipe_run_params=pipe_run_params
+ )
+
+ assert not [record for record in caplog.records if record.getMessage() == PIPE_RUN_STARTS_MESSAGE]
diff --git a/tests/unit/pipelex/pipe_operators/shared/test_operator_output_title.py b/tests/unit/pipelex/pipe_operators/shared/test_operator_output_title.py
new file mode 100644
index 000000000..65e6c707a
--- /dev/null
+++ b/tests/unit/pipelex/pipe_operators/shared/test_operator_output_title.py
@@ -0,0 +1,117 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+import pytest
+from rich.text import Text
+from typing_extensions import override
+
+from pipelex.core.concepts.concept_factory import ConceptFactory
+from pipelex.core.concepts.native.concept_native import NativeConceptCode
+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.pipes.pipe_output import PipeOutput
+from pipelex.core.pipes.stuff_spec.stuff_spec import StuffSpec
+from pipelex.core.stuffs.list_content import ListContent
+from pipelex.core.stuffs.stuff import Stuff
+from pipelex.core.stuffs.stuff_factory import StuffFactory
+from pipelex.core.stuffs.text_content import TextContent
+from pipelex.pipe_operators.pipe_operator import PipeOperator
+from pipelex.pipe_run.pipe_run_params import PipeRunParams
+from pipelex.system.job_metadata import JobMetadata, RunMetadata
+from pipelex.system.pipe_run_mode import PipeRunMode
+from pipelex.tools.misc.pretty import plain_markup_text
+
+if TYPE_CHECKING:
+ from pytest_mock import MockerFixture
+
+ from pipelex.core.memory.working_memory import WorkingMemory
+
+PIPE_CODE = "list_the_names"
+
+
+class ListingOperator(PipeOperator[PipeOutput]):
+ """An operator whose live run outputs a list of texts, as many as it is built with, and does nothing else."""
+
+ type: Any = "PipeFunc"
+ nb_items: int = 0
+
+ @override
+ def validate_inputs_with_library(self) -> None: ...
+
+ @override
+ def validate_inputs_static(self) -> None: ...
+
+ @override
+ def validate_output_with_library(self) -> None: ...
+
+ @override
+ def validate_output_static(self) -> None: ...
+
+ @override
+ async def _validate_before_run(self, **kwargs: Any) -> None: ...
+
+ @override
+ async def _validate_after_run(self, **kwargs: Any) -> None: ...
+
+ @override
+ def required_variables(self) -> set[str]:
+ return set()
+
+ @override
+ def needed_inputs(self, *, visited_pipes: set[str] | None = None) -> Any:
+ return self.inputs
+
+ @override
+ async def _live_run_operator_pipe(
+ self,
+ *,
+ job_metadata: JobMetadata,
+ working_memory: WorkingMemory,
+ pipe_run_params: PipeRunParams,
+ output_name: str | None = None,
+ ) -> PipeOutput:
+ items = [TextContent(text=f"name {index_item}") for index_item in range(self.nb_items)]
+ main_stuff = StuffFactory.make_stuff(
+ concept=ConceptFactory.make_native_concept(NativeConceptCode.TEXT),
+ content=ListContent[TextContent](items=items),
+ name="names",
+ )
+ return PipeOutput(
+ working_memory=WorkingMemoryFactory.make_from_single_stuff(main_stuff),
+ pipeline_run_id=job_metadata.run_metadata.pipeline_run_id,
+ )
+
+
+class TestOperatorOutputTitle:
+ @pytest.mark.parametrize(
+ ("nb_items", "multiplicity"),
+ [
+ pytest.param(0, "[empty list]", id="empty"),
+ pytest.param(1, "[1 item]", id="one"),
+ pytest.param(3, "[3 items]", id="several"),
+ ],
+ )
+ async def test_the_output_title_keeps_the_list_multiplicity(self, mocker: MockerFixture, nb_items: int, multiplicity: str) -> None:
+ """The bracketed multiplicity after the output concept reads as written in both the rich and the poor printer's reading."""
+ pretty_print_spy = mocker.patch.object(Stuff, "pretty_print_stuff", autospec=True)
+ pipe = ListingOperator(
+ code=PIPE_CODE,
+ domain_code="test_output_title",
+ description="An operator listing names",
+ output=StuffSpec(concept=ConceptFactory.make_native_concept(NativeConceptCode.TEXT)),
+ nb_items=nb_items,
+ )
+
+ await pipe.live_run_pipe(
+ job_metadata=JobMetadata(
+ run_metadata=RunMetadata(user_id="pytest", storage_scope="test/scope", read_scope=None, pipeline_run_id="plr-output-title")
+ ),
+ working_memory=WorkingMemoryFactory.make_empty(),
+ pipe_run_params=PipeRunParams(run_mode=PipeRunMode.LIVE, batch_max_concurrency=1, pipe_stack_limit=10, pipe_stack=[PIPE_CODE]),
+ )
+
+ title = pretty_print_spy.call_args.kwargs["title"]
+ expected = f"Output of pipe {PIPE_CODE} → Text {multiplicity}"
+ assert Text.from_markup(title).plain == expected
+ assert plain_markup_text(markup=title) == expected
diff --git a/tests/unit/pipelex/tools/test_log_console_fields.py b/tests/unit/pipelex/tools/test_log_console_fields.py
index 55b2992c3..10a97d1ff 100644
--- a/tests/unit/pipelex/tools/test_log_console_fields.py
+++ b/tests/unit/pipelex/tools/test_log_console_fields.py
@@ -11,13 +11,12 @@ from pipelex.tools.log.log_fields import (
COLLIDING_FIELD_PREFIX,
FIELD_NAMES_MARK,
RICH_HIGHLIGHTER_ATTRIBUTE,
- VERBATIM_MARK,
+ RICH_MARKUP_ATTRIBUTE,
attach_log_record_extra,
)
from tests.helpers.console_log_rendering import (
console_sink_on_buffer,
installed_log,
- package_log_config,
record_with_fields,
rendered_text,
styles_of,
@@ -63,7 +62,7 @@ class TestConsoleFields:
def test_a_field_named_like_a_record_attribute_or_a_mark_shows_under_the_name_it_landed_on(self, console_log: tuple[Log, io.StringIO]) -> None:
fresh, buffer = console_log
- fresh.info("Renamed", fields={"name": "alpha", VERBATIM_MARK: True})
+ fresh.info("Renamed", fields={"name": "alpha", RICH_MARKUP_ATTRIBUTE: True})
assert "Renamed field_name=alpha field_markup=true" in buffer.getvalue()
@@ -120,15 +119,12 @@ class TestConsoleFields:
assert "\x1b" not in rendered
assert not any(line.lstrip().startswith("FORGED") for line in rendered.splitlines())
- def test_a_value_carrying_markup_prints_as_written_while_the_message_still_reads_markup(self, console_log: tuple[Log, io.StringIO]) -> None:
- assert package_log_config().rich_log.is_markup_enabled, "the suffix must hold whatever the setting it runs beside"
+ def test_neither_the_message_nor_a_value_is_read_as_markup(self, console_log: tuple[Log, io.StringIO]) -> None:
fresh, buffer = console_log
fresh.info("[bold]Received[/bold]", fields={"detail": "[red]x[/red]"})
- rendered = buffer.getvalue()
- assert "Received detail=[red]x[/red]" in rendered
- assert "[bold]" not in rendered
+ assert "[bold]Received[/bold] detail=[red]x[/red]" in buffer.getvalue()
def test_a_long_value_is_cut_short(self, console_log: tuple[Log, io.StringIO]) -> None:
fresh, buffer = console_log
@@ -170,7 +166,7 @@ class TestConsoleFields:
def test_what_a_record_factory_or_the_runtime_stamped_is_not_shown(self) -> None:
record = record_with_fields(message="Stamped", extra={"files": 7})
record.otelSpanID = "0"
- record.__dict__[VERBATIM_MARK] = False
+ record.__dict__[RICH_MARKUP_ATTRIBUTE] = False
text = rendered_text(record=record)
diff --git a/tests/unit/pipelex/tools/test_log_console_layouts.py b/tests/unit/pipelex/tools/test_log_console_layouts.py
index c4584a862..33da4674e 100644
--- a/tests/unit/pipelex/tools/test_log_console_layouts.py
+++ b/tests/unit/pipelex/tools/test_log_console_layouts.py
@@ -31,9 +31,9 @@ class TestConsoleLayouts:
[
({}, "PipeCompose: compose_company → Company"),
({"pipe_depth": 1}, " ↳ PipeCompose: compose_company → Company"),
- ({"pipe_depth": 2, "is_dry_run": True}, " ↳ Dry run: PipeCompose: compose_company → Company"),
+ ({"pipe_depth": 2}, " ↳ PipeCompose: compose_company → Company"),
],
- ids=["top level", "nested", "nested dry run"],
+ ids=["top level", "nested", "nested twice"],
)
def test_the_pipe_run_layout_draws_the_tree_in_the_style_map_colours(self, overrides: dict[str, Any], expected: str) -> None:
text = rendered_text(record=record_with_fields(message="Pipe run starts", extra={**PIPE_RUN_FIELDS, **overrides}, layout=LogLayout.PIPE_RUN))
@@ -68,13 +68,11 @@ class TestConsoleLayouts:
@pytest.mark.parametrize(
("overrides", "expected_values"),
[
- ({"pipe_depth": "deep"}, "pipe_depth=deep is_dry_run=false"),
- ({"pipe_depth": 10**20}, "pipe_depth=100000000000000000000 is_dry_run=false"),
- ({"pipe_depth": PIPE_RUN_MAX_DEPTH + 1}, f"pipe_depth={PIPE_RUN_MAX_DEPTH + 1} is_dry_run=false"),
- ({"pipe_depth": -1}, "pipe_depth=-1 is_dry_run=false"),
- ({"pipe_depth": True}, "pipe_depth=true is_dry_run=false"),
- ({"is_dry_run": "false"}, "pipe_depth=0 is_dry_run=false"),
- ({"is_dry_run": 1}, "pipe_depth=0 is_dry_run=1"),
+ ({"pipe_depth": "deep"}, "pipe_depth=deep"),
+ ({"pipe_depth": 10**20}, "pipe_depth=100000000000000000000"),
+ ({"pipe_depth": PIPE_RUN_MAX_DEPTH + 1}, f"pipe_depth={PIPE_RUN_MAX_DEPTH + 1}"),
+ ({"pipe_depth": -1}, "pipe_depth=-1"),
+ ({"pipe_depth": True}, "pipe_depth=true"),
],
ids=[
"a depth that is no integer",
@@ -82,17 +80,12 @@ class TestConsoleLayouts:
"a depth past the bound",
"a negative depth",
"a boolean depth",
- "a dry-run flag spelled as a string",
- "a dry-run flag given as an integer",
],
)
def test_a_value_the_pipe_run_derivation_refuses_falls_back_to_the_message_and_every_field(
self, overrides: dict[str, Any], expected_values: str
) -> None:
- """A depth used to be repeated as given, so a huge one raised ``OverflowError`` past the fallback and lost the line, or built a huge string.
-
- The flag used to be judged by truthiness, so ``"false"`` labelled a live run ``Dry run:``.
- """
+ """A depth was repeated as given: a huge one raised ``OverflowError`` past the fallback and lost the line, or built a huge string."""
record = record_with_fields(message="Pipe run starts", extra={**PIPE_RUN_FIELDS, **overrides}, layout=LogLayout.PIPE_RUN)
text = rendered_text(record=record)
@@ -122,7 +115,7 @@ class TestConsoleLayouts:
rendered = buffer.getvalue()
assert "Pipe run starts:" in rendered
assert '"the_content_key": 1' in rendered
- assert "} pipe_type=PipeCompose pipe_code=compose_company output_concept=Company pipe_depth=0 is_dry_run=false attempt=2" in rendered
+ assert "} pipe_type=PipeCompose pipe_code=compose_company output_concept=Company pipe_depth=0 attempt=2" in rendered
assert "→" not in rendered
def test_a_content_json_refuses_keeps_its_message_rather_than_a_layout(self, caplog: pytest.LogCaptureFixture) -> None:
diff --git a/tests/unit/pipelex/tools/test_log_console_sink.py b/tests/unit/pipelex/tools/test_log_console_sink.py
index efc9a7f45..aaaeea273 100644
--- a/tests/unit/pipelex/tools/test_log_console_sink.py
+++ b/tests/unit/pipelex/tools/test_log_console_sink.py
@@ -1,9 +1,10 @@
-"""The ``console`` sink renders byte for byte what the Rich handler rendered before it existed, plus the fields.
+"""The ``console`` sink renders byte for byte what a plain Rich handler renders, plus the fields.
-The reference is the handler ``log.configure`` used to build inline: a ``RichHandler`` fed every
-``[runtime.log.rich_log]`` setting and the emoji formatter. Both handlers are pointed at the same kind
-of non-terminal console and handed the same fixed record set. A record whose call gave no field renders
-identically through both; a record with fields renders the same line with the fields after the message.
+The reference is the handler ``log.configure`` used to build inline before the sink existed, reading no
+message as markup as the console now does: a ``RichHandler`` fed every ``[runtime.log.rich_log]`` setting,
+``markup=False`` and the emoji formatter. Both handlers are pointed at the same kind of non-terminal console
+and handed the same fixed record set. A record whose call gave no field renders identically through both; a
+record with fields renders the same line with the fields after the message.
"""
from __future__ import annotations
@@ -22,8 +23,9 @@ from pipelex.system.console_target import ConsoleTarget
from pipelex.tools.log.console_layouts import LogLayout
from pipelex.tools.log.console_log_sink import ConsoleLogSink
from pipelex.tools.log.log_config import HighlighterName, LogConfig, RichLogConfig
-from pipelex.tools.log.log_fields import VERBATIM_MARK, attach_log_record_extra
+from pipelex.tools.log.log_fields import RICH_MARKUP_ATTRIBUTE, attach_log_record_extra
from pipelex.tools.log.log_formatter import EmojiLogFormatter
+from pipelex.tools.log.log_redaction import CYCLE_TEXT
from pipelex.tools.misc.toml_utils import load_toml_from_path
from tests.helpers.console_log_rendering import (
PIPE_RUN_FIELDS,
@@ -41,7 +43,7 @@ def _package_rich_log_config() -> RichLogConfig:
def _reference_handler(*, config: RichLogConfig) -> logging.Handler:
- """The handler ``log.configure`` built before the sink seam, setting for setting."""
+ """The handler ``log.configure`` built before the sink seam, setting for setting, reading no message as markup."""
highlighter: Highlighter
match config.highlighter_name:
case HighlighterName.JSON:
@@ -54,7 +56,7 @@ def _reference_handler(*, config: RichLogConfig) -> logging.Handler:
show_level=config.is_show_level,
enable_link_path=config.is_link_path_enabled,
highlighter=highlighter,
- markup=config.is_markup_enabled,
+ markup=False,
rich_tracebacks=config.is_rich_tracebacks,
tracebacks_word_wrap=config.is_tracebacks_word_wrap,
tracebacks_show_locals=config.is_tracebacks_show_locals,
@@ -135,6 +137,7 @@ class TestConsoleLogSink:
assert rendered
assert "🧠: Running the pipe" in rendered
+ assert "[myapp.jobs.nightly]: A logger with no emoji" in rendered
assert "ValueError" in rendered
assert rendered == reference
@@ -234,48 +237,62 @@ class TestConsoleLogSink:
"message",
[
"expected list[int], got str",
+ "Expected list[int], got [red]str[/red]",
"no such file: [/etc/pipelex.toml]",
+ "[bold]not a style[/bold] :fire:",
],
- ids=["a type complaint", "a bracketed path"],
+ ids=["a type complaint", "a type complaint carrying a style tag", "a bracketed path", "a style tag and an emoji code"],
)
- def test_a_message_shaped_like_markup_renders_intact_only_when_the_record_asks_for_verbatim(self, message: str) -> None:
- """An error message carries text nobody chose, and Rich reads a tag-shaped span in it as markup.
-
- Rich's tag has to start with a lowercase letter, `#`, `/` or `@`, so `[Errno 2]` is safe and the
- two shapes this codebase's messages are actually made of are not: `list[int]` loses the bracketed
- span, and a bracketed path opens what Rich reads as a closing tag and raises inside the handler,
- where `handleError` costs the whole line. The configuration asks for markup and Pipelex's own
- lines use it, so the opt-out is per record: Rich reads `markup` off the record ahead of its
- handler's setting, which is what `TracebackMessageError` stamps through `VERBATIM_MARK`.
+ def test_a_message_shaped_like_markup_prints_exactly_as_written(self, message: str) -> None:
+ """An error message carries text nobody chose, and Rich read a tag-shaped span in it as markup.
+
+ Rich's tag starts with a lowercase letter, `#`, `/` or `@`, so `list[int]` lost its bracketed span, a
+ style tag coloured what it wrapped and vanished, and a bracketed path opened what Rich reads as a
+ closing tag and raised inside the handler, costing the whole line. The console reads no message as
+ markup, so each prints as written, an emoji code included.
"""
- config = _package_rich_log_config()
- assert config.is_markup_enabled, "the mark is an opt-out, so an enabled setting is what it opts out of"
+ rendered = _render_one(config=_package_rich_log_config(), message=message, extra=None)
- interpreted = _render_one(config=config, message=message, extra=None)
- verbatim = _render_one(config=config, message=message, extra={VERBATIM_MARK: False})
+ assert message in rendered
- assert message not in interpreted
- assert message in verbatim
+ def test_a_logger_named_in_lowercase_keeps_its_channel_prefix(self, caplog: pytest.LogCaptureFixture) -> None:
+ """A logger with no emoji is prefixed with its name in brackets, which Rich read as a tag and removed when the name starts lowercase.
- @pytest.mark.parametrize(
- "message",
- [
- "expected list[int], got str",
- "no such file: [/etc/pipelex.toml]",
- ],
- ids=["a type complaint", "a bracketed path"],
- )
- def test_a_message_rich_refuses_costs_its_rendering_and_never_the_log_call(self, message: str) -> None:
+ The line printed as ``: Pipe run failed``, with nothing saying which library wrote it.
+ """
+ caplog.set_level(logging.INFO, logger="myapp.jobs.nightly")
+ buffer = io.StringIO()
+ with installed_log(sink=console_sink_on_buffer(buffer=buffer)):
+ logging.getLogger("myapp.jobs.nightly").error("Pipe run failed")
+
+ assert "[myapp.jobs.nightly]: Pipe run failed" in buffer.getvalue()
+
+ def test_a_circular_content_prints_its_cycle_marker(self, caplog: pytest.LogCaptureFixture) -> None:
+ """A content JSON refuses is rendered as its ``repr``, where the redaction cuts the cycle with ``[cycle]``.
+
+ Read as markup, the marker was a tag Rich removed, so the line printed ``'self': ''``.
+ """
+ caplog.set_level(logging.INFO, logger=__name__)
+ circular: dict[str, Any] = {"the_content_key": 1}
+ circular["self"] = circular
+ buffer = io.StringIO()
+ with installed_log(sink=console_sink_on_buffer(buffer=buffer)) as fresh:
+ fresh.info(circular)
+
+ assert f"{{'the_content_key': 1, 'self': '{CYCLE_TEXT}'}}" in buffer.getvalue()
+
+ def test_a_line_rich_refuses_costs_its_rendering_and_never_the_log_call(self) -> None:
"""Rich overrides ``emit`` without the stdlib's ``handleError`` guard, so a refusal left the log call.
- A bracketed path reads as a closing tag with nothing open and Rich raises ``MarkupError`` from
- inside the handler: unguarded, that propagated out of `log.error` and replaced the failure being
- reported with itself. The sink's handler restores the guard, so the worst a line Rich refuses costs
- is its own rendering.
+ No Pipelex message is read as markup, but Rich still honours its own per-record ``markup`` attribute,
+ which a third-party library may stamp on its record: a bracketed path in such a line reads as a closing
+ tag with nothing open, and Rich raises ``MarkupError`` from inside the handler. Unguarded, that
+ propagated out of the call that logged it and replaced whatever was being reported with itself. The
+ sink's handler restores the guard, so the worst a line Rich refuses costs is its own rendering.
"""
- config = _package_rich_log_config()
+ message = "no such file: [/etc/pipelex.toml]"
- rendered = _render_one(config=config, message=message, extra=None)
+ rendered = _render_one(config=_package_rich_log_config(), message=message, extra={RICH_MARKUP_ATTRIBUTE: True})
assert message not in rendered
diff --git a/tests/unit/pipelex/tools/test_log_console_sink_without_rich.py b/tests/unit/pipelex/tools/test_log_console_sink_without_rich.py
index bd7f3f3e4..543ee8f4f 100644
--- a/tests/unit/pipelex/tools/test_log_console_sink_without_rich.py
+++ b/tests/unit/pipelex/tools/test_log_console_sink_without_rich.py
@@ -36,7 +36,6 @@ _GUARD_SCRIPT = textwrap.dedent(
is_show_level=True,
is_link_path_enabled=True,
highlighter_name=HighlighterName.JSON,
- is_markup_enabled=True,
is_rich_tracebacks=True,
is_tracebacks_word_wrap=True,
is_tracebacks_show_locals=False,
diff --git a/tests/unit/pipelex/tools/test_log_fields.py b/tests/unit/pipelex/tools/test_log_fields.py
index d258574f3..80a2831a0 100644
--- a/tests/unit/pipelex/tools/test_log_fields.py
+++ b/tests/unit/pipelex/tools/test_log_fields.py
@@ -23,7 +23,7 @@ from pipelex.tools.log.log_fields import (
FORWARDED_MARK,
LAYOUT_MARK,
RICH_HIGHLIGHTER_ATTRIBUTE,
- VERBATIM_MARK,
+ RICH_MARKUP_ATTRIBUTE,
attached_field_names,
carried_attributes,
)
@@ -255,12 +255,12 @@ class TestLogFields:
assert FORWARDED_MARK not in carried
assert carried[f"{COLLIDING_FIELD_PREFIX}{FORWARDED_MARK}"] is True
- @pytest.mark.parametrize("name", [VERBATIM_MARK, RICH_HIGHLIGHTER_ATTRIBUTE])
+ @pytest.mark.parametrize("name", [RICH_MARKUP_ATTRIBUTE, RICH_HIGHLIGHTER_ATTRIBUTE])
def test_a_field_named_like_a_rich_per_record_override_is_prefixed_and_reaches_no_sink(self, caplog: pytest.LogCaptureFixture, name: str) -> None:
"""``markup`` and ``highlighter`` are what Rich's console handler reads off a record ahead of its own settings.
- Left unreserved, a caller naming ``markup`` would decide whether the console interprets markup for
- that line, and one naming ``highlighter`` would have its value called as the highlighter, which
+ Left unreserved, a caller naming ``markup`` would turn markup back on for that line, which the
+ console otherwise never reads, and one naming ``highlighter`` would have its value called as the highlighter, which
raises for a string and loses the line. Either value would also ride into the `json` and `otlp`
payloads as a field, where it means nothing. Reserved, the caller's value is carried under the
prefix and is not a carried attribute.
@@ -276,25 +276,19 @@ class TestLogFields:
assert carried[f"{COLLIDING_FIELD_PREFIX}{name}"] is True
assert carried["safe"] == 1
- def test_what_a_third_party_stamped_as_rich_s_highlighter_is_no_carried_field(self, caplog: pytest.LogCaptureFixture) -> None:
- """A library steering the console through ``extra`` puts an object there that means nothing on a wire.
+ @pytest.mark.parametrize(("name", "value"), [(RICH_HIGHLIGHTER_ATTRIBUTE, object()), (RICH_MARKUP_ATTRIBUTE, True)])
+ def test_what_a_third_party_stamped_as_a_rich_per_record_override_is_no_carried_field(
+ self, caplog: pytest.LogCaptureFixture, name: str, value: object
+ ) -> None:
+ """A library steering the console through ``extra`` puts a value there that means nothing on a wire.
- Carried, the scrub would also turn that object into text, which Rich would then fail to call.
+ Carried, the scrub would also turn a highlighter object into text, which Rich would then fail to call.
"""
with caplog.at_level(logging.INFO):
log.info("a line", fields={"safe": 1})
(record,) = _own_records(caplog)
- setattr(record, RICH_HIGHLIGHTER_ATTRIBUTE, object())
- assert carried_attributes(record=record) == {"safe": 1}
-
- def test_a_record_the_error_path_stamped_verbatim_hands_no_such_field_to_a_sink(self, caplog: pytest.LogCaptureFixture) -> None:
- """`TracebackMessageError` stamps the mark itself, and a structured sink must not write it as a field."""
- with caplog.at_level(logging.INFO):
- log.info("a line", fields={"safe": 1})
-
- (record,) = _own_records(caplog)
- setattr(record, VERBATIM_MARK, False)
+ setattr(record, name, value)
assert carried_attributes(record=record) == {"safe": 1}
def test_the_names_the_call_attached_are_recorded_in_order_and_are_no_field(self, caplog: pytest.LogCaptureFixture) -> None: