Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
feature/Decisions-at-decision-time · L-260927-638e5f (#1427)
Louis Choquel1f2d210e82fc
diff --git a/.drift/acks/hub-layering-convention.toml b/.drift/acks/hub-layering-convention.tomlindex e6fdf4150..a5ac63e3b 100644--- a/.drift/acks/hub-layering-convention.toml+++ b/.drift/acks/hub-layering-convention.toml@@ -1,10 +1,10 @@contract = "hub-layering-convention"-digest = "sha256:a936b7ba24211f227bcc1b60046075b56522c24b7f6de366ee2fa091ce821267"+digest = "sha256:558d2a12672e159edf2827791bcb5370781df2b576b958e0ea48989833087f8b"reviewed_by = "Louis Choquel"-reviewed_at = "2026-10-12T02:37:48Z"-rationale = "real-catch: interpreter_hub.py gains the module-level does_pipe_run_leave_process (and InterpreterHub.set_pipe_run takes leaves_process, stored beside the pipe run), so the run row of the partition table now names it. No import was added to either hub, KERNEL_LAYER_PACKAGES is unchanged, and every public module-level function and class of both hubs is in the tables."+reviewed_at = "2026-10-13T02:26:31Z"+rationale = "real-catch: runtime_hub.py gains the module-level resolve_effective_run_mode, moved from the pipe tier so the kernel layer can reach it; the run-mode row of the partition table now names it and the resolve_run_mode_for_boot paragraph says why it sits beside it. No import crosses the boundary: runtime_hub imports cogt_run_params, which is kernel-layer, and KERNEL_LAYER_PACKAGES is unchanged."[trigger_files]"pipelex/cli/dev_cli/commands/hub_layering_guard.py" = "blob:fda2237457fe0b6830ff2ae8f720698735bec4c3""pipelex/interpreter_hub.py" = "blob:023f8452daf76dde409afc7c0966b149b11edabb"-"pipelex/runtime_hub.py" = "blob:65beba670c857897858b75fd17bbbc03ba84e750"+"pipelex/runtime_hub.py" = "blob:8269c5d9d549dd06eb8329e3763efed403f95fed"diff --git a/.drift/acks/pipelex-kernel-docs.toml b/.drift/acks/pipelex-kernel-docs.tomlindex 3b94fed28..bc914050b 100644--- a/.drift/acks/pipelex-kernel-docs.toml+++ b/.drift/acks/pipelex-kernel-docs.toml@@ -1,14 +1,14 @@contract = "pipelex-kernel-docs"-digest = "sha256:f6449898b2c6db663c5a11f287652c487620dd24dec35654a0883fb62c2fd91d"+digest = "sha256:1ed250b7f7895e5a94b70b4329c99cef9f1f2c41803c69494deeacbce044392c"reviewed_by = "Louis Choquel"-reviewed_at = "2026-10-10T20:46:23Z"-rationale = "real-catch: run_multi_judgment now hands the verdicts over by field name, so an aliased field keeps its verdict, and decides a refusal by class_field_may_hold_nothing, the predicate the load's concept walk reads a class field by, storing None outright when the field's type admits it. pipelex-kernel.md said a refused field was left absent when neither required nor defaulted; the judgment paragraph now states the shared rule and the by-name handover, and gains the PipeJudge execution-data paragraph moved out of the building-methods page."+reviewed_at = "2026-10-13T02:26:32Z"+rationale = "real-catch: PipelexKernel.make now resolves its mode through runtime_hub.resolve_effective_run_mode and refuses a trace_context stating another mode with the new TraceContextRunModeError in kernel/exceptions.py. pipelex-kernel.md's usage example built a TraceContext without the now-required run_mode; it now states the effective mode on the context and on make, a paragraph explains the refusal and why the kernel refuses rather than corrects its copy, and the keyless-boot sentence names resolve_effective_run_mode."[trigger_files]"pipelex/kernel/__init__.py" = "blob:c4f1b0fe2ccc17db9110733f71b612f4ac1ba4ce""pipelex/kernel/compose_ops.py" = "blob:ccd8ba141246380eb5e9d55e693fa6fedf38233a""pipelex/kernel/compose_results.py" = "blob:a837ae9680c7d4d0eb4dee37d4657c410e5b5cea"-"pipelex/kernel/exceptions.py" = "blob:e6c9fc0ddae10e533f8aaf2fd0f8afcb46881461"+"pipelex/kernel/exceptions.py" = "blob:ec784b58d732cca19b9278c83afd009422792e81""pipelex/kernel/extract_ops.py" = "blob:19d556f0254017d7d52c30f4f1a46e2235289042""pipelex/kernel/extract_results.py" = "blob:79405a56760f3d4b483660d76441fff95f58c64d""pipelex/kernel/func_ops.py" = "blob:97c68c977ddc260c9b36b7eda5f4c7964e956a27"@@ -22,7 +22,7 @@ rationale = "real-catch: run_multi_judgment now hands the verdicts over by field"pipelex/kernel/llm_prompt_content.py" = "blob:f996cead6e7789494f01f69c128ae71d6dcf8466""pipelex/kernel/llm_results.py" = "blob:6e3b8a2878b5bb284ece5fac10262fb59ac2029d""pipelex/kernel/memory_ops.py" = "blob:7fbbfaaec99f9712d8e215520cb02e26aad3fb40"-"pipelex/kernel/pipelex_kernel.py" = "blob:5dec4e68d517d5ab1d7e9cbb91d836f37b10e713"+"pipelex/kernel/pipelex_kernel.py" = "blob:a923ec3c37769d0eac1b37be075971464c27c06c""pipelex/kernel/prompt_assembly.py" = "blob:7559725bc0d813cd136bf258b8178d96a8f49194""pipelex/kernel/prompt_references.py" = "blob:a4a8c5535c7e51a16ac6c8d1296439b331192a7f""pipelex/kernel/search_ops.py" = "blob:b7de7ff62c45dde7f7aa8a9f409aecf3b664cce5"diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 89e50d6d6..de86ddb9d 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -2,6 +2,16 @@## [Unreleased]+### Added++- **`condition_resolved` and `batch_fanned_out` trace events**: a live `PipeCondition` writes `condition_resolved` the moment its choice is final, before anything runs under it, with the `rendered_value` its expression rendered and the `selected_outcome` key of the outcome map it matched, `null` when it took the default outcome, and a chosen pipe whose inputs are missing fails the condition with no outcome recorded; both graph builders derive from that event and the start of the chosen pipe the `selected_outcome` edge from the condition, labelled with that key, which no run drew before, whichever workflow the chosen pipe started in. A `PipeBatch` writes `batch_fanned_out` once it knows its width, before any item starts, with `item_count`, `branch_pipe_code`, the bare code of the pipe each item runs, `branch_pipe_ref`, its qualified reference, and `branch_pipe_type`. A dry run resolves no condition, so it writes no `condition_resolved`, and both graph builders build the same graph from a run's events.+- **`step_count` and `branch_count` on `pipe_start`**: a `PipeSequence`'s start event states the number of steps its author wrote, the binding of a dotted `batch_over` not among them, and a `PipeParallel`'s the number of its branches; both are `null` on every other pipe.++### Changed++- **Every trace event states its `mode` (Breaking)**: each trace event carries `mode`, `live` or `dry`, the mode the run's pipes run in, which the run's `TraceContext` carries as the new `run_mode`, so a usage event an activity writes states it too, and a live run a keyless boot (`needs_inference=False`) forces dry states `dry` on its trace context, its events and its graph, and, like every dry run, opens no OpenTelemetry trace; an event stored without `mode` by an earlier version is refused with `EventLogSchemaMismatchError` rather than read as live, and a consumer that builds trace events passes it. Callers follow new signatures: `TraceContext.run_mode` is required, with no default, and so is `run_mode` on `GraphTracerManager.open_tracer` and on the tracers' `setup` (`GraphTracerProtocol`, `GraphTracer` and `GraphTracerNoOp`), where it replaces `mode: GraphSpecMode`, so a workflow opening a tracer of its own for part of a run passes the `run_mode` of the context it received, and `PipelexKernel.make` refuses, with `TraceContextRunModeError`, a `trace_context` stating another mode than the one its run takes, which the new `runtime_hub.resolve_effective_run_mode` gives, dry on a keyless boot. `GraphSpecAssembler.assemble` no longer takes `mode` and raises the new `TraceEventModeError` on a set of events that mixes modes or holds none, and `assemble_tracing` and `assemble_tracing_on_output` no longer take `run_mode`: the graph states the mode its events state, so a dry run assembled from its events in a durable workflow gets a dry graph where it got a live one, and `assemble_tracing` reports a `TraceEventModeError` as the run's `graph_assembly_error`.+- **Controllers register their execution data as they decide (Breaking)**: a `PipeSequence` registers `step_count` as it starts, now the number of steps its author wrote, and the new `current_step` as each step starts; a `PipeParallel` registers its `branch_count`, `add_each_output` and `combined_output_concept` before its branches run, a `PipeBatch` its `item_count` and `branch_pipe_code` before its items run, and a live `PipeCondition` its `evaluated_expression` and `selected_outcome` once its choice is final, before the chosen pipe runs, where each used to register them once it had finished. `GraphTracerProtocol` adds `register_batch_fan_out` and `register_condition_resolution`, and `add_selected_outcome_edge` takes `selected_outcome: str | None` in place of `outcome_value`.+### Fixed- **Trace events written to DynamoDB expire (Breaking)**: the DynamoDB event log now writes every event with a `ttl` in epoch seconds, its write time plus the new `runtime.tracing.dynamodb.retention_days` (30 by default, at least 1), so a table that expires items on `ttl` deletes a run's events, full output data included, where it used to keep them forever. Only events written from this version on carry a `ttl`, so the rows an earlier version wrote stay until they are deleted or given a `ttl` by hand. `DynamoDBEventLog` takes `retention_days` and now takes all its arguments by keyword; `make_event_log` reads the setting from the configuration.diff --git a/docs/contribute/hub-layering.md b/docs/contribute/hub-layering.mdindex c305e09cf..31addb953 100644--- a/docs/contribute/hub-layering.md+++ b/docs/contribute/hub-layering.md@@ -45,7 +45,7 @@ There is deliberately **no** `pipelex.hub`. It was deleted rather than kept as a| inference | `get_inference_manager`, `get_llm_worker`, `get_img_gen_worker`, `get_extract_worker` || content generation | `get_content_generator`, `scoped_content_generator` || reporting | `get_report_delegate`, `is_in_isolated_execution` |-| run mode | `is_dry_run_forced`, `resolve_run_mode_for_boot` |+| run mode | `is_dry_run_forced`, `resolve_run_mode_for_boot`, `resolve_effective_run_mode` || boot orchestrator | `get_boot_orchestrator` || tracing | `scoped_event_log`, `get_event_log_override` || plugin registries | `get_inference_backend_registry`, `get_model_lister_registry`, `get_orchestrator_registry`, `get_bundle_validator_registry`, `get_storage_provider_registry`, `get_secrets_provider_registry`, `get_optional_service_error_vocabulary` |@@ -78,7 +78,7 @@ Placement is decided by *kind*, not by who currently imports it. Several accesso`get_pipe_func_executor_registry` is the one placement worth flagging: it is a plugin registry by kind (which would put it in the kernel layer) but its protocol lives in `pipe_operators/func/` (which puts it in the interpreter layer). It is **interpreter-layer**. See [Known inversions](#known-inversions) for the underlying wart.-Not everything a hub module exports is an accessor. `resolve_run_mode_for_boot` *applies* the keyless-boot forced-DRY flag rather than reading it, and it lives beside `is_dry_run_forced`, the accessor it consumes. The lifecycle question above still decides placement — the flag is set once at boot and identical for every method the process runs, so the rule that interprets it is kernel-layer by kind. What makes the *module* the right home rather than either caller is that there are now two: the pipe tier's `PipeRunParamsFactory.make_run_params` and the kernel tier's `PipelexKernel.make` both mint run params, and a boot-scoped invariant copied into each is one that drifts. When a rule over hub state acquires a second caller, move the rule to the hub — do not let the second caller reimplement it.+Not everything a hub module exports is an accessor. `resolve_run_mode_for_boot` *applies* the keyless-boot forced-DRY flag rather than reading it, and it lives beside `is_dry_run_forced`, the accessor it consumes. The lifecycle question above still decides placement — the flag is set once at boot and identical for every method the process runs, so the rule that interprets it is kernel-layer by kind. What makes the *module* the right home rather than either caller is that there are now two: the pipe tier's `PipeRunParamsFactory.make_run_params` and the kernel tier's `PipelexKernel.make` both mint run params, and a boot-scoped invariant copied into each is one that drifts. When a rule over hub state acquires a second caller, move the rule to the hub — do not let the second caller reimplement it. `resolve_effective_run_mode` sits beside it for the same reason: it validates the requested mode against `is_mock_usage` before applying the flag, and its callers are `make_run_params`, `pipeline_run_setup`, which needs the mode before it opens the run's tracer, and `PipelexKernel.make`, which is kernel-layer and so could not reach it in the pipe tier.## Why the boundary existsdiff --git a/docs/errors/execution-and-runtime.md b/docs/errors/execution-and-runtime.mdindex 5154d7282..3e713272d 100644--- a/docs/errors/execution-and-runtime.md+++ b/docs/errors/execution-and-runtime.md@@ -57,5 +57,6 @@ own page. Classes are grouped by subsystem.- [`EventLogReadError`](event-log-read-error.md) — Event log read- [`EventLogSchemaMismatchError`](event-log-schema-mismatch-error.md) — Event log schema mismatch- [`EventLogSetupError`](event-log-setup-error.md) — Event log setup+- [`TraceEventModeError`](trace-event-mode-error.md) — Trace event mode[Back to Error Reference](index.md)diff --git a/docs/errors/trace-event-mode-error.md b/docs/errors/trace-event-mode-error.mdnew file mode 100644index 000000000..0f7073a5d--- /dev/null+++ b/docs/errors/trace-event-mode-error.md@@ -0,0 +1,21 @@+---+title: "Trace event mode"+description: "Reference for the `TraceEventModeError` Pipelex error class."+---++<!-- pipelex:generated -->++# Trace event mode++Raised when a set of trace events states no single run mode to build a graph in.++| Field | Value |+|---|---|+| `error_type` | `TraceEventModeError` |+| `title` | Trace event mode |+| `type_uri` | `https://docs.pipelex.com/latest/errors/trace-event-mode-error/` |+| `error_domain` | _(inherited from parent)_ |+| Defined in | `pipelex.tracing.exceptions` |+| Parent class | [`PipelexError`](pipelex-error.md) |++[Back to Error Reference](index.md)diff --git a/docs/under-the-hood/execution-graph-tracing.md b/docs/under-the-hood/execution-graph-tracing.mdindex ceaba87e8..b92979b1b 100644--- a/docs/under-the-hood/execution-graph-tracing.md+++ b/docs/under-the-hood/execution-graph-tracing.md@@ -183,6 +183,8 @@ class GraphSpec(BaseModel):`meta.format` is always `"mthds"` on newly emitted Pipelex graphs. `meta.mode` records provenance for renderers and shared tooling: Pipelex execution graphs emit `"dry"` for dry-run/mock execution and `"live"` for real execution. The shared renderer also accepts `"static"` for graphs produced by a static MTHDS graph builder.+Each builder takes the mode from the run itself, never from a default. The in-process tracer takes the `run_mode` its opener states. The assembler takes the `mode` its events state: it is given no mode of its own, and it refuses, with `TraceEventModeError`, a set of events that states more than one mode, which holds more than one run, and an empty set, which states none. `assemble_tracing` records that refusal as the run's `graph_assembly_error` and keeps the run's usage.+### Node Types| NodeKind | Description |@@ -203,7 +205,7 @@ class GraphSpec(BaseModel):| `CONTROL` | Execution flow between pipes || `DATA` | Data flow (stuff passed between pipes) || `CONTAINS` | Parent-child containment (controller → children) |-| `SELECTED_OUTCOME` | Condition outcome selection |+| `SELECTED_OUTCOME` | From a live `PipeCondition` to the pipe of the outcome it chose, labelled with the outcome-map key the value matched, unlabelled when the default outcome was taken. Both builders derive it from the condition's `condition_resolved` event and the start of the chosen pipe (see [Decisions at Decision Time](#decisions-at-decision-time)) || `BATCH_ITEM` | Batch fan-out: input list → item extracted for each batch iteration || `BATCH_AGGREGATE` | Batch fan-in: item outputs → aggregated output list || `PARALLEL_COMBINE` | Branch outputs → combined output in PipeParallel |@@ -237,6 +239,7 @@ trace_context = manager.open_tracer(data_inclusion=config.data_inclusion,pipeline_ref_domain="my_domain",pipeline_ref_main_pipe="my_pipe",+ run_mode=PipeRunMode.LIVE, # required: whoever opens the trace states the run's mode)# 2. Context flows through JobMetadata to each pipe@@ -293,6 +296,49 @@ Semantics:This is what lets a graph-producing dry-run trace entirely in memory (no NDJSON file, no DynamoDB round-trip). Both dry-run entrypoints rely on it: `dry_run_pipe_in_process` (`pipelex/pipe_run/dry_run_in_process.py` — the graph arm of protocol `validate` and of the single Temporal validation activity) and `dry_run_pipeline` (`pipelex/pipeline/dry_run_pipeline.py`) itself — these functions exist to produce a graph, so they install their own scoped `InMemoryEventLog` rather than depending on the host having tracing configured (a host with `runtime.tracing.is_enabled = false`, like pipelex-api's `/validate` in direct mode, still gets its graph). A run that nonetheless finishes without a graph raises the typed `DryRunGraphNotProducedError`.+### Trace Events++The tracer writes one event for each thing that happens in the run, in the order it happens (`pipelex/tracing/trace_events.py`). Every backend carries the events whole: the NDJSON and DynamoDB logs store each one as its JSON and read it back through the discriminated union `AnyTraceEvent`, and the in-memory and buffering logs hold the objects themselves. Every event carries these fields:++- `pipeline_run_id`, `workflow_id` and `writer_id`, which name the run, the workflow and the process that wrote it;+- `timestamp`, and `sequence`, which rises with each event one writer writes;+- `event_kind`, which tells the event's kind;+- `mode`, `live` or `dry`, the mode of the run that wrote it.++The run's `TraceContext` carries the mode as `run_mode`, so a usage event that an activity writes from the context alone states it too. `mode` is required: an event stored by a version that did not write it is refused with `EventLogSchemaMismatchError` rather than read as live, and running the pipeline again writes a log in the current shape. `run_mode` is required too, with no default, on `TraceContext`, on `GraphTracerManager.open_tracer` and on the tracer's `setup`, since a default would stamp a dry run's events as live: whoever opens a trace states the mode, and a workflow that opens a tracer of its own for part of a run passes the `run_mode` of the context it received.++The mode a trace states is the one the run's pipes run in, which is not always the one requested: a keyless boot (`needs_inference=False`) runs a requested live run dry. `pipeline_run_setup` therefore resolves the run's mode once, before it opens the tracer, through `resolve_effective_run_mode` (`pipelex/runtime_hub.py`), which first refuses `is_mock_usage` on a live request and then applies the keyless coercion. The tracer, the trace context, the OpenTelemetry context and the pipe job all take that one value, so a forced dry run states `dry` on every event and on its graph, and opens no OpenTelemetry trace, as no dry run does. The kernel tier holds to the same rule from the other side: its caller opens the trace, so `PipelexKernel.make` refuses, with `TraceContextRunModeError`, a trace context stating another mode than the one `resolve_effective_run_mode` gives the run (see [The Pipelex Kernel](pipelex-kernel.md)).++| `event_kind` | Written when | What it carries |+|---|---|---|+| `pipe_start` | A pipe starts | The node, its parent, the pipe's code, type, description and domain, its node kind and its inputs; a `PipeSequence` also states `step_count`, the number of steps its author wrote, and a `PipeParallel` states `branch_count`, the number of its branches, both `null` on every other pipe |+| `pipe_end_success` | A pipe ends with an output | The node, its end time, its output and its metrics |+| `pipe_end_error` | A pipe fails | The node, its end time and the error |+| `pipe_end_skipped` | A pipe is lifted because a plain input is absent | The node, its end time, the reason, and the empty list a plural output still writes |+| `edge` | The tracer draws an edge | Its source, target, kind, label and stuff digests |+| `controller_output` | A controller registers an output it produced | The node and the output |+| `batch_item` | A `PipeBatch` extracts an item from its input list | The list and item digests, the item index and the batch's node |+| `batch_aggregate` | A `PipeBatch` adds an item's result to its output list | The output list and item digests, the item index and the batch's node |+| `parallel_combine` | A `PipeParallel` combines its branch outputs | The combined digest, the branch digests and their producers |+| `condition_output_merge` | A dry-run `PipeCondition` merges its outcomes' outputs (see [A Dry-Run Condition's Output](#a-dry-run-conditions-output)) | The merge record |+| `condition_resolved` | A live `PipeCondition`'s choice is final, before anything runs under it | The condition's node, `rendered_value`, the value its expression rendered, and `selected_outcome`, the outcome-map key the value matched, or `null` when the default outcome was taken |+| `batch_fanned_out` | A `PipeBatch` knows how many items it runs, before any item starts | The batch's node, `item_count`, and the pipe each item runs: `branch_pipe_code`, its bare code, `branch_pipe_ref`, its domain-qualified reference, and `branch_pipe_type` |+| `execution_data` | A pipe records details of its execution | The node, and the entries merged into its `execution_data` |+| `usage_report` | An inference job reports what it used | The node it is attributed to and its usage |++#### Decisions at Decision Time++A controller records each decision the moment it is final, so a reader following the trace while the run goes on knows a condition's outcome and a batch's width before the work under them runs. A live `PipeCondition` writes `condition_resolved` once its choice is final: at once for an outcome mapped to `continue` or `fail`, which runs nothing, and, for an outcome that runs a pipe, once it has found that pipe's inputs in working memory. A chosen pipe whose inputs are missing fails the condition before it runs, so the trace records no outcome taken, neither the event nor the execution data. A `PipeBatch` writes `batch_fanned_out` before its first item starts. A dry run runs every outcome of a condition and chooses none, so it writes no `condition_resolved` and no `SELECTED_OUTCOME` edge; a dry batch still writes `batch_fanned_out` for the items it runs. Both are graph events about a node of the tracer's graph: a run that emits no graph events, in costs-only mode, writes neither, and a tracer records a decision only on a node it started itself, as it records execution data, so a decision looked up on a tracer that does not hold the controller's node is dropped rather than written for a node that tracer's graph lacks.++The controllers register their execution data at the same moments:++- A `PipeSequence` registers `step_count` as it starts, and `current_step`, the step's number among the steps its author wrote, as each step starts. The binding written for a dotted `batch_over` shares the number of the batch step after it.+- A `PipeParallel` registers `branch_count`, `add_each_output` and `combined_output_concept` before any branch runs.+- A `PipeBatch` registers `item_count` and `branch_pipe_code` with its fan-out. The two spellings of `branch_pipe_code` differ: the execution data carries the blueprint's field as its author wrote it, which may be domain-qualified, such as `my_domain.summarize`, while the `batch_fanned_out` event's `branch_pipe_code` is always the bare code, `summarize`, with `branch_pipe_ref` the qualified reference, `my_domain.summarize`.+- A live `PipeCondition` registers `evaluated_expression` and `selected_outcome` with its resolution. Here `selected_outcome` is the outcome itself, the code of the pipe it runs or `continue` or `fail`, where the `condition_resolved` event names the outcome-map key.++Both graph builders derive the `SELECTED_OUTCOME` edge from two facts of the run, by the one rule in `pipelex/graph/selected_outcome.py`: a condition's `condition_resolved`, and the start of a pipe whose parent is that condition's node, which can only be the pipe of the outcome it chose. The in-process tracer applies the rule at teardown to the resolutions it recorded and the nodes it holds; the assembler applies it after reading every event, whichever tracer wrote them. The edge is never drawn when the chosen pipe starts, because in a distributed run that pipe can start in another workflow, under a tracer of its own that never saw the resolution, so the edge would be missing from one graph of the run and present in the other. An outcome mapped to `continue` or `fail` starts no pipe under the condition, so it has no edge, and a resolution whose condition node is not in the graph draws none either, since the edge would have no source. The tracer still writes the edge as an `edge` event at teardown, as it does every derived edge, and the assembler regenerates it rather than reading that event, so it is never doubled. Neither builder reads `batch_fanned_out`, because a batch's items are already its children. Both builders therefore build the same graph from the same events.+### TraceContext PropagationTraceContext is a serializable context that flows through pipe execution:@@ -303,6 +349,7 @@ class TraceContext(BaseModel):parent_node_id: str | None # Parent pipe's node IDnode_sequence: int # Counter for generating node IDsdata_inclusion: DataInclusionConfig # What data to capture+ run_mode: PipeRunMode # Live or dry, stated on every trace eventdef copy_for_child(self, child_node_id: str, *, next_sequence: int) -> TraceContext:"""Create context for nested pipe execution."""@@ -311,6 +358,7 @@ class TraceContext(BaseModel):parent_node_id=child_node_id,node_sequence=next_sequence,data_inclusion=self.data_inclusion,+ run_mode=self.run_mode,)```@@ -564,6 +612,8 @@ validate_graphspec(graph_spec)| File | Purpose ||------|---------|| `pipelex/graph/graphspec.py` | Canonical GraphSpec model |+| `pipelex/tracing/trace_events.py` | The trace events and their discriminated union |+| `pipelex/tracing/graphspec_assembler.py` | Builds the GraphSpec from a run's trace events || `pipelex/tracing/usage_attribution.py` | Per-node usage accumulation and subtree rollup || `pipelex/graph/graph_tracer.py` | GraphTracer implementation || `pipelex/graph/graph_tracer_manager.py` | Singleton manager for tracers |@@ -572,6 +622,7 @@ validate_graphspec(graph_spec)| `pipelex/system/data_inclusion_config.py` | Data-capture flags carried by the trace context, surfaced in the TOML under `interpreter.pipeline_execution.graph.data_inclusion` || `pipelex/graph/graph_analysis.py` | Pre-computed graph analysis || `pipelex/graph/condition_output_merge.py` | The merge record of a dry-run condition and the rewrite both graph builders apply |+| `pipelex/graph/selected_outcome.py` | The rule both graph builders derive a live condition's `SELECTED_OUTCOME` edges by || `pipelex/graph/graph_factory.py` | Output generation factory || `pipelex/graph/graph_config.py` | Configuration models || `pipelex/graph/validation.py` | GraphSpec validation |diff --git a/docs/under-the-hood/pipelex-kernel.md b/docs/under-the-hood/pipelex-kernel.mdindex f4ac015dd..f080d3d63 100644--- a/docs/under-the-hood/pipelex-kernel.md+++ b/docs/under-the-hood/pipelex-kernel.md@@ -34,7 +34,7 @@ That boot stands up the model deck, the content generator, the class registry, t**`read_scope` is required too, and `None` is a value you pass, not a default.** It bounds what the run may read: every `pipelex-storage://` key a step reads must lie under it, compared segment by segment, and no step reads a bare path or a `file://` URI from the local disk. `https://` and `data:` URLs are untouched. A value can carry any URL its method or its model chose, so the check runs where values are read, in every content-generation leaf before the dry-run branch, and `shape_inputs` applies it to a table given as a CSV input, which is always a local file. The storage scope must lie under the read scope, since a run reads its own outputs back, and a mismatch is refused at construction. A multi-tenant host passes its tenant's prefix (the hosted platform passes the organization id); `None` is the explicit statement that the run is unscoped, which is the truth for a single-tenant service or a laptop. It has no default for the storage scope's reason: a host that forgot it would run unscoped, and an unscoped run reads every key the process's credentials reach. A refused read raises `UriReadRefusedError`, a caller-facing input error that names where the URL sat and never quotes it.-`needs_inference=False` boots keyless (every enabled backend and model loaded, no credential resolved, no model-deck validation) and sets the forced-DRY flag: every run the process initiates is coerced to `run_mode=DRY`, so the leaves mock instead of calling a provider. `PipelexKernel.make` applies that rule through the same `runtime_hub.resolve_run_mode_for_boot` the pipe tier uses — a second copy of the rule at a second factory is how the two would drift apart.+`needs_inference=False` boots keyless (every enabled backend and model loaded, no credential resolved, no model-deck validation) and sets the forced-DRY flag: every run the process initiates is coerced to `run_mode=DRY`, so the leaves mock instead of calling a provider. `PipelexKernel.make` applies that rule through the same `runtime_hub.resolve_effective_run_mode` the pipe tier uses, which refuses `is_mock_usage` on a live request before it coerces — a second copy of the rule at a second factory is how the two would drift apart.---@@ -149,22 +149,27 @@ Nothing derived from config or the model deck is cached on the instance — resoEverything else is yours:```python-from pipelex.runtime_hub import get_report_delegate+from pipelex.runtime_hub import get_report_delegate, resolve_effective_run_mode+from pipelex.system.pipe_run_mode import PipeRunModefrom pipelex.system.trace_context import TraceContextfrom pipelex.tracing.in_memory_event_log import InMemoryEventLogfrom pipelex.tracing.usage_aggregator import UsageAggregator+# The mode the run takes on this boot: a boot without inference runs a live request dry.+run_mode = resolve_effective_run_mode(requested=PipeRunMode.LIVE, is_mock_usage=False)event_log = InMemoryEventLog()-trace_context = TraceContext(graph_id=run_id, data_inclusion=data_inclusion, emit_graph_events=False, emit_usage_events=True)+trace_context = TraceContext(graph_id=run_id, data_inclusion=data_inclusion, emit_graph_events=False, emit_usage_events=True, run_mode=run_mode)get_report_delegate().set_event_log(context_key=trace_context.lookup_key, event_log=event_log, workflow_id="direct", pipeline_run_id=run_id)try:- kernel = PipelexKernel.make(user_id="my-service", storage_scope="my-service", read_scope=None, trace_context=trace_context)+ kernel = PipelexKernel.make(user_id="my-service", storage_scope="my-service", read_scope=None, run_mode=run_mode, trace_context=trace_context)...tokens_usages = UsageAggregator.aggregate(event_log.read_events(run_id))finally:get_report_delegate().clear_event_log(context_key=trace_context.lookup_key)```+The trace context states the mode the run takes, which `resolve_effective_run_mode` gives: every usage event a step writes states the context's `run_mode`, so a context saying live for a run whose steps run dry would stamp the whole run live. `PipelexKernel.make` refuses a context stating another mode with `TraceContextRunModeError` (`pipelex/kernel/exceptions.py`), a `ValueError` raised before any step runs. It refuses rather than correct its own copy because the trace is the caller's, possibly shared with writers the kernel never sees, which a corrected copy would leave disagreeing with the kernel's events.+Passing a `trace_context` adopts its `graph_id` as the run's `pipeline_run_id`, and passing a `pipeline_run_id` that differs from it raises `ValueError` rather than letting either win. The two are one identity: letting them diverge would scatter a single run's usage events across two ids, because the registered-context emit path stamps the event log's id while the runner fallback stamps the metadata's — and a read-back keyed on either would silently miss the other's.`pipelex.tracing` holds both halves a caller needs (`make_event_log` for a configured backend, `UsageAggregator` for the read-back) and is kernel-layer, so none of this costs the boot contract. The records that come out are the same [`TokensUsage` wire records](./tokens-usage-wire-records.md) an `/execute` response carries — pinned by an integration test that runs the same step through both callers and compares them.diff --git a/pipelex/graph/graph_tracer.py b/pipelex/graph/graph_tracer.pyindex 407598d89..a79ce6f7f 100644--- a/pipelex/graph/graph_tracer.py+++ b/pipelex/graph/graph_tracer.py@@ -12,7 +12,6 @@ from pipelex.graph.graphspec import (EdgeSpec,ErrorSpec,GraphSpec,- GraphSpecMode,IOSpec,NodeIOSpec,NodeKind,@@ -23,13 +22,17 @@ from pipelex.graph.graphspec import (make_graphspec_meta,output_digest_is_optional,)+from pipelex.graph.selected_outcome import derive_selected_outcome_edgesfrom pipelex.system.data_inclusion_config import DataInclusionConfig+from pipelex.system.pipe_run_mode import PipeRunModefrom pipelex.system.trace_context import TraceContextfrom pipelex.tracing.event_log_protocol import EventLogProtocol # ruff: ignore[typing-only-first-party-import] - used in __init__ annotationsfrom pipelex.tracing.trace_events import (BatchAggregateEvent,+ BatchFannedOutEvent,BatchItemEvent,ConditionOutputMergeEvent,+ ConditionResolvedEvent,ControllerOutputEvent,EdgeEvent,ExecutionDataEvent,@@ -161,7 +164,10 @@ class GraphTracer(GraphTracerProtocol):# Registries for pipe and concept data (keyed by pipe_ref and concept_ref)self._pipe_registry: dict[str, dict[str, Any]] = {}self._concept_registry: dict[str, dict[str, Any]] = {}- self._mode: GraphSpecMode = GraphSpecMode.LIVE+ self._run_mode: PipeRunMode = PipeRunMode.LIVE+ # The live conditions that have chosen their outcome, by condition node id, each with the outcome-map key it+ # matched (None for the default outcome): teardown derives the SELECTED_OUTCOME edges from them+ self._condition_resolutions: dict[str, str | None] = {}@propertydef is_active(self) -> bool:@@ -235,7 +241,7 @@ class GraphTracer(GraphTracerProtocol):pipeline_run_id: str | None = None,emit_graph_events: bool = True,emit_usage_events: bool = True,- mode: GraphSpecMode = GraphSpecMode.LIVE,+ run_mode: PipeRunMode,) -> TraceContext:"""Initialize tracing for a new pipeline run.@@ -253,7 +259,8 @@ class GraphTracer(GraphTracerProtocol):teardown skips the discarded spec build; the returned TraceContext carries the flag.emit_usage_events: Whether this run emits usage (cost) events. Stamped onto the returnedTraceContext so it is born with the correct flag.- mode: Provenance mode to stamp onto generated GraphSpecs.+ run_mode: Whether the run is live or a dry run: stamped on every trace event and on the returned+ TraceContext, and turned into the provenance mode of the generated GraphSpec."""self._is_active = Trueself._emit_graph_events = emit_graph_events@@ -278,7 +285,8 @@ class GraphTracer(GraphTracerProtocol):self._event_sequence = 0self._pipe_registry = {}self._concept_registry = {}- self._mode = mode+ self._run_mode = run_mode+ self._condition_resolutions = {}return TraceContext(graph_id=graph_id,@@ -287,6 +295,7 @@ class GraphTracer(GraphTracerProtocol):data_inclusion=data_inclusion,emit_graph_events=emit_graph_events,emit_usage_events=emit_usage_events,+ run_mode=run_mode,)@override@@ -314,6 +323,7 @@ class GraphTracer(GraphTracerProtocol):self._generate_batch_item_edges()self._generate_batch_aggregate_edges()self._generate_parallel_combine_edges()+ self._generate_selected_outcome_edges()self._is_active = False@@ -326,7 +336,7 @@ class GraphTracer(GraphTracerProtocol):pipeline_ref=self._pipeline_ref or PipelineRef(),nodes=nodes,edges=self._edges,- meta=make_graphspec_meta(mode=self._mode),+ meta=make_graphspec_meta(mode=self._run_mode.graphspec_mode),pipe_registry=dict(self._pipe_registry),concept_registry=dict(self._concept_registry),)@@ -353,7 +363,8 @@ class GraphTracer(GraphTracerProtocol):self._event_sequence = 0self._pipe_registry = {}self._concept_registry = {}- self._mode = GraphSpecMode.LIVE+ self._run_mode = PipeRunMode.LIVE+ self._condition_resolutions = {}return graph@@ -494,6 +505,20 @@ class GraphTracer(GraphTracerProtocol):target_stuff_digest=combined_stuff_code,)+ def _generate_selected_outcome_edges(self) -> None:+ """Generate the SELECTED_OUTCOME edges from each resolved condition to the pipe it then started.++ The rule is the one the event-replay assembler applies (`derive_selected_outcome_edges`), so both graphs of a run+ carry the same edges.+ """+ parent_by_node = {node_id: node_data.parent_node_id for node_id, node_data in self._nodes.items()}+ for edge in derive_selected_outcome_edges(resolutions=self._condition_resolutions, parent_by_node=parent_by_node):+ self.add_selected_outcome_edge(+ condition_node_id=edge.condition_node_id,+ outcome_node_id=edge.outcome_node_id,+ selected_outcome=edge.selected_outcome,+ )+@overridedef register_batch_item_extraction(self,@@ -529,6 +554,7 @@ class GraphTracer(GraphTracerProtocol):pipeline_run_id=self._event_pipeline_run_id,writer_id=self._event_writer_id(),workflow_id=self._workflow_id,+ mode=self._run_mode,timestamp=datetime.now(UTC),sequence=self._next_event_sequence(),list_stuff_code=list_stuff_code,@@ -573,6 +599,7 @@ class GraphTracer(GraphTracerProtocol):pipeline_run_id=self._event_pipeline_run_id,writer_id=self._event_writer_id(),workflow_id=self._workflow_id,+ mode=self._run_mode,timestamp=datetime.now(UTC),sequence=self._next_event_sequence(),output_list_stuff_code=output_list_stuff_code,@@ -616,6 +643,7 @@ class GraphTracer(GraphTracerProtocol):pipeline_run_id=self._event_pipeline_run_id,writer_id=self._event_writer_id(),workflow_id=self._workflow_id,+ mode=self._run_mode,timestamp=datetime.now(UTC),sequence=self._next_event_sequence(),combined_stuff_code=combined_stuff_code,@@ -647,12 +675,99 @@ class GraphTracer(GraphTracerProtocol):pipeline_run_id=self._event_pipeline_run_id,writer_id=self._event_writer_id(),workflow_id=self._workflow_id,+ mode=self._run_mode,timestamp=datetime.now(UTC),sequence=self._next_event_sequence(),merge=merge,))+ @override+ def register_batch_fan_out(+ self,+ *,+ batch_node_id: str,+ item_count: int,+ branch_pipe_code: str,+ branch_pipe_ref: str,+ branch_pipe_type: str,+ ) -> None:+ """Record that a PipeBatch knows how many items it runs, before it starts any of them.++ Nothing of the graph depends on it: the event is for whoever reads the trace while the run goes on. It is a graph+ event, recorded on a node of this tracer's graph: a run that emits no graph events (costs-only) records none, and+ neither does a tracer that never started the batch's node.++ Args:+ batch_node_id: The PipeBatch's node ID.+ item_count: The number of items the batch runs.+ branch_pipe_code: The bare code of the pipe each item runs.+ branch_pipe_ref: The `domain.code` reference of the pipe each item runs.+ branch_pipe_type: The type of the pipe each item runs.+ """+ if not self._is_active or not self._emit_graph_events:+ return+ if batch_node_id not in self._nodes:+ return+ if self._event_log is not None:+ self._emit_event(+ BatchFannedOutEvent(+ pipeline_run_id=self._event_pipeline_run_id,+ writer_id=self._event_writer_id(),+ workflow_id=self._workflow_id,+ mode=self._run_mode,+ timestamp=datetime.now(UTC),+ sequence=self._next_event_sequence(),+ node_id=batch_node_id,+ item_count=item_count,+ branch_pipe_code=branch_pipe_code,+ branch_pipe_ref=branch_pipe_ref,+ branch_pipe_type=branch_pipe_type,+ )+ )++ @override+ def register_condition_resolution(+ self,+ *,+ condition_node_id: str,+ rendered_value: str,+ selected_outcome: str | None,+ ) -> None:+ """Record that a live PipeCondition has chosen its outcome, before the chosen pipe starts.++ The graph's SELECTED_OUTCOME edge runs from the condition to the pipe it then starts under its node, derived at+ teardown, as the event-replay assembler derives it from this resolution's event and that pipe's start, whichever+ tracer wrote the start. An outcome mapped to `continue` or `fail` starts no pipe, and so draws no edge. A run that+ emits no graph events (costs-only) records no resolution, and neither does a tracer that never started the+ condition's node, which would otherwise write an event the assembler could draw an edge from a node absent from+ the graph with.++ Args:+ condition_node_id: The PipeCondition's node ID.+ rendered_value: The value the condition's expression rendered.+ selected_outcome: The outcome-map key the value matched, or None when the default outcome was taken.+ """+ if not self._is_active or not self._emit_graph_events:+ return+ if condition_node_id not in self._nodes:+ return+ self._condition_resolutions[condition_node_id] = selected_outcome+ if self._event_log is not None:+ self._emit_event(+ ConditionResolvedEvent(+ pipeline_run_id=self._event_pipeline_run_id,+ writer_id=self._event_writer_id(),+ workflow_id=self._workflow_id,+ mode=self._run_mode,+ timestamp=datetime.now(UTC),+ sequence=self._next_event_sequence(),+ node_id=condition_node_id,+ rendered_value=rendered_value,+ selected_outcome=selected_outcome,+ )+ )+@overridedef on_pipe_start(self,@@ -667,6 +782,8 @@ class GraphTracer(GraphTracerProtocol):concept_data: list[dict[str, Any]] | None = None,description: str | None = None,domain_code: str | None = None,+ step_count: int | None = None,+ branch_count: int | None = None,) -> tuple[str, TraceContext]:"""Record the start of a pipe execution."""if not self._is_active:@@ -711,6 +828,7 @@ class GraphTracer(GraphTracerProtocol):pipeline_run_id=self._event_pipeline_run_id,writer_id=self._event_writer_id(),workflow_id=self._workflow_id,+ mode=self._run_mode,timestamp=started_at,sequence=self._next_event_sequence(),node_id=node_id,@@ -723,6 +841,8 @@ class GraphTracer(GraphTracerProtocol):input_specs=input_specs or [],pipe_data=pipe_data or {},concept_data=concept_data or [],+ step_count=step_count,+ branch_count=branch_count,))@@ -764,6 +884,7 @@ class GraphTracer(GraphTracerProtocol):pipeline_run_id=self._event_pipeline_run_id,writer_id=self._event_writer_id(),workflow_id=self._workflow_id,+ mode=self._run_mode,timestamp=datetime.now(UTC),sequence=self._next_event_sequence(),node_id=node_id,@@ -824,6 +945,7 @@ class GraphTracer(GraphTracerProtocol):pipeline_run_id=self._event_pipeline_run_id,writer_id=self._event_writer_id(),workflow_id=self._workflow_id,+ mode=self._run_mode,timestamp=ended_at,sequence=self._next_event_sequence(),node_id=node_id,@@ -869,6 +991,7 @@ class GraphTracer(GraphTracerProtocol):pipeline_run_id=self._event_pipeline_run_id,writer_id=self._event_writer_id(),workflow_id=self._workflow_id,+ mode=self._run_mode,timestamp=datetime.now(UTC),sequence=self._next_event_sequence(),node_id=node_id,@@ -910,6 +1033,7 @@ class GraphTracer(GraphTracerProtocol):pipeline_run_id=self._event_pipeline_run_id,writer_id=self._event_writer_id(),workflow_id=self._workflow_id,+ mode=self._run_mode,timestamp=ended_at,sequence=self._next_event_sequence(),node_id=node_id,@@ -967,6 +1091,7 @@ class GraphTracer(GraphTracerProtocol):pipeline_run_id=self._event_pipeline_run_id,writer_id=self._event_writer_id(),workflow_id=self._workflow_id,+ mode=self._run_mode,timestamp=ended_at,sequence=self._next_event_sequence(),node_id=node_id,@@ -1014,6 +1139,7 @@ class GraphTracer(GraphTracerProtocol):pipeline_run_id=self._event_pipeline_run_id,writer_id=self._event_writer_id(),workflow_id=self._workflow_id,+ mode=self._run_mode,timestamp=datetime.now(UTC),sequence=self._next_event_sequence(),edge_id=edge_id,@@ -1032,18 +1158,19 @@ class GraphTracer(GraphTracerProtocol):*,condition_node_id: str,outcome_node_id: str,- outcome_value: str,+ selected_outcome: str | None,) -> None:- """Add an edge indicating a selected condition outcome.+ """Add an edge from a condition to the pipe of the outcome it chose.Args:condition_node_id: The condition pipe's node ID.- outcome_node_id: The selected outcome pipe's node ID.- outcome_value: The value that was selected.+ outcome_node_id: The node ID of the pipe the chosen outcome runs.+ selected_outcome: The outcome-map key the condition matched, which labels the edge, or None when it took+ its default outcome, which leaves the edge unlabelled."""self.add_edge(source_node_id=condition_node_id,target_node_id=outcome_node_id,edge_kind=EdgeKind.SELECTED_OUTCOME,- label=outcome_value,+ label=selected_outcome,)diff --git a/pipelex/graph/graph_tracer_manager.py b/pipelex/graph/graph_tracer_manager.pyindex a08f9f2ac..3312e7553 100644--- a/pipelex/graph/graph_tracer_manager.py+++ b/pipelex/graph/graph_tracer_manager.py@@ -7,8 +7,9 @@ from pipelex import logfrom pipelex.graph.condition_output_merge import ConditionOutputMergefrom pipelex.graph.graph_tracer import GraphTracerfrom pipelex.graph.graph_tracer_protocol import GraphTracerProtocol-from pipelex.graph.graphspec import GraphSpec, GraphSpecMode, IOSpec, NodeKind+from pipelex.graph.graphspec import GraphSpec, IOSpec, NodeKindfrom pipelex.system.data_inclusion_config import DataInclusionConfig+from pipelex.system.pipe_run_mode import PipeRunModefrom pipelex.system.registries.singleton import ABCSingletonMeta, MetaSingletonfrom pipelex.system.trace_context import TraceContextfrom pipelex.tools.log.error_fields import error_fields@@ -107,7 +108,7 @@ class GraphTracerManager(metaclass=ABCSingletonMeta):emit_graph_events: bool = True,emit_usage_events: bool = True,describe_pipe_io: bool = True,- mode: GraphSpecMode = GraphSpecMode.LIVE,+ run_mode: PipeRunMode,) -> TraceContext:"""Create and initialize a new tracer for a pipeline run.@@ -131,7 +132,9 @@ class GraphTracerManager(metaclass=ABCSingletonMeta):describe_pipe_io: Whether the run builds the I/O artifacts describing its graphspec'sdata. Read by the run, never by the tracer, so it is stamped on the context hererather than threaded into setup.- mode: Provenance mode to stamp onto generated GraphSpecs.+ run_mode: Whether the run is live or a dry run: stamped on every trace event and on the returned+ TraceContext, and turned into the provenance mode of the generated GraphSpec. A workflow that opens+ a tracer of its own for part of a run passes the mode of the context it received.Returns:Initial TraceContext to pass through JobMetadata.@@ -172,7 +175,7 @@ class GraphTracerManager(metaclass=ABCSingletonMeta):pipeline_run_id=pipeline_run_id,emit_graph_events=emit_graph_events,emit_usage_events=emit_usage_events,- mode=mode,+ run_mode=run_mode,)# Set the tracer_key on the TraceContext so downstream lookups use the same keyif tracer_key is not None:@@ -242,6 +245,8 @@ class GraphTracerManager(metaclass=ABCSingletonMeta):concept_data: list[dict[str, Any]] | None = None,description: str | None = None,domain_code: str | None = None,+ step_count: int | None = None,+ branch_count: int | None = None,) -> tuple[str | None, TraceContext | None]:"""Record the start of a pipe execution.@@ -256,6 +261,8 @@ class GraphTracerManager(metaclass=ABCSingletonMeta):concept_data: Optional list of serialized concept dicts for the concept registry.description: Optional human-readable pipe description (mirrored onto NodeSpec).domain_code: Optional domain code of the pipe (mirrored onto NodeSpec).+ step_count: On a PipeSequence, the number of steps its author wrote.+ branch_count: On a PipeParallel, the number of its branches.Returns:Tuple of (node_id, child_trace_context) if tracing is active, (None, None) otherwise.@@ -275,6 +282,8 @@ class GraphTracerManager(metaclass=ABCSingletonMeta):concept_data=concept_data,description=description,domain_code=domain_code,+ step_count=step_count,+ branch_count=branch_count,)def on_pipe_end_success(@@ -528,3 +537,59 @@ class GraphTracerManager(metaclass=ABCSingletonMeta):if tracer is None:returntracer.register_condition_output_merge(merge=merge)++ def register_batch_fan_out(+ self,+ *,+ lookup_key: str,+ batch_node_id: str,+ item_count: int,+ branch_pipe_code: str,+ branch_pipe_ref: str,+ branch_pipe_type: str,+ ) -> None:+ """Register that a PipeBatch knows how many items it runs, before it starts any of them.++ Args:+ lookup_key: The tracer lookup key.+ batch_node_id: The PipeBatch's node ID.+ item_count: The number of items the batch runs.+ branch_pipe_code: The bare code of the pipe each item runs.+ branch_pipe_ref: The `domain.code` reference of the pipe each item runs.+ branch_pipe_type: The type of the pipe each item runs.+ """+ tracer = self._get_tracer(lookup_key)+ if tracer is None:+ return+ tracer.register_batch_fan_out(+ batch_node_id=batch_node_id,+ item_count=item_count,+ branch_pipe_code=branch_pipe_code,+ branch_pipe_ref=branch_pipe_ref,+ branch_pipe_type=branch_pipe_type,+ )++ def register_condition_resolution(+ self,+ *,+ lookup_key: str,+ condition_node_id: str,+ rendered_value: str,+ selected_outcome: str | None,+ ) -> None:+ """Register that a live PipeCondition has chosen its outcome, before the chosen pipe starts.++ Args:+ lookup_key: The tracer lookup key.+ condition_node_id: The PipeCondition's node ID.+ rendered_value: The value the condition's expression rendered.+ selected_outcome: The outcome-map key the value matched, or None when the default outcome was taken.+ """+ tracer = self._get_tracer(lookup_key)+ if tracer is None:+ return+ tracer.register_condition_resolution(+ condition_node_id=condition_node_id,+ rendered_value=rendered_value,+ selected_outcome=selected_outcome,+ )diff --git a/pipelex/graph/graph_tracer_protocol.py b/pipelex/graph/graph_tracer_protocol.pyindex fd9bce8d7..0f3b7fc35 100644--- a/pipelex/graph/graph_tracer_protocol.py+++ b/pipelex/graph/graph_tracer_protocol.py@@ -4,8 +4,9 @@ from typing import Any, Protocolfrom typing_extensions import overridefrom pipelex.graph.condition_output_merge import ConditionOutputMerge-from pipelex.graph.graphspec import EdgeKind, GraphSpec, GraphSpecMode, IOSpec, NodeKind+from pipelex.graph.graphspec import EdgeKind, GraphSpec, IOSpec, NodeKindfrom pipelex.system.data_inclusion_config import DataInclusionConfig+from pipelex.system.pipe_run_mode import PipeRunModefrom pipelex.system.trace_context import TraceContextfrom pipelex.tracing.event_log_protocol import EventLogProtocol # ruff: ignore[typing-only-first-party-import] - used in setup signature@@ -29,7 +30,7 @@ class GraphTracerProtocol(Protocol):pipeline_run_id: str | None = None,emit_graph_events: bool = True,emit_usage_events: bool = True,- mode: GraphSpecMode = GraphSpecMode.LIVE,+ run_mode: PipeRunMode,) -> TraceContext:"""Initialize tracing for a new pipeline run.@@ -45,7 +46,8 @@ class GraphTracerProtocol(Protocol):Stamped onto the returned TraceContext so it is born with the correct flag.emit_usage_events: Whether usage (cost) trace events should be emitted for this run.Stamped onto the returned TraceContext so it is born with the correct flag.- mode: Provenance mode to stamp onto generated GraphSpecs.+ run_mode: Whether the run is live or a dry run: stamped on every trace event and on the returned+ TraceContext, and turned into the provenance mode of the generated GraphSpec.Returns:Initial TraceContext to pass through JobMetadata.@@ -73,6 +75,8 @@ class GraphTracerProtocol(Protocol):concept_data: list[dict[str, Any]] | None = None,description: str | None = None,domain_code: str | None = None,+ step_count: int | None = None,+ branch_count: int | None = None,) -> tuple[str, TraceContext]:"""Record the start of a pipe execution.@@ -87,6 +91,8 @@ class GraphTracerProtocol(Protocol):concept_data: Optional list of serialized concept dicts for the concept registry.description: Optional human-readable pipe description (mirrored onto NodeSpec).domain_code: Optional domain code of the pipe (mirrored onto NodeSpec).+ step_count: On a PipeSequence, the number of steps its author wrote.+ branch_count: On a PipeParallel, the number of its branches.Returns:Tuple of (node_id for this pipe, updated TraceContext for children).@@ -276,6 +282,45 @@ class GraphTracerProtocol(Protocol):"""...+ def register_batch_fan_out(+ self,+ *,+ batch_node_id: str,+ item_count: int,+ branch_pipe_code: str,+ branch_pipe_ref: str,+ branch_pipe_type: str,+ ) -> None:+ """Register that a PipeBatch knows how many items it runs, before it starts any of them.++ Args:+ batch_node_id: The PipeBatch's node ID.+ item_count: The number of items the batch runs.+ branch_pipe_code: The bare code of the pipe each item runs.+ branch_pipe_ref: The `domain.code` reference of the pipe each item runs.+ branch_pipe_type: The type of the pipe each item runs.+ """+ ...++ def register_condition_resolution(+ self,+ *,+ condition_node_id: str,+ rendered_value: str,+ selected_outcome: str | None,+ ) -> None:+ """Register that a live PipeCondition has chosen its outcome, before the chosen pipe starts.++ A graph builder derives the SELECTED_OUTCOME edge from this resolution and the start of the pipe it then runs+ under the condition's node (see `pipelex.graph.selected_outcome`).++ Args:+ condition_node_id: The PipeCondition's node ID.+ rendered_value: The value the condition's expression rendered.+ selected_outcome: The outcome-map key the value matched, or None when the default outcome was taken.+ """+ ...+def register_execution_data(self,*,@@ -310,13 +355,14 @@ class GraphTracerNoOp(GraphTracerProtocol):pipeline_run_id: str | None = None,emit_graph_events: bool = True,emit_usage_events: bool = True,- mode: GraphSpecMode = GraphSpecMode.LIVE,+ run_mode: PipeRunMode,) -> TraceContext:return TraceContext(graph_id=graph_id,data_inclusion=data_inclusion,emit_graph_events=emit_graph_events,emit_usage_events=emit_usage_events,+ run_mode=run_mode,)@override@@ -337,6 +383,8 @@ class GraphTracerNoOp(GraphTracerProtocol):concept_data: list[dict[str, Any]] | None = None,description: str | None = None,domain_code: str | None = None,+ step_count: int | None = None,+ branch_count: int | None = None,) -> tuple[str, TraceContext]:node_id = trace_context.make_node_id()child_context = trace_context.copy_for_child(node_id, next_sequence=trace_context.node_sequence + 1)@@ -450,3 +498,25 @@ class GraphTracerNoOp(GraphTracerProtocol):merge: ConditionOutputMerge,) -> None:pass++ @override+ def register_batch_fan_out(+ self,+ *,+ batch_node_id: str,+ item_count: int,+ branch_pipe_code: str,+ branch_pipe_ref: str,+ branch_pipe_type: str,+ ) -> None:+ pass++ @override+ def register_condition_resolution(+ self,+ *,+ condition_node_id: str,+ rendered_value: str,+ selected_outcome: str | None,+ ) -> None:+ passdiff --git a/pipelex/graph/selected_outcome.py b/pipelex/graph/selected_outcome.pynew file mode 100644index 000000000..c9e32d293--- /dev/null+++ b/pipelex/graph/selected_outcome.py@@ -0,0 +1,57 @@+"""The SELECTED_OUTCOME edge of a live condition, which both graph builders derive by this one rule.++A live `PipeCondition` records its resolution, and then starts the pipe of the outcome it chose under its own node. The+edge from the condition to that pipe is derived from those two facts once the graph is built, never drawn by the tracer+the chosen pipe starts under: in a distributed run that pipe can start in another workflow, under a tracer of its own+that never saw the resolution, and the event-replay assembler reads both facts from the run's events whichever tracer+wrote them. The in-process tracer and the assembler derive the edge through `derive_selected_outcome_edges`, so the two+graphs of one run carry the same edges.+"""++from collections.abc import Mapping+from typing import NamedTuple+++class SelectedOutcomeEdge(NamedTuple):+ """An edge from a live condition to a pipe it ran, the pipe of the outcome it chose."""++ condition_node_id: str+ outcome_node_id: str+ selected_outcome: str | None+ """The outcome-map key the condition matched, which labels the edge, or None when it took its default outcome."""+++def derive_selected_outcome_edges(+ *,+ resolutions: Mapping[str, str | None],+ parent_by_node: Mapping[str, str | None],+) -> list[SelectedOutcomeEdge]:+ """The SELECTED_OUTCOME edges of a graph, one to each pipe started under a resolved condition.++ A live condition starts no pipe but the one of the outcome it chose, and an outcome mapped to `continue` or `fail`+ starts none, so every pipe whose parent is a resolved condition is the pipe of its chosen outcome. A dry run resolves+ no condition, so its graph has no such edge. A resolution whose condition node is not in the graph draws no edge,+ since the edge would have no source: the event-replay assembler reads resolutions from events whichever writer wrote+ them, and a pipe can name as its parent a node whose start is not among the events it reads.++ Args:+ resolutions: The selected outcome of each resolved condition, by the condition's node ID: the outcome-map key+ the value it rendered matched, or None when it took its default outcome.+ parent_by_node: The parent node ID of every node of the graph, None for a root, in the order the builder holds+ its nodes.++ Returns:+ The edges, in the order of their target nodes in `parent_by_node`.+ """+ edges: list[SelectedOutcomeEdge] = []+ for node_id, parent_node_id in parent_by_node.items():+ if parent_node_id is None or parent_node_id not in resolutions or parent_node_id not in parent_by_node:+ continue+ edges.append(+ SelectedOutcomeEdge(+ condition_node_id=parent_node_id,+ outcome_node_id=node_id,+ selected_outcome=resolutions[parent_node_id],+ )+ )+ return edgesdiff --git a/pipelex/kernel/exceptions.py b/pipelex/kernel/exceptions.pyindex e6c9fc0dd..ec784b58d 100644--- a/pipelex/kernel/exceptions.py+++ b/pipelex/kernel/exceptions.py@@ -16,3 +16,15 @@ class JudgmentOutputFieldsError(ValueError):it is raised before the judging model is called. A `ValueError`, as `PromptContentError` is: thecaller handed the kernel a value it cannot work with."""+++class TraceContextRunModeError(ValueError):+ """A kernel run was handed a trace context stating a run mode other than the one the run takes.++ Every trace event of a run states the mode its steps run in, and the usage events of a kernel run state the mode+ of the trace context its caller supplied. The caller opened that trace, and may share it with writers the kernel+ never sees, so the kernel cannot correct it: it refuses the context before any step runs, rather than let the run+ write events that say live while its steps run dry, as a keyless boot forces them to. The mode a run takes is the+ one `resolve_effective_run_mode` returns, which is what the caller states on the context. A `ValueError`, as the+ other refusals of `PipelexKernel.make` are: the caller handed the kernel a value it cannot work with.+ """diff --git a/pipelex/kernel/pipelex_kernel.py b/pipelex/kernel/pipelex_kernel.pyindex 5dec4e68d..a923ec3c3 100644--- a/pipelex/kernel/pipelex_kernel.py+++ b/pipelex/kernel/pipelex_kernel.py@@ -42,7 +42,7 @@ from contextlib import AbstractContextManagerfrom typing import Any, Selffrom uuid import uuid4-from pipelex.cogt.content_generation.cogt_run_params import CogtRunParams, check_mock_usage_requires_dry+from pipelex.cogt.content_generation.cogt_run_params import CogtRunParamsfrom pipelex.cogt.llm.llm_setting import LLMModelChoicefrom pipelex.core.concepts.concept import Conceptfrom pipelex.core.concepts.concept_factory import ConceptFactory@@ -50,6 +50,7 @@ from pipelex.core.concepts.native.concept_native import NativeConceptCodefrom pipelex.core.memory.working_memory import WorkingMemoryfrom pipelex.core.stuffs.stuff_content import StuffContentfrom pipelex.core.stuffs.text_content import TextContent+from pipelex.kernel.exceptions import TraceContextRunModeErrorfrom pipelex.kernel.llm_ops import (resolve_llm_setting_for_object,resolve_llm_setting_for_text,@@ -59,7 +60,7 @@ from pipelex.kernel.llm_ops import (from pipelex.kernel.llm_prompt_content import LlmPromptContentfrom pipelex.kernel.llm_results import LlmObjectResult, LlmTextResultfrom pipelex.kernel.templating_style_ops import resolve_templating_style-from pipelex.runtime_hub import resolve_run_mode_for_boot+from pipelex.runtime_hub import resolve_effective_run_modefrom pipelex.system.job_metadata import JobMetadata, RunMetadatafrom pipelex.system.pipe_run_mode import PipeRunModefrom pipelex.system.trace_context import TraceContext@@ -106,12 +107,19 @@ class PipelexKernel:This is a run-params factory, so it owes the same keyless-boot contract as the pipe tier's:a process booted with ``needs_inference=False`` forces every run it initiates to DRY, and akernel that skipped that would spend real money on the exact boot this package documents as- its target. Applied through the shared ``resolve_run_mode_for_boot`` — a second copy of the+ its target. Applied through the shared ``resolve_effective_run_mode`` — a second copy of therule at a second factory is how the two would drift. And, like the pipe tier's factory, theREQUESTED mode is validated before that coercion: ``is_mock_usage`` is a DRY-only sub-flag,so an illegal LIVE request must fail loud on every boot rather than be silently legalised bya keyless process forcing it to DRY.+ A ``trace_context`` must state the mode the run takes, the coerced one on a keyless boot: every+ usage event a step writes states the context's ``run_mode``, so a context saying live for a run+ whose steps run dry would stamp the whole run live. The caller opened that trace and may share it+ with writers the kernel never sees, so the kernel refuses a context stating another mode with+ ``TraceContextRunModeError``, before any step runs, rather than correct its own copy and leave+ the others disagreeing.+``request_id`` is the inbound request this run serves, the identifier a hosted deploymentfilters its logs on. It rides the run metadata beside ``user_id``, reaches every step'smetadata and the log context each step binds, and is validated by ``RunMetadata`` exactly as@@ -157,7 +165,15 @@ class PipelexKernel:replay-safe source here; the kernel stays ignorant of it, which is the point of this tier —nothing about a durable backend reaches the runtime or the code it runs."""- check_mock_usage_requires_dry(run_mode=run_mode, is_mock_usage=is_mock_usage)+ effective_run_mode = resolve_effective_run_mode(requested=run_mode, is_mock_usage=is_mock_usage)+ if trace_context is not None and trace_context.run_mode != effective_run_mode:+ msg = (+ f"PipelexKernel.make was given a trace context stating a {trace_context.run_mode} run for a run that "+ f"runs {effective_run_mode}: every event of the run would state a mode its steps do not run in. Open "+ f"the trace with run_mode={effective_run_mode}, the mode resolve_effective_run_mode gives this request "+ "on this boot; a boot without inference runs every run dry."+ )+ raise TraceContextRunModeError(msg)return cls(job_metadata=JobMetadata(run_metadata=RunMetadata(@@ -170,7 +186,7 @@ class PipelexKernel:),trace_context=trace_context,),- cogt_run_params=CogtRunParams(run_mode=resolve_run_mode_for_boot(requested=run_mode), is_mock_usage=is_mock_usage),+ cogt_run_params=CogtRunParams(run_mode=effective_run_mode, is_mock_usage=is_mock_usage),step_id_source=step_id_source,)diff --git a/pipelex/pipe_controllers/batch/pipe_batch.py b/pipelex/pipe_controllers/batch/pipe_batch.pyindex 28429d0d0..6408b100f 100644--- a/pipelex/pipe_controllers/batch/pipe_batch.py+++ b/pipelex/pipe_controllers/batch/pipe_batch.py@@ -27,6 +27,7 @@ if TYPE_CHECKING:from pipelex.core.stuffs.stuff_content import StuffContentfrom pipelex.libraries.library_crate import LibraryCrate+ from pipelex.pipe_machinery.pipe_abstract import PipeAbstract# When a single PipeBatch fans out over more than this many items, log a one-time advisory pointing at# durable execution — bounded fan-out is a basic backpressure effort, not durable, rate-limited execution.@@ -148,6 +149,8 @@ class PipeBatch(PipeController):batch_output_stuff_code = StuffFactory.make_stuff_code()item_count = len(input_content.items)+ # The batch records its width the moment it knows it, before it starts any item.+ self._register_fan_out(job_metadata=job_metadata, item_count=item_count, branch_pipe=sub_pipe)# Read off the payload, never off live config: the bound was frozen into the run params at# submit time precisely so the fan-out shape stays a pure function of the run. See# `PipeRunParams.batch_max_concurrency`.@@ -275,16 +278,28 @@ class PipeBatch(PipeController):name=output_name,)- # Capture execution data for the graph tracer+ return PipeOutput(+ working_memory=working_memory,+ pipeline_run_id=job_metadata.run_metadata.pipeline_run_id,+ )++ def _register_fan_out(self, *, job_metadata: JobMetadata, item_count: int, branch_pipe: "PipeAbstract") -> None:+ """Record the batch's width and the pipe each item runs, in its execution data and as its fan-out trace event."""execution_data_dict: dict[str, Any] = {- "item_count": len(input_content.items),+ "item_count": item_count,"branch_pipe_code": self.branch_pipe_code,}self._register_execution_data(job_metadata=job_metadata, execution_data=execution_data_dict)-- return PipeOutput(- working_memory=working_memory,- pipeline_run_id=job_metadata.run_metadata.pipeline_run_id,+ trace_node = self._trace_node(job_metadata=job_metadata)+ if trace_node is None:+ return+ trace_node.tracer_manager.register_batch_fan_out(+ lookup_key=trace_node.lookup_key,+ batch_node_id=trace_node.node_id,+ item_count=item_count,+ branch_pipe_code=branch_pipe.code,+ branch_pipe_ref=branch_pipe.pipe_ref,+ branch_pipe_type=branch_pipe.type,)@overridediff --git a/pipelex/pipe_controllers/condition/pipe_condition.py b/pipelex/pipe_controllers/condition/pipe_condition.pyindex d35d3016f..c067a898a 100644--- a/pipelex/pipe_controllers/condition/pipe_condition.py+++ b/pipelex/pipe_controllers/condition/pipe_condition.py@@ -473,18 +473,15 @@ class PipeCondition(PipeController):# Select the outcome based on the evaluated expressionoutcome = self.outcome_map.get(evaluated_expression, self.default_outcome)- # Capture execution data for the graph tracer- execution_data_dict: dict[str, Any] = {- "evaluated_expression": evaluated_expression,- "selected_outcome": str(outcome),- }+ # The condition records its choice the moment the choice is final, before anything runs under it: at once for+ # `continue` and `fail`, which run nothing, and for a pipe once its inputs are known to be there to run it.# Handle continue case (phase 1): `continue` resolves the declared output as# ABSENT — a declared-absent record with provenance, memory otherwise unchanged. A previous# main stuff stays under its own name (the migration idiom: consume it explicitly# downstream); it no longer passes through as this pipe's output.if SpecialOutcome.is_continue(outcome):- self._register_execution_data(job_metadata=job_metadata, execution_data=execution_data_dict)+ self._record_resolution(job_metadata=job_metadata, evaluated_expression=evaluated_expression, outcome=outcome)# The reason names only the pipe and the outcome: a pipe that force-unwraps this output quotes it to the caller# under STRICT disclosure, and the value the expression rendered can be literal text of a condition a host# library declared. The execution data above keeps that value.@@ -496,7 +493,7 @@ class PipeCondition(PipeController):return PipeOutput(working_memory=working_memory, pipeline_run_id=job_metadata.run_metadata.pipeline_run_id)if SpecialOutcome.is_fail(outcome):- self._register_execution_data(job_metadata=job_metadata, execution_data=execution_data_dict)+ self._record_resolution(job_metadata=job_metadata, evaluated_expression=evaluated_expression, outcome=outcome)# The caller's method maps this outcome to 'fail' on purpose, a refusal of the run it was given. The# message names only the pipe: the value the expression rendered can be literal text of the expression,# or an outcome key, of a condition a host library declared. The execution data above keeps that value.@@ -540,9 +537,11 @@ class PipeCondition(PipeController):))- pipe_output = await get_pipe_router().run(+ # Recorded only now: a choice whose pipe cannot run is a failure of the condition, never an outcome it took.+ self._record_resolution(job_metadata=job_metadata, evaluated_expression=evaluated_expression, outcome=outcome)+ return await get_pipe_router().run(pipe_job=PipeJobFactory.make_pipe_job(- pipe=get_required_pipe(pipe_code=outcome),+ pipe=chosen_pipe,job_metadata=job_metadata,working_memory=working_memory,pipe_run_params=pipe_run_params,@@ -550,8 +549,33 @@ class PipeCondition(PipeController):library_crate=library_crate,),)++ def _record_resolution(self, *, job_metadata: JobMetadata, evaluated_expression: str, outcome: str) -> None:+ """Record the outcome a live run chose, in the execution data and as the condition's resolution trace event.++ The execution data keeps the value the expression rendered and the outcome it selected, a pipe code or a special+ outcome. The trace event names the outcome by the key of the outcome map the value matched, or by None when it+ matched none and the default outcome was taken, since `"default"` can itself be a key.++ Args:+ job_metadata: The condition's job metadata, whose trace context names its graph node.+ evaluated_expression: The value the condition's expression rendered.+ outcome: The outcome selected: a pipe code, `continue` or `fail`.+ """+ execution_data_dict: dict[str, Any] = {+ "evaluated_expression": evaluated_expression,+ "selected_outcome": outcome,+ }self._register_execution_data(job_metadata=job_metadata, execution_data=execution_data_dict)- return pipe_output+ trace_node = self._trace_node(job_metadata=job_metadata)+ if trace_node is None:+ return+ trace_node.tracer_manager.register_condition_resolution(+ lookup_key=trace_node.lookup_key,+ condition_node_id=trace_node.node_id,+ rendered_value=evaluated_expression,+ selected_outcome=evaluated_expression if evaluated_expression in self.outcome_map else None,+ )@overrideasync def _dry_run_controller_pipe(diff --git a/pipelex/pipe_controllers/parallel/pipe_parallel.py b/pipelex/pipe_controllers/parallel/pipe_parallel.pyindex d44e217d5..c0ea18729 100644--- a/pipelex/pipe_controllers/parallel/pipe_parallel.py+++ b/pipelex/pipe_controllers/parallel/pipe_parallel.py@@ -438,6 +438,10 @@ class PipeParallel(PipeController):variable_names=[sub_pipe.output_name],)+ @override+ def traced_branch_count(self) -> int:+ return len(self.parallel_sub_pipes)+@overridedef pipe_dependencies(self) -> set[str]:return {sub_pipe.pipe_code for sub_pipe in self.parallel_sub_pipes}@@ -463,6 +467,14 @@ class PipeParallel(PipeController):final_stuff_code = pipe_run_params.final_stuff_codepipe_run_params.final_stuff_code = None+ # The parallel records how many branches it runs, and how it combines them, as it starts, before any branch runs.+ execution_data_dict: dict[str, Any] = {+ "branch_count": self.traced_branch_count(),+ "add_each_output": self.add_each_output,+ "combined_output_concept": self.output.concept.concept_ref,+ }+ self._register_execution_data(job_metadata=job_metadata, execution_data=execution_data_dict)+tasks: list[Coroutine[Any, Any, PipeOutput]] = []for sub_pipe in self.parallel_sub_pipes:@@ -596,14 +608,6 @@ class PipeParallel(PipeController):output_stuffs=output_stuffs,)- # Capture execution data for the graph tracer- execution_data_dict: dict[str, Any] = {- "branch_count": len(self.parallel_sub_pipes),- "add_each_output": self.add_each_output,- "combined_output_concept": self.output.concept.concept_ref,- }- self._register_execution_data(job_metadata=job_metadata, execution_data=execution_data_dict)-return PipeOutput(working_memory=working_memory,pipeline_run_id=job_metadata.run_metadata.pipeline_run_id,diff --git a/pipelex/pipe_controllers/sequence/pipe_sequence.py b/pipelex/pipe_controllers/sequence/pipe_sequence.pyindex cc78c3b03..926f67c59 100644--- a/pipelex/pipe_controllers/sequence/pipe_sequence.py+++ b/pipelex/pipe_controllers/sequence/pipe_sequence.py@@ -1,5 +1,5 @@import sys-from typing import TYPE_CHECKING, Any, Literal+from typing import TYPE_CHECKING, Literalfrom pydantic import PrivateAttr, field_validatorfrom typing_extensions import override@@ -37,6 +37,7 @@ from pipelex.pipe_controllers.sequence.sequence_typed_flow import (SequenceFlowMemo,SequenceStep,SequenceTypedFlow,+ authored_step_count,authored_step_number,build_sequence_typed_flow,derive_binding_spec,@@ -89,6 +90,11 @@ class PipeSequence(PipeController):"""The number messages give a step: its place among the steps the author wrote, counting from 1."""return authored_step_number(steps=self.sequential_sub_pipes, step_index=step_index)+ @override+ def traced_step_count(self) -> int:+ """The number of steps the author wrote: the binding written for a dotted `batch_over` is no step of theirs."""+ return authored_step_count(steps=self.sequential_sub_pipes)+@propertydef has_binding_step(self) -> bool:"""Whether a step binds a value, which only a run of a sequence with one reads from the typed flow.@@ -1032,9 +1038,17 @@ class PipeSequence(PipeController):typed_flow: SequenceTypedFlow | None = None# By step index, each binding the run derived from the value its root held, the flow having no spec for that root.runtime_bindings: dict[int, DerivedBinding] = {}+ # The sequence records how many steps it runs as it starts, and the number of each step as that step starts.+ self._register_execution_data(job_metadata=job_metadata, execution_data={"step_count": self.traced_step_count()})+ current_step: int | None = Nonefor sub_pipe_index, sub_pipe in enumerate(self.sequential_sub_pipes):is_last_step = sub_pipe_index == len(self.sequential_sub_pipes) - 1+ step_number = self._step_number(step_index=sub_pipe_index)+ # The binding written for a dotted `batch_over` shares the number of the batch step after it: one record each.+ if step_number != current_step:+ current_step = step_number+ self._register_execution_data(job_metadata=job_metadata, execution_data={"current_step": step_number})if isinstance(sub_pipe, BindingStep):if typed_flow is None:typed_flow = self.build_typed_flow()@@ -1066,11 +1080,6 @@ class PipeSequence(PipeController):library_crate=library_crate,)evolving_memory = pipe_output.working_memory- # Capture execution data for the graph tracer- execution_data_dict: dict[str, Any] = {- "step_count": len(self.sequential_sub_pipes),- }- self._register_execution_data(job_metadata=job_metadata, execution_data=execution_data_dict)return PipeOutput(working_memory=evolving_memory,diff --git a/pipelex/pipe_controllers/sequence/sequence_typed_flow.py b/pipelex/pipe_controllers/sequence/sequence_typed_flow.pyindex 0af585a59..11efb1252 100644--- a/pipelex/pipe_controllers/sequence/sequence_typed_flow.py+++ b/pipelex/pipe_controllers/sequence/sequence_typed_flow.py@@ -177,16 +177,26 @@ class DerivedBinding(NamedTuple):stuff_spec: StuffSpec+def is_authored_step(*, step: SequenceStep) -> bool:+ """Whether the author wrote this step: the binding a sequence writes for a dotted `batch_over` is no step of theirs."""+ return not (isinstance(step, BindingStep) and step.is_dotted_batch_over)++def authored_step_number(*, steps: list[SequenceStep], step_index: int) -> int:"""The number of a step, counting from 1, as its author wrote the sequence.The binding a sequence writes for a dotted `batch_over` is no step of the author's: it shares the number of the pipestep it was written for, which follows it."""- rewritten_before = sum(1 for step in steps[:step_index] if isinstance(step, BindingStep) and step.is_dotted_batch_over)+ rewritten_before = sum(1 for step in steps[:step_index] if not is_authored_step(step=step))return step_index + 1 - rewritten_before+def authored_step_count(*, steps: list[SequenceStep]) -> int:+ """The number of steps the author wrote, the number `authored_step_number` gives the last of them."""+ return sum(1 for step in steps if is_authored_step(step=step))++def resolve_library_concept(*, concept_library: ConceptLibraryAbstract, concept_ref: str) -> Concept | None:"""The library's concept for a ref, looked up directly, then among a dependency's aliased keys when one alone matches."""try:diff --git a/pipelex/pipe_machinery/pipe_abstract.py b/pipelex/pipe_machinery/pipe_abstract.pyindex fb2d86372..9a64351ca 100644--- a/pipelex/pipe_machinery/pipe_abstract.py+++ b/pipelex/pipe_machinery/pipe_abstract.py@@ -115,6 +115,14 @@ class CompanionSlot(NamedTuple):producing_pipe_code: str+class PipeTraceNode(NamedTuple):+ """A running pipe's own node in the run's trace, with the tracer manager that records what happens on it."""++ tracer_manager: GraphTracerManager+ lookup_key: str+ node_id: str++class InputPresenceScan(NamedTuple):"""The runtime trichotomy (D3) applied to a pipe's needed inputs against working memory.@@ -170,6 +178,20 @@ class PipeAbstract(ABC, BaseModel):# overrides this to True. (`pipe_category` no longer encodes signature-ness.)return False+ def traced_step_count(self) -> int | None:+ """The number of steps a sequence's author wrote, which the trace records on the pipe's start.++ None for every pipe that runs no steps; `PipeSequence` overrides it.+ """+ return None++ def traced_branch_count(self) -> int | None:+ """The number of branches a parallel runs, which the trace records on the pipe's start.++ None for every pipe that runs no branches; `PipeParallel` overrides it.+ """+ return None+def pipe_dependencies(self) -> set[str]:"""Return the set of pipe codes that this pipe depends on.@@ -196,21 +218,32 @@ class PipeAbstract(ABC, BaseModel):return unique_concepts+ def _trace_node(self, *, job_metadata: JobMetadata) -> PipeTraceNode | None:+ """This running pipe's own node in the run's trace, or None when nothing traces the run.++ The pipe's start hands it the trace context its children run under, whose `parent_node_id` is the pipe's own+ node: whatever the pipe records about itself while it runs is recorded on that node.+ """+ trace_context = job_metadata.trace_context+ if trace_context is None or trace_context.parent_node_id is None:+ return None+ tracer_manager = GraphTracerManager.get_instance()+ if tracer_manager is None:+ return None+ return PipeTraceNode(tracer_manager=tracer_manager, lookup_key=trace_context.lookup_key, node_id=trace_context.parent_node_id)+def _register_execution_data(self, *, job_metadata: JobMetadata, execution_data: dict[str, Any]) -> None:"""Register execution metadata with the graph tracer.Called by pipe subclasses during execution to capture runtime-resolved data(rendered prompts, resolved models, etc.) for the GraphSpec."""- trace_context = job_metadata.trace_context- if trace_context is None:- return- tracer_manager = GraphTracerManager.get_instance()- if tracer_manager is None or trace_context.parent_node_id is None:+ trace_node = self._trace_node(job_metadata=job_metadata)+ if trace_node is None:return- tracer_manager.register_execution_data(- lookup_key=trace_context.lookup_key,- node_id=trace_context.parent_node_id,+ trace_node.tracer_manager.register_execution_data(+ lookup_key=trace_node.lookup_key,+ node_id=trace_node.node_id,execution_data=execution_data,)@@ -787,6 +820,8 @@ class PipeAbstract(ABC, BaseModel):concept_data=concept_data,description=self.description,domain_code=self.domain_code,+ step_count=self.traced_step_count(),+ branch_count=self.traced_branch_count(),)# Update job metadata with child trace context for nested pipesif child_trace_context is not None:diff --git a/pipelex/pipe_run/dry_run_in_process.py b/pipelex/pipe_run/dry_run_in_process.pyindex 70adbbb54..2d5b28033 100644--- a/pipelex/pipe_run/dry_run_in_process.py+++ b/pipelex/pipe_run/dry_run_in_process.py@@ -16,7 +16,7 @@ from pipelex.base_exceptions import PipelexErrorfrom pipelex.cogt.content_generation.content_generator import ContentGeneratorfrom pipelex.config import get_configfrom pipelex.graph.graph_tracer_manager import GraphTracerManager-from pipelex.graph.graphspec import GraphSpec, GraphSpecMode+from pipelex.graph.graphspec import GraphSpecfrom pipelex.interpreter_hub import get_library_manager, scoped_pipe_routerfrom pipelex.observer.observer_protocol import ObserverNoOpfrom pipelex.pipe_machinery.pipe_abstract import PipeAbstract@@ -160,7 +160,7 @@ async def dry_run_pipe_in_process(pipe: PipeAbstract, *, library_id: str, calleremit_usage_events=False,# Validate builds the I/O artifacts once, for its report; this dry run keeps only the graph.describe_pipe_io=False,- mode=GraphSpecMode.DRY,+ run_mode=PipeRunMode.DRY,)try:with (diff --git a/pipelex/pipe_run/pipe_run.py b/pipelex/pipe_run/pipe_run.pyindex 67977ac0a..56007752a 100644--- a/pipelex/pipe_run/pipe_run.py+++ b/pipelex/pipe_run/pipe_run.py@@ -136,7 +136,6 @@ class PipeRun(PipeRunProtocol):assemble_usage=trace_context.emit_usage_events,domain_code=pipe_job.pipe.domain_code,main_pipe_code=pipe_job.pipe.code,- run_mode=pipe_job.pipe_run_params.run_mode,trace_events=trace_events,)if trace_context.describe_pipe_io and pipe_output.graph_spec is not None:diff --git a/pipelex/pipe_run/pipe_run_params_factory.py b/pipelex/pipe_run/pipe_run_params_factory.pyindex 5d13a282a..1c978ad8c 100644--- a/pipelex/pipe_run/pipe_run_params_factory.py+++ b/pipelex/pipe_run/pipe_run_params_factory.py@@ -1,10 +1,9 @@from typing import Any, Literal-from pipelex.cogt.content_generation.cogt_run_params import check_mock_usage_requires_dryfrom pipelex.config import get_configfrom pipelex.core.pipes.variable_multiplicity import VariableMultiplicityfrom pipelex.pipe_run.pipe_run_params import BatchParams, PipeRunParams-from pipelex.runtime_hub import resolve_run_mode_for_boot+from pipelex.runtime_hub import resolve_effective_run_modefrom pipelex.system.pipe_run_mode import PipeRunMode@@ -45,13 +44,13 @@ class PipeRunParamsFactory:The REQUESTED mode is validated before the forced-DRY coercion, so a contract violation(``is_mock_usage`` on a LIVE request) fails loud on every boot — the keyless coercion must- not silently turn an illegal request into a legal one.+ not silently turn an illegal request into a legal one. Both steps are ``resolve_effective_run_mode``,+ which ``pipeline_run_setup`` also calls before it opens the run's tracer.``batch_max_concurrency`` is resolved here for the same single-writer reason and because theread must happen ONCE, at submit time: see the field's docstring on ``PipeRunParams``."""- check_mock_usage_requires_dry(run_mode=pipe_run_mode, is_mock_usage=is_mock_usage)- pipe_run_mode = resolve_run_mode_for_boot(requested=pipe_run_mode)+ pipe_run_mode = resolve_effective_run_mode(requested=pipe_run_mode, is_mock_usage=is_mock_usage)config = get_config().interpreterpipe_stack_limit = pipe_stack_limit or config.pipe_run.pipe_stack_limitreturn PipeRunParams(diff --git a/pipelex/pipe_run/tracing_assembly.py b/pipelex/pipe_run/tracing_assembly.pyindex db332d300..d29855830 100644--- a/pipelex/pipe_run/tracing_assembly.py+++ b/pipelex/pipe_run/tracing_assembly.py@@ -24,10 +24,9 @@ from pipelex.reporting.reporting_types import AnyTokensUsagefrom pipelex.runtime_hub import get_event_log_overridefrom pipelex.system.exceptions import MissingDependencyErrorfrom pipelex.system.job_metadata import RunMetadata-from pipelex.system.pipe_run_mode import PipeRunModefrom pipelex.tools.log.error_fields import error_fieldsfrom pipelex.tracing.event_log_factory import make_event_log-from pipelex.tracing.exceptions import EventLogError+from pipelex.tracing.exceptions import EventLogError, TraceEventModeErrorfrom pipelex.tracing.graphspec_assembler import GraphSpecAssemblerfrom pipelex.tracing.trace_events import TraceEventfrom pipelex.tracing.usage_aggregator import UsageAggregator@@ -137,7 +136,6 @@ def assemble_tracing(assemble_usage: bool,domain_code: str | None = None,main_pipe_code: str | None = None,- run_mode: PipeRunMode = PipeRunMode.LIVE,run_metadata: RunMetadata | None = None,trace_events: RunTraceEvents | None = None,) -> TracingAssembly:@@ -154,8 +152,10 @@ def assemble_tracing(errors, missing optional dependencies) are caught and recorded in the ``*_error``fields, not propagated. Either backend refuses a log holding an event the currentmodels do not accept with ``EventLogSchemaMismatchError``, also an ``EventLogError``:- the read fails as a whole, so such a run gets neither its graph nor its usage total. Programming bugs (KeyError,- AttributeError, etc.) propagate so they surface during development.+ the read fails as a whole, so such a run gets neither its graph nor its usage total. The graph's provenance mode is+ the run mode its events state: events that mix modes hold more than one run, and get no graph, the assembler's+ ``TraceEventModeError`` recorded in ``graph_assembly_error``. Programming bugs (KeyError, AttributeError, etc.)+ propagate so they surface during development.Args:pipeline_run_id: The pipeline run ID to query events for.@@ -163,7 +163,6 @@ def assemble_tracing(assemble_usage: Whether to aggregate token usage from the events.domain_code: Domain code for the graph's pipeline ref.main_pipe_code: Main pipe code for the graph's pipeline ref.- run_mode: Final pipe run mode used to stamp GraphSpec provenance.run_metadata: The run this assembly belongs to, stamped onto the result so atransport that offloads it keys it inside the run's namespace. None for arun with no host-supplied scope.@@ -186,7 +185,6 @@ def assemble_tracing(assemble_usage=assemble_usage,domain_code=domain_code,main_pipe_code=main_pipe_code,- run_mode=run_mode,trace_events=trace_events,)@@ -199,7 +197,6 @@ def _assemble_tracing_events(assemble_usage: bool,domain_code: str | None,main_pipe_code: str | None,- run_mode: PipeRunMode,trace_events: RunTraceEvents | None,) -> TracingAssembly:"""The body of ``assemble_tracing``, run inside the log context it binds: read the events once, assemble onto ``result``."""@@ -230,12 +227,11 @@ def _assemble_tracing_events(domain=domain_code,main_pipe=main_pipe_code,),- mode=run_mode.graphspec_mode,)log.debug("Assembled the graph from the trace events", fields={"event_count": len(events)})- except ValidationError as validation_error:- log.warning("The graph could not be assembled from the trace events", fields=error_fields(exc=validation_error))- result.graph_assembly_error = f"Graph assembly failed for pipeline_run_id={pipeline_run_id}: {validation_error}"+ except (ValidationError, TraceEventModeError) as assembly_error:+ log.warning("The graph could not be assembled from the trace events", fields=error_fields(exc=assembly_error))+ result.graph_assembly_error = f"Graph assembly failed for pipeline_run_id={pipeline_run_id}: {assembly_error}"return result@@ -248,7 +244,6 @@ def assemble_tracing_on_output(assemble_usage: bool,domain_code: str | None = None,main_pipe_code: str | None = None,- run_mode: PipeRunMode = PipeRunMode.LIVE,trace_events: RunTraceEvents | None = None,) -> None:"""Assemble graph and/or usage from trace events and set them on pipe_output (DIRECT mode).@@ -264,7 +259,6 @@ def assemble_tracing_on_output(assemble_usage: Whether to aggregate and set token usage.domain_code: Domain code for the graph's pipeline ref.main_pipe_code: Main pipe code for the graph's pipeline ref.- run_mode: Final pipe run mode used to stamp GraphSpec provenance.trace_events: The run's events when the caller already read them; None to read them here."""result = assemble_tracing(@@ -273,7 +267,6 @@ def assemble_tracing_on_output(assemble_usage=assemble_usage,domain_code=domain_code,main_pipe_code=main_pipe_code,- run_mode=run_mode,# Inherited from the output this assembly decorates, so both key together.run_metadata=pipe_output.job_metadata.run_metadata if pipe_output.job_metadata else None,trace_events=trace_events,diff --git a/pipelex/pipeline/pipeline_run_setup.py b/pipelex/pipeline/pipeline_run_setup.pyindex 56ac749bc..d7ef4e23f 100644--- a/pipelex/pipeline/pipeline_run_setup.py+++ b/pipelex/pipeline/pipeline_run_setup.py@@ -25,7 +25,13 @@ from pipelex.pipe_run.pipe_run_params import ()from pipelex.pipeline.exceptions import PipeExecutionErrorfrom pipelex.pipeline.execution_seams import acquire_library, prepare_pipe_job-from pipelex.runtime_hub import get_event_log_override, get_otel_tracer, get_report_delegate, get_telemetry_manager+from pipelex.runtime_hub import (+ get_event_log_override,+ get_otel_tracer,+ get_report_delegate,+ get_telemetry_manager,+ resolve_effective_run_mode,+)from pipelex.system.configuration.configs import PipelineExecutionConfigfrom pipelex.system.environment import get_optional_envfrom pipelex.system.job_metadata import OtelContext@@ -122,7 +128,9 @@ async def pipeline_run_setup(pipe_run_mode:Pipe run mode: ``PipeRunMode.LIVE`` or ``PipeRunMode.DRY``. If not specified,inferred from the environment variable ``PIPELEX_FORCE_DRY_RUN_MODE``. Defaults- to ``PipeRunMode.LIVE`` if the environment variable is not set.+ to ``PipeRunMode.LIVE`` if the environment variable is not set. A keyless boot+ (``needs_inference=False``) runs a LIVE request DRY, and the run's trace, its+ graph and its pipe job all state DRY.is_mock_usage:Internal sub-flag of ``run_mode=DRY`` — only CLI access is the hidden ``--mock-usage``test trigger (requires ``--dry-run``, not shown in --help): when True, the dry@@ -266,6 +274,15 @@ async def pipeline_run_setup(else:pipe_run_mode = PipeRunMode.LIVE+ # The mode the run takes, resolved ONCE, here, above everything this function opens. A keyless+ # boot forces a requested LIVE run DRY (eng review D4), and the tracer, the trace context, the+ # OTel context and the pipe job must all state the mode the pipes run in. The coercion used to+ # happen only in the run-params factory, below the tracer, so a forced-dry run stamped every+ # trace event live, and the graph built from those events said live too. The requested mode is+ # validated against `is_mock_usage` first, so the coercion cannot legalise a LIVE request+ # carrying the dry sub-flag; the factory below sees DRY and coerces nothing again.+ pipe_run_mode = resolve_effective_run_mode(requested=pipe_run_mode, is_mock_usage=is_mock_usage)+pipeline = get_pipeline_manager().add_new_pipeline(pipe_code=pipe_code, pipeline_run_id=pipeline_run_id)pipeline_run_id = pipeline.pipeline_run_id@@ -387,7 +404,7 @@ async def pipeline_run_setup(emit_usage_events=is_generate_usage,# The I/O artifacts are the graphspec's companions: a run that writes no graphspec builds none.describe_pipe_io=is_generate_graph and execution_config.graph.graphs_inclusion.graphspec_json,- mode=pipe_run_mode.graphspec_mode,+ run_mode=pipe_run_mode,)# Register the event log on the report delegate for usage event emission — only when cost reportingdiff --git a/pipelex/reporting/reporting_manager.py b/pipelex/reporting/reporting_manager.pyindex 3a3596945..8930abf6d 100644--- a/pipelex/reporting/reporting_manager.py+++ b/pipelex/reporting/reporting_manager.py@@ -239,6 +239,7 @@ class ReportingManager(ReportingProtocol):pipeline_run_id=context.pipeline_run_id,workflow_id=context.workflow_id,writer_id=context.event_log.writer_id,+ mode=trace_context.run_mode,timestamp=datetime.now(UTC),sequence=seq,node_id=node_id,@@ -315,6 +316,7 @@ class ReportingManager(ReportingProtocol):pipeline_run_id=inference_job.job_metadata.run_metadata.pipeline_run_id,workflow_id=workflow_id,writer_id=process_event_log.writer_id,+ mode=trace_context.run_mode,timestamp=datetime.now(UTC),sequence=seq,node_id=node_id,diff --git a/pipelex/runtime_hub.py b/pipelex/runtime_hub.pyindex 65beba670..8269c5d9d 100644--- a/pipelex/runtime_hub.py+++ b/pipelex/runtime_hub.py@@ -29,6 +29,7 @@ from typing import TYPE_CHECKING, Any, ClassVar, Optional, TextIOfrom kajson.class_registry_abstract import ClassRegistryAbstractfrom pipelex import log+from pipelex.cogt.content_generation.cogt_run_params import check_mock_usage_requires_dryfrom pipelex.cogt.content_generation.content_generator_protocol import (ContentGeneratorProtocol,)@@ -671,6 +672,31 @@ def resolve_run_mode_for_boot(*, requested: PipeRunMode) -> PipeRunMode:return requested+def resolve_effective_run_mode(*, requested: PipeRunMode, is_mock_usage: bool) -> PipeRunMode:+ """The mode a run this process initiates takes on this boot: the requested mode, validated, then forced DRY when keyless.++ The REQUESTED mode is validated before the keyless-boot coercion (eng review D4), so a contract violation+ (``is_mock_usage`` on a LIVE request) fails loud on every boot: the coercion must not silently turn an illegal+ request into a legal one. Every entry point resolves its mode through it: the pipe tier's+ ``PipeRunParamsFactory.make_run_params``, ``pipeline_run_setup``, which needs the effective mode before it opens+ the run's tracer so that the trace states the mode the pipes run in, and the kernel tier's ``PipelexKernel.make``.+ It lives here, beside the flag it applies, so the kernel layer reaches it without importing the interpreter. A+ caller that opens a trace for a kernel run states this mode on its ``TraceContext``.++ Args:+ requested: The run mode the caller asked for.+ is_mock_usage: Whether the caller asked for the dry sub-flag that reports synthetic usage.++ Returns:+ The requested mode, or DRY when a keyless boot forces a LIVE request dry.++ Raises:+ ValueError: When ``is_mock_usage`` is set on a LIVE request.+ """+ check_mock_usage_requires_dry(run_mode=requested, is_mock_usage=is_mock_usage)+ return resolve_run_mode_for_boot(requested=requested)++def get_content_generator() -> ContentGeneratorProtocol:override = _content_generator_override.get()if override is not None:diff --git a/pipelex/system/trace_context.py b/pipelex/system/trace_context.pyindex 6cf2b235c..f16265e47 100644--- a/pipelex/system/trace_context.py+++ b/pipelex/system/trace_context.py@@ -17,6 +17,7 @@ The node tree is always built when either stream is on — including costs-onlyfrom pydantic import BaseModel, ConfigDict, Fieldfrom pipelex.system.data_inclusion_config import DataInclusionConfig+from pipelex.system.pipe_run_mode import PipeRunModeclass TraceContext(BaseModel):@@ -41,6 +42,9 @@ class TraceContext(BaseModel):describe_pipe_io: Whether the run builds the three I/O artifacts that describe its graphspec'sdata (driven by ``graphs_inclusion.graphspec_json``). Off for a run that writes nographspec, and for validate's own graph dry run, which already built them for its report.+ run_mode: Whether the run is live or a dry run. Every trace event the run emits states it, the+ usage events an activity emits included, which have this context and nothing else of the run.+ Required, with no default: a context built without it would stamp a dry run's events as live."""model_config = ConfigDict(strict=True, extra="forbid")@@ -53,6 +57,7 @@ class TraceContext(BaseModel):emit_graph_events: bool = Field(default=True, description="Whether to emit graph (node/edge) trace events")emit_usage_events: bool = Field(default=True, description="Whether to emit usage (cost) trace events")describe_pipe_io: bool = Field(default=True, description="Whether to build the I/O artifacts describing the graphspec's data")+ run_mode: PipeRunMode = Field(strict=False, description="Whether the run is live or a dry run, stated by whoever opens the trace")@propertydef lookup_key(self) -> str:@@ -86,4 +91,5 @@ class TraceContext(BaseModel):emit_graph_events=self.emit_graph_events,emit_usage_events=self.emit_usage_events,describe_pipe_io=self.describe_pipe_io,+ run_mode=self.run_mode,)diff --git a/pipelex/tracing/exceptions.py b/pipelex/tracing/exceptions.pyindex e6203cf1a..7da25a7d1 100644--- a/pipelex/tracing/exceptions.py+++ b/pipelex/tracing/exceptions.py@@ -41,3 +41,13 @@ class EventLogSetupError(EventLogError):cannot even be built degrades to an ``*_assembly_error`` note rather thanaborting the pipeline run."""+++class TraceEventModeError(PipelexError):+ """Raised when a set of trace events states no single run mode to build a graph in.++ Every trace event states the mode of the run that wrote it, `live` or `dry`, and the graph the event-replay+ assembler builds from a run's events takes that mode as its provenance. A set that mixes modes holds the events of+ more than one run, such as a dry run and a live run written under one pipeline run id, and an empty set states no+ mode at all: neither describes one run, so the assembler builds no graph from it rather than guess its mode.+ """diff --git a/pipelex/tracing/graphspec_assembler.py b/pipelex/tracing/graphspec_assembler.pyindex 2f0c1cd43..12e14add9 100644--- a/pipelex/tracing/graphspec_assembler.py+++ b/pipelex/tracing/graphspec_assembler.py@@ -3,7 +3,8 @@This is the core algorithm for distributed tracing — equivalent toGraphTracer.teardown() but working from serialized events instead ofin-memory state. Used after Temporal execution to assemble cross-worker-graphs from NDJSON event files.+graphs from NDJSON event files. The graph's provenance mode is the run mode+its events state, never one the caller passes."""from collections.abc import Sequence@@ -30,11 +31,15 @@ from pipelex.graph.graphspec import (make_graphspec_meta,output_digest_is_optional,)+from pipelex.graph.selected_outcome import derive_selected_outcome_edges+from pipelex.tracing.exceptions import TraceEventModeErrorfrom pipelex.tracing.trace_events import (UNATTRIBUTED_NODE_ID,BatchAggregateEvent,+ BatchFannedOutEvent,BatchItemEvent,ConditionOutputMergeEvent,+ ConditionResolvedEvent,ControllerOutputEvent,EdgeEvent,ExecutionDataEvent,@@ -124,7 +129,9 @@ class GraphSpecAssembler:- Pass 1: Build nodes, producer map, per-node usage totals, and collect metadatafrom all events.- Pass 2: Generate DATA, BATCH_ITEM, BATCH_AGGREGATE, PARALLEL_COMBINE edges- using the complete producer map, then attribute usage — resolve each usage+ using the complete producer map, and SELECTED_OUTCOME edges from each live+ condition's resolution to the pipe it then started, whichever workflow wrote+ that start, then attribute usage — resolve each usagetotal against the now-complete node set, roll subtrees up the parentage chain,and compute the run total.@@ -140,25 +147,46 @@ class GraphSpecAssembler:*,graph_id: str,pipeline_ref: PipelineRef | None = None,- mode: GraphSpecMode = GraphSpecMode.LIVE,) -> GraphSpec:"""Assemble a GraphSpec from a flat list of trace events.+ The GraphSpec's provenance mode is the run mode every event states.+Args:events: Flat list of trace events, pre-sorted by (workflow_id, sequence).graph_id: The graph identifier for the assembled GraphSpec.pipeline_ref: Optional pipeline reference metadata.- mode: Provenance mode to stamp onto the assembled GraphSpec.Returns:A complete GraphSpec with nodes and edges.++ Raises:+ TraceEventModeError: When the events mix modes, or there are none, and so state no single run mode."""+ mode = _graphspec_mode_of_events(events=events, graph_id=graph_id)assembler = _AssemblerState(graph_id=graph_id, pipeline_ref=pipeline_ref or PipelineRef(), mode=mode)assembler.pass_one(events)assembler.pass_two()return assembler.build_graph_spec()+def _graphspec_mode_of_events(*, events: Sequence[TraceEvent], graph_id: str) -> GraphSpecMode:+ """The provenance mode of the graph built from `events`: the one run mode they all state.++ Raises:+ TraceEventModeError: When the events mix modes, or there are none.+ """+ run_modes = {event.mode for event in events}+ if not run_modes:+ msg = f"No trace events to assemble graph '{graph_id}' from, so no run mode to state as its provenance."+ raise TraceEventModeError(msg)+ if len(run_modes) > 1:+ modes_str = ", ".join(sorted(run_modes))+ msg = f"The trace events of graph '{graph_id}' mix run modes ({modes_str}), so they hold more than one run; no graph is built from them."+ raise TraceEventModeError(msg)+ return run_modes.pop().graphspec_mode++class _AssemblerState:"""Mutable state for a single assembler invocation."""@@ -176,6 +204,9 @@ class _AssemblerState:self._batch_aggregate_map: dict[str, tuple[str | None, list[tuple[str, int]]]] = {}self._parallel_combine_map: dict[str, tuple[str, list[tuple[str, str]]]] = {}self._condition_output_merges: list[ConditionOutputMerge] = []+ # The live conditions that have chosen their outcome, by condition node id, each with the outcome-map key it+ # matched (None for the default outcome): pass 2 derives the SELECTED_OUTCOME edges from them+ self._condition_resolutions: dict[str, str | None] = {}# Registries (accumulated from events, deduplicated)self._pipe_registry: dict[str, dict[str, Any]] = {}@@ -221,6 +252,12 @@ class _AssemblerState:self._handle_execution_data(event)elif isinstance(event, UsageReportEvent):self._handle_usage_report(event)+ elif isinstance(event, ConditionResolvedEvent):+ self._handle_condition_resolved(event)+ elif isinstance(event, BatchFannedOutEvent):+ # A batch's width is for the readers following a run while it goes on: the graph already holds its+ # items, as the batch's children.+ passelse:log.warning("A trace event of a type the assembler does not handle was dropped", fields={"trace_event_type": type(event).__name__})@@ -233,6 +270,7 @@ class _AssemblerState:self._generate_batch_item_edges()self._generate_batch_aggregate_edges()self._generate_parallel_combine_edges()+ self._generate_selected_outcome_edges()self._attribute_usage()def build_graph_spec(self) -> GraphSpec:@@ -357,10 +395,10 @@ class _AssemblerState:self._stuff_producer_map[event.output_spec.digest] = event.node_iddef _handle_edge_event(self, event: EdgeEvent) -> None:- # DATA, BATCH_ITEM, BATCH_AGGREGATE, PARALLEL_COMBINE are regenerated+ # DATA, BATCH_ITEM, BATCH_AGGREGATE, PARALLEL_COMBINE and SELECTED_OUTCOME are regenerated# in pass 2 with full cross-worker visibility — skip to avoid duplicates.match event.edge_kind:- case EdgeKind.CONTAINS | EdgeKind.SELECTED_OUTCOME | EdgeKind.CONTROL:+ case EdgeKind.CONTAINS | EdgeKind.CONTROL:edge = EdgeSpec(edge_id=event.edge_id,source=event.source_node_id,@@ -372,7 +410,7 @@ class _AssemblerState:target_stuff_digest=event.target_stuff_digest,)self._explicit_edges.append(edge)- case EdgeKind.DATA | EdgeKind.BATCH_ITEM | EdgeKind.BATCH_AGGREGATE | EdgeKind.PARALLEL_COMBINE:+ case EdgeKind.DATA | EdgeKind.BATCH_ITEM | EdgeKind.BATCH_AGGREGATE | EdgeKind.PARALLEL_COMBINE | EdgeKind.SELECTED_OUTCOME:pass # Regenerated in pass 2def _handle_controller_output(self, event: ControllerOutputEvent) -> None:@@ -429,6 +467,10 @@ class _AssemblerState:def _handle_condition_output_merge(self, event: ConditionOutputMergeEvent) -> None:self._condition_output_merges.append(event.merge)+ def _handle_condition_resolved(self, event: ConditionResolvedEvent) -> None:+ # No node lookup: the condition's start can come from another workflow, read later in the stream.+ self._condition_resolutions[event.node_id] = event.selected_outcome+def _mark_canceled_nodes(self) -> None:"""Mark any still-RUNNING nodes as CANCELED (mirrors graph_tracer.py:160-164)."""for node_data in self._nodes.values():@@ -579,6 +621,22 @@ class _AssemblerState:target_stuff_digest=combined_stuff_code,)+ def _generate_selected_outcome_edges(self) -> None:+ """Generate the SELECTED_OUTCOME edges from each resolved condition to the pipe it then started.++ Mirrors GraphTracer._generate_selected_outcome_edges(), through the same rule. The chosen pipe's start can come+ from a workflow other than the condition's, under a tracer that never saw the resolution, so the edge is derived+ here from both events rather than read from an edge event one tracer would have had to draw.+ """+ parent_by_node = {node_id: node_data.parent_node_id for node_id, node_data in self._nodes.items()}+ for edge in derive_selected_outcome_edges(resolutions=self._condition_resolutions, parent_by_node=parent_by_node):+ self._add_generated_edge(+ source_node_id=edge.condition_node_id,+ target_node_id=edge.outcome_node_id,+ edge_kind=EdgeKind.SELECTED_OUTCOME,+ label=edge.selected_outcome,+ )+# ------------------------------------------------------------------# Pass 2 usage attribution# ------------------------------------------------------------------diff --git a/pipelex/tracing/trace_events.py b/pipelex/tracing/trace_events.pyindex 997f666f5..1135d80de 100644--- a/pipelex/tracing/trace_events.py+++ b/pipelex/tracing/trace_events.py@@ -14,6 +14,7 @@ from pydantic import BaseModel, Field, ValidationErrorfrom pipelex.graph.condition_output_merge import ConditionOutputMergefrom pipelex.graph.graphspec import EdgeKind, ErrorSpec, IOSpec, NodeKindfrom pipelex.reporting.reporting_types import AnyTokensUsage+from pipelex.system.pipe_run_mode import PipeRunModefrom pipelex.tools.typing.pydantic_utils import empty_list_factory_of@@ -30,6 +31,8 @@ class TraceEventKind(StrEnum):BATCH_AGGREGATE = "batch_aggregate"PARALLEL_COMBINE = "parallel_combine"CONDITION_OUTPUT_MERGE = "condition_output_merge"+ BATCH_FANNED_OUT = "batch_fanned_out"+ CONDITION_RESOLVED = "condition_resolved"EXECUTION_DATA = "execution_data"USAGE_REPORT = "usage_report"@@ -47,6 +50,8 @@ class TraceEvent(BaseModel):sort-key shape; non-default values come from the per-process activityevent log used by separate-process workers.- event_kind: discriminator for the event subclass (defined on each subclass as Literal)+ - mode: whether the run that emitted the event is live or a dry run. Every event of one run+ carries the same mode, so a reader tells dry events from live ones by this field alone.- timestamp: UTC, for display/debugging only- sequence: per-writer monotonic counter, for ordering and deduplication"""@@ -54,6 +59,7 @@ class TraceEvent(BaseModel):pipeline_run_id: strworkflow_id: strwriter_id: str = "primary"+ mode: PipeRunModetimestamp: datetimesequence: int@@ -64,7 +70,7 @@ class TraceEvent(BaseModel):class PipeStartEvent(TraceEvent):- """Emitted when a pipe begins execution."""+ """Emitted when a pipe begins execution, before it runs anything."""event_kind: Literal[TraceEventKind.PIPE_START] = TraceEventKind.PIPE_STARTnode_id: str@@ -77,6 +83,11 @@ class PipeStartEvent(TraceEvent):input_specs: list[IOSpec] = Field(default_factory=empty_list_factory_of(IOSpec))pipe_data: dict[str, Any] = Field(default_factory=dict)concept_data: list[dict[str, Any]] = Field(default_factory=empty_list_factory_of(dict))+ step_count: int | None = None+ """On a PipeSequence, the number of steps its author wrote. The binding a sequence writes for a dotted+ `batch_over` is no step of the author's and is not counted. None on every other pipe."""+ branch_count: int | None = None+ """On a PipeParallel, the number of its branches. None on every other pipe."""class PipeEndSuccessEvent(TraceEvent):@@ -183,6 +194,42 @@ class ConditionOutputMergeEvent(TraceEvent):merge: ConditionOutputMerge+class BatchFannedOutEvent(TraceEvent):+ """Emitted when a PipeBatch knows how many items it runs, before it starts any of them.++ A live reader learns the batch's width the moment it is fixed, so it never has to guess it from the+ items that have started.+ """++ event_kind: Literal[TraceEventKind.BATCH_FANNED_OUT] = TraceEventKind.BATCH_FANNED_OUT+ node_id: str+ """The PipeBatch's own node."""+ item_count: int = Field(ge=0)+ branch_pipe_code: str+ """The bare code of the pipe each item runs."""+ branch_pipe_ref: str+ """The `domain.code` reference of the pipe each item runs."""+ branch_pipe_type: str+ """The type of the pipe each item runs, such as `PipeSequence` or `PipeLLM`."""+++class ConditionResolvedEvent(TraceEvent):+ """Emitted when a live PipeCondition has chosen its outcome, before the chosen pipe starts.++ A dry run emits none, since its condition runs the pipe of every outcome.+ """++ event_kind: Literal[TraceEventKind.CONDITION_RESOLVED] = TraceEventKind.CONDITION_RESOLVED+ node_id: str+ """The PipeCondition's own node."""+ rendered_value: str+ """The value the condition's expression rendered. It may quote text of the method, or of a library the method+ depends on, so it is kept for whoever reads the trace and never reaches a caller-facing summary."""+ selected_outcome: str | None+ """The key of the outcome map the rendered value matched, or None when it matched none and the condition took its+ default outcome: `"default"` can itself be a key of an outcome map, so it cannot stand for the default."""++class ExecutionDataEvent(TraceEvent):"""Emitted when a pipe registers execution metadata (rendered prompts, resolved models, etc.)."""@@ -220,6 +267,8 @@ AnyTraceEvent = Annotated[| BatchAggregateEvent| ParallelCombineEvent| ConditionOutputMergeEvent+ | BatchFannedOutEvent+ | ConditionResolvedEvent| ExecutionDataEvent| UsageReportEvent,Field(discriminator="event_kind"),diff --git a/subject_grants.toml b/subject_grants.tomlindex a9df7958d..d1d6a0413 100644--- a/subject_grants.toml+++ b/subject_grants.toml@@ -6932,6 +6932,10 @@ rationale = "Dispatch handler: handles the event; single operand."param = "event"rationale = "An event handler of the assembler's pass one, called with the one event it folds in, like its sibling handlers."+["pipelex/tracing/graphspec_assembler.py::_AssemblerState._handle_condition_resolved"]+param = "event"+rationale = "An event handler of the assembler's pass one, called with the one event it folds in, like its sibling handlers."+["pipelex/tracing/graphspec_assembler.py::_AssemblerState._handle_controller_output"]param = "event"rationale = "Dispatch handler: handles the event; single operand."diff --git a/tests/data/errors/error_identity.txt b/tests/data/errors/error_identity.txtindex 043dcd040..700782a19 100644--- a/tests/data/errors/error_identity.txt+++ b/tests/data/errors/error_identity.txt@@ -338,6 +338,7 @@ TelemetryConfigValidationError | Telemetry config validation | https://docs.pipeTemplateSigilSyntaxError | Template sigil syntax | https://docs.pipelex.com/latest/errors/template-sigil-syntax-error/TomlError | TOML parse error | https://docs.pipelex.com/latest/errors/toml-error/ToolError | Tool error | https://docs.pipelex.com/latest/errors/tool-error/+TraceEventModeError | Trace event mode | https://docs.pipelex.com/latest/errors/trace-event-mode-error/TracebackMessageError | Traceback message | https://docs.pipelex.com/latest/errors/traceback-message-error/TypesafeError | Typesafe | https://docs.pipelex.com/latest/errors/typesafe-error/TypesafeJudgmentResponseError | Typesafe judgment response | https://docs.pipelex.com/latest/errors/typesafe-judgment-response-error/diff --git a/tests/helpers/usage_stats_run_events.py b/tests/helpers/usage_stats_run_events.pyindex 3cb9abd2b..a545d53d3 100644--- a/tests/helpers/usage_stats_run_events.py+++ b/tests/helpers/usage_stats_run_events.py@@ -8,6 +8,7 @@ from pipelex.cogt.usage.cost_category import CostCategoryfrom pipelex.cogt.usage.token_category import TokenCategoryfrom pipelex.graph.graphspec import NodeKindfrom pipelex.system.job_metadata import JobCategory, JobMetadata, RunMetadata, UnitJobId+from pipelex.system.pipe_run_mode import PipeRunModefrom pipelex.system.usage_stats.usage_stats_contract import UsageStatsArch, UsageStatsIntegration, UsageStatsOs, UsageStatsSurfacefrom pipelex.system.usage_stats.usage_stats_events import CommonProperties, UsageStatsEvent, make_envelopefrom pipelex.tracing.trace_events import PipeStartEvent, UsageReportEvent@@ -50,6 +51,7 @@ def llm_usage_event(workflow_id="direct",timestamp=datetime.now(UTC),sequence=sequence,+ mode=PipeRunMode.LIVE,node_id=f"{USAGE_STATS_RUN_ID}:node_{sequence}",tokens_usage=tokens_usage,)@@ -61,6 +63,7 @@ def pipe_start_event(*, pipe_type: str, sequence: int) -> PipeStartEvent:workflow_id="direct",timestamp=datetime.now(UTC),sequence=sequence,+ mode=PipeRunMode.LIVE,node_id=f"{USAGE_STATS_RUN_ID}:node_{sequence}",pipe_code=f"secret_pipe_{sequence}",pipe_type=pipe_type,diff --git a/tests/integration/pipelex/kernel/test_usage_parity.py b/tests/integration/pipelex/kernel/test_usage_parity.pyindex 31cfbf7ec..3b728d33c 100644--- a/tests/integration/pipelex/kernel/test_usage_parity.py+++ b/tests/integration/pipelex/kernel/test_usage_parity.py@@ -118,6 +118,7 @@ class TestKernelUsageParity:data_inclusion=get_config().interpreter.pipeline_execution.graph.data_inclusion,emit_graph_events=False,emit_usage_events=True,+ run_mode=PipeRunMode.DRY,)get_report_delegate().set_event_log(context_key=trace_context.lookup_key,diff --git a/tests/integration/pipelex/pipeline/test_decisions_at_decision_time.py b/tests/integration/pipelex/pipeline/test_decisions_at_decision_time.pynew file mode 100644index 000000000..fcf5a3774--- /dev/null+++ b/tests/integration/pipelex/pipeline/test_decisions_at_decision_time.py@@ -0,0 +1,285 @@+"""A run's NDJSON trace states each controller decision before the work under it, and every event states its mode.++The trace is read as a reader tailing the file would read it: line by line, in the order the run wrote it. A condition's+resolution comes before the first event of the pipe it chose, a batch's fan-out before the first event of its items, a+sequence and a parallel state their width on their own start, and every line says whether the run is live or dry.+"""++import json+from pathlib import Path+from typing import TYPE_CHECKING, Any++import pytest+from pytest_mock import MockerFixture++import pipelex.pipeline.pipeline_run_setup as pipeline_run_setup_module+from pipelex.config import get_config+from pipelex.pipeline.runner import PipelexMTHDSProtocol+from pipelex.runtime_hub import scoped_event_log+from pipelex.system.pipe_run_mode import PipeRunMode+from pipelex.tracing.ndjson_event_log import NdjsonEventLog++if TYPE_CHECKING:+ from pipelex.pipe_run.pipe_job import PipeJob++_DOMAIN = "decisions_at_decision_time"+_BUNDLE = f"""+domain = "{_DOMAIN}"+description = "Route a request, answer every ticket of an inbox, then weigh the outcome two ways"+main_pipe = "triage"++[concept.Ticket]+description = "A support ticket"++[concept.Ticket.structure]+subject = {{ type = "text", description = "What the ticket is about", required = true }}++[concept.Inbox]+description = "An inbox of support tickets"++[concept.Inbox.structure]+tickets = {{ type = "list", item_type = "concept", item_concept_ref = "Ticket", description = "The tickets", required = true }}++[pipe.triage]+type = "PipeSequence"+description = "Route, answer each ticket, weigh, conclude"+inputs = {{ request = "Text", inbox = "Inbox" }}+output = "Text"+steps = [+ {{ pipe = "route", result = "routed" }},+ {{ pipe = "answer_ticket", batch_over = "inbox.tickets", batch_as = "ticket", result = "answers" }},+ {{ pipe = "weigh", result = "weighed" }},+ {{ pipe = "conclude", result = "conclusion" }},+]++[pipe.route]+type = "PipeCondition"+description = "Escalate an urgent request, file any other"+inputs = {{ request = "Text" }}+output = "Text"+expression = "request"+outcomes = {{ urgent = "escalate" }}+default_outcome = "file_away"++[pipe.escalate]+type = "PipeCompose"+description = "Escalate the request"+inputs = {{ request = "Text" }}+output = "Text"+template = "Escalated: {{{{ request }}}}"++[pipe.file_away]+type = "PipeCompose"+description = "File the request"+inputs = {{ request = "Text" }}+output = "Text"+template = "Filed: {{{{ request }}}}"++[pipe.answer_ticket]+type = "PipeCompose"+description = "Answer one ticket"+inputs = {{ ticket = "Ticket" }}+output = "Text"+template = "Answer to {{{{ ticket.subject }}}}"++[pipe.weigh]+type = "PipeParallel"+description = "Weigh the routed request two ways at once"+inputs = {{ routed = "Text" }}+output = "Composite"+add_each_output = true+branches = [+ {{ pipe = "weigh_short", result = "short_view" }},+ {{ pipe = "weigh_long", result = "long_view" }},+]++[pipe.weigh_short]+type = "PipeCompose"+description = "The short view"+inputs = {{ routed = "Text" }}+output = "Text"+template = "Short: {{{{ routed }}}}"++[pipe.weigh_long]+type = "PipeCompose"+description = "The long view"+inputs = {{ routed = "Text" }}+output = "Text"+template = "Long: {{{{ routed }}}}"++[pipe.conclude]+type = "PipeCompose"+description = "Conclude on both views and the answers"+inputs = {{ short_view = "Text", long_view = "Text", answers = "Text[]" }}+output = "Text"+template = "{{{{ short_view }}}} / {{{{ long_view }}}} / {{{{ answers }}}}"+"""++_INBOX = {"tickets": [{"subject": "Login"}, {"subject": "Billing"}]}+++def _read_trace_lines(traces_dir: Path) -> list[dict[str, Any]]:+ """Every line of the run's one NDJSON file, in the order the run wrote them."""+ ndjson_files = list(traces_dir.rglob("*.ndjson"))+ assert len(ndjson_files) == 1, ndjson_files+ return [json.loads(line) for line in ndjson_files[0].read_text(encoding="utf-8").splitlines() if line.strip()]+++def _descendants(*, lines: list[dict[str, Any]], node_id: str) -> set[str]:+ """The node IDs of every pipe started under `node_id`, at any depth."""+ children_by_parent: dict[str, list[str]] = {}+ for line in lines:+ if line["event_kind"] == "pipe_start" and line["parent_node_id"] is not None:+ children_by_parent.setdefault(line["parent_node_id"], []).append(line["node_id"])+ found: set[str] = set()+ pending = [node_id]+ while pending:+ for child_id in children_by_parent.get(pending.pop(), []):+ found.add(child_id)+ pending.append(child_id)+ return found+++def _first_line_under(*, lines: list[dict[str, Any]], node_id: str) -> int:+ """The position of the first line naming a pipe started under `node_id`: its start, an edge into it, or its data."""+ under = _descendants(lines=lines, node_id=node_id)+ assert under, f"nothing ran under {node_id}"+ return min(+ position+ for position, line in enumerate(lines)+ if line.get("node_id") in under or line.get("target_node_id") in under or line.get("batch_controller_node_id") in under+ )+++def _one(*, lines: list[dict[str, Any]], event_kind: str) -> tuple[int, dict[str, Any]]:+ found = [(position, line) for position, line in enumerate(lines) if line["event_kind"] == event_kind]+ assert len(found) == 1, f"{event_kind}: {found}"+ return found[0]+++def _starts_by_code(lines: list[dict[str, Any]]) -> dict[str, list[dict[str, Any]]]:+ starts: dict[str, list[dict[str, Any]]] = {}+ for line in lines:+ if line["event_kind"] == "pipe_start":+ starts.setdefault(line["pipe_code"], []).append(line)+ return starts++[email protected](loop_scope="class")+class TestDecisionsAtDecisionTime:+ @pytest.mark.parametrize(+ ("request_text", "selected_outcome", "chosen_pipe_code"),+ [("urgent", "urgent", "escalate"), ("routine", None, "file_away")],+ ids=["matched_key", "default_outcome"],+ )+ async def test_live_trace_states_each_decision_before_the_work_under_it(+ self, tmp_path: Path, request_text: str, selected_outcome: str | None, chosen_pipe_code: str+ ) -> None:+ execution_config = get_config().interpreter.pipeline_execution.with_execution_overrides(generate_graph=True)+ runner = PipelexMTHDSProtocol(execution_config=execution_config)++ with scoped_event_log(NdjsonEventLog(traces_dir=str(tmp_path))):+ response = await runner.execute(+ mthds_contents=[_BUNDLE],+ inputs={"request": request_text, "inbox": {"concept": f"{_DOMAIN}.Inbox", "content": _INBOX}},+ )++ prefix = "Escalated" if chosen_pipe_code == "escalate" else "Filed"+ assert response.pipe_output.main_stuff.as_text.text.startswith(f"Short: {prefix}: {request_text} / Long: {prefix}: {request_text} / ")+ lines = _read_trace_lines(tmp_path)++ # Every line states the run's mode, and so does the graph built from them.+ assert {line["mode"] for line in lines} == {"live"}+ assert response.pipe_output.graph_spec is not None+ assert response.pipe_output.graph_spec.meta["mode"] == "live"++ # The condition's resolution precedes the first event of the pipe it chose.+ resolution_position, resolution = _one(lines=lines, event_kind="condition_resolved")+ starts = _starts_by_code(lines)+ route_start = starts["route"][0]+ assert resolution["node_id"] == route_start["node_id"]+ assert resolution["rendered_value"] == request_text+ assert resolution["selected_outcome"] == selected_outcome+ assert resolution_position < _first_line_under(lines=lines, node_id=route_start["node_id"])+ chosen_start = starts[chosen_pipe_code][0]+ assert chosen_start["parent_node_id"] == route_start["node_id"]+ # The graph built from those events draws the choice as its SELECTED_OUTCOME edge, labelled with the key.+ graph_spec = response.pipe_output.graph_spec+ assert graph_spec is not None+ assert [(edge.source, edge.target, edge.label) for edge in graph_spec.edges if edge.kind.is_selected_outcome] == [+ (route_start["node_id"], chosen_start["node_id"], selected_outcome)+ ]++ # The batch's fan-out precedes the first event of its items.+ fan_out_position, fan_out = _one(lines=lines, event_kind="batch_fanned_out")+ batch_start = next(line for line in lines if line["event_kind"] == "pipe_start" and line["node_id"] == fan_out["node_id"])+ assert batch_start["pipe_type"] == "PipeBatch"+ assert (fan_out["item_count"], fan_out["branch_pipe_code"], fan_out["branch_pipe_ref"], fan_out["branch_pipe_type"]) == (+ 2,+ "answer_ticket",+ f"{_DOMAIN}.answer_ticket",+ "PipeCompose",+ )+ assert fan_out_position < _first_line_under(lines=lines, node_id=fan_out["node_id"])+ assert len(starts["answer_ticket"]) == 2++ # A sequence states the steps its author wrote, the binding of the dotted `batch_over` not among them, and a+ # parallel its branches, each on its own start; no other pipe states either.+ assert (starts["triage"][0]["step_count"], starts["triage"][0]["branch_count"]) == (4, None)+ assert (starts["weigh"][0]["step_count"], starts["weigh"][0]["branch_count"]) == (None, 2)+ for pipe_code, code_starts in starts.items():+ if pipe_code not in {"triage", "weigh"}:+ assert all(start["step_count"] is None and start["branch_count"] is None for start in code_starts), pipe_code++ async def test_dry_trace_states_its_mode_and_resolves_no_condition(self, tmp_path: Path) -> None:+ """A dry run runs every outcome of a condition and chooses none, so it records no resolution."""+ execution_config = get_config().interpreter.pipeline_execution.with_execution_overrides(generate_graph=True, mock_inputs=True)+ runner = PipelexMTHDSProtocol(pipe_run_mode=PipeRunMode.DRY, execution_config=execution_config)++ with scoped_event_log(NdjsonEventLog(traces_dir=str(tmp_path))):+ response = await runner.execute(mthds_contents=[_BUNDLE])++ lines = _read_trace_lines(tmp_path)+ assert {line["mode"] for line in lines} == {"dry"}+ assert [line for line in lines if line["event_kind"] == "condition_resolved"] == []+ graph_spec = response.pipe_output.graph_spec+ assert graph_spec is not None+ assert graph_spec.meta["mode"] == "dry"+ assert [edge for edge in graph_spec.edges if edge.kind.is_selected_outcome] == []+ starts = _starts_by_code(lines)+ assert starts["triage"][0]["step_count"] == 4+ assert starts["weigh"][0]["branch_count"] == 2++ async def test_a_live_request_on_a_keyless_boot_is_dry_throughout(self, tmp_path: Path, mocker: MockerFixture) -> None:+ """A keyless boot runs a requested live run dry: its job, its trace context, every event and its graph all say dry."""+ mocker.patch("pipelex.runtime_hub.is_dry_run_forced", return_value=True)+ prepare_pipe_job_spy = mocker.spy(pipeline_run_setup_module, "prepare_pipe_job")+ execution_config = get_config().interpreter.pipeline_execution.with_execution_overrides(generate_graph=True)+ runner = PipelexMTHDSProtocol(pipe_run_mode=PipeRunMode.LIVE, execution_config=execution_config)++ with scoped_event_log(NdjsonEventLog(traces_dir=str(tmp_path))):+ response = await runner.execute(+ mthds_contents=[_BUNDLE],+ inputs={"request": "urgent", "inbox": {"concept": f"{_DOMAIN}.Inbox", "content": _INBOX}},+ )++ pipe_job: PipeJob = prepare_pipe_job_spy.spy_return+ assert pipe_job.pipe_run_params.run_mode.is_dry+ trace_context = pipe_job.job_metadata.trace_context+ assert trace_context is not None+ assert trace_context.run_mode.is_dry+ lines = _read_trace_lines(tmp_path)+ assert {line["mode"] for line in lines} == {"dry"}+ graph_spec = response.pipe_output.graph_spec+ assert graph_spec is not None+ assert graph_spec.meta["mode"] == "dry"+ # The pipes ran dry too: the condition ran every outcome and resolved none.+ assert [line for line in lines if line["event_kind"] == "condition_resolved"] == []++ async def test_a_keyless_boot_still_refuses_mock_usage_on_a_live_request(self, mocker: MockerFixture) -> None:+ """The requested mode is validated before the keyless coercion, so it cannot legalise mock usage on a live request."""+ mocker.patch("pipelex.runtime_hub.is_dry_run_forced", return_value=True)+ runner = PipelexMTHDSProtocol(pipe_run_mode=PipeRunMode.LIVE, is_mock_usage=True)++ with pytest.raises(ValueError, match="is_mock_usage"):+ await runner.execute(mthds_contents=[_BUNDLE], inputs={"request": "urgent"})diff --git a/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_method_faults.py b/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_method_faults.pyindex 7f0c5f5a6..66a3f8349 100644--- a/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_method_faults.py+++ b/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_method_faults.py@@ -7,6 +7,7 @@ its message under STRICT disclosure with a next step, and quotes no text of thehost library's."""+from datetime import UTC, datetimefrom pathlib import Pathfrom typing import Any, Callable@@ -17,6 +18,8 @@ from pipelex.core.concepts.native.concept_native import NativeConceptCodefrom pipelex.core.domains.domain import SpecialDomainfrom pipelex.core.memory.working_memory_factory import WorkingMemoryFactoryfrom pipelex.core.pipes.exceptions import PipeRunError+from pipelex.graph.graph_tracer_manager import GraphTracerManager+from pipelex.graph.graphspec import NodeKindfrom pipelex.pipe_controllers.condition.pipe_condition import PipeConditionfrom pipelex.pipe_controllers.condition.pipe_condition_blueprint import PipeConditionBlueprintfrom pipelex.pipe_machinery.pipe_factory import PipeFactory@@ -24,6 +27,9 @@ from pipelex.pipe_run.located_failure import find_root_faultfrom pipelex.pipe_run.pipe_run_params_factory import PipeRunParamsFactoryfrom pipelex.system.job_metadata import JobMetadatafrom pipelex.system.pipe_run_mode import PipeRunMode+from pipelex.tracing.in_memory_event_log import InMemoryEventLog+from pipelex.tracing.trace_events import ConditionResolvedEvent, ExecutionDataEvent+from tests.unit.pipelex.graph.conftest import make_defaulted_data_inclusion_config_TEXT = f"{SpecialDomain.NATIVE}.{NativeConceptCode.TEXT}"_CONDITION_LIBRARY = Path("tests/integration/pipelex/pipes/controller/pipe_condition")@@ -79,6 +85,77 @@ class TestPipeConditionMethodFaults:),}+ @pytest.mark.parametrize(+ ("outcome", "resolution"),+ [+ ("test_integration2.capitalize_long_text", None),+ ("fail", ("long", "long")),+ ],+ ids=["chosen_pipe_missing_its_inputs", "fail_outcome"],+ )+ async def test_the_trace_records_a_choice_only_once_it_is_final(+ self,+ job_metadata: JobMetadata,+ load_test_library: Callable[[list[Path]], None],+ outcome: str,+ resolution: tuple[str, str] | None,+ ) -> None:+ """A chosen pipe whose inputs are missing fails the condition before it runs: the trace records no outcome taken.++ A `fail` outcome is the condition's own final choice, so its resolution is recorded before the condition raises.+ Neither starts a pipe under the condition, so the graph draws no SELECTED_OUTCOME edge.+ """+ load_test_library([_CONDITION_LIBRARY])+ pipe_condition = _make_condition(+ pipe_code="length_gate",+ expression_template="long",+ outcomes={"long": outcome},+ default_outcome="test_integration2.add_prefix_short_text",+ )+ pipeline_run_id = job_metadata.run_metadata.pipeline_run_id+ event_log = InMemoryEventLog()+ tracer_manager = GraphTracerManager.get_or_create_instance()+ root_context = tracer_manager.open_tracer(+ graph_id=pipeline_run_id,+ data_inclusion=make_defaulted_data_inclusion_config(),+ event_log=event_log,+ pipeline_run_id=pipeline_run_id,+ run_mode=PipeRunMode.LIVE,+ )+ try:+ condition_node_id, condition_context = tracer_manager.on_pipe_start(+ trace_context=root_context,+ pipe_code=pipe_condition.code,+ pipe_type=pipe_condition.type,+ node_kind=NodeKind.CONTROLLER,+ started_at=datetime.now(UTC),+ )+ assert condition_node_id is not None+ with pytest.raises(PipeRunError):+ await pipe_condition._live_run_controller_pipe( # ruff: ignore[private-member-access] # pyright: ignore[reportPrivateUsage]+ job_metadata=job_metadata.model_copy(update={"trace_context": condition_context}),+ working_memory=WorkingMemoryFactory.make_empty(),+ pipe_run_params=PipeRunParamsFactory.make_run_params(pipe_run_mode=PipeRunMode.LIVE),+ )+ finally:+ graph_spec = tracer_manager.close_tracer(pipeline_run_id)++ events = event_log.read_events(pipeline_run_id)+ resolutions = [(event.rendered_value, event.selected_outcome) for event in events if isinstance(event, ConditionResolvedEvent)]+ selected_outcomes = [+ event.execution_data["selected_outcome"]+ for event in events+ if isinstance(event, ExecutionDataEvent) and "selected_outcome" in event.execution_data+ ]+ if resolution is None:+ assert resolutions == []+ assert selected_outcomes == []+ else:+ assert resolutions == [resolution]+ assert selected_outcomes == [outcome]+ assert graph_spec is not None+ assert [edge for edge in graph_spec.edges if edge.kind.is_selected_outcome] == []+async def test_an_expression_that_does_not_parse_names_its_line_only(self, job_metadata: JobMetadata, load_test_library: Callable[[list[Path]], None]) -> None:diff --git a/tests/unit/pipelex/graph/conftest.py b/tests/unit/pipelex/graph/conftest.pyindex ac394f073..3f3551bc6 100644--- a/tests/unit/pipelex/graph/conftest.py+++ b/tests/unit/pipelex/graph/conftest.py@@ -5,6 +5,7 @@ import pytestfrom pipelex.config import get_configfrom pipelex.graph.graph_config import GraphConfigfrom pipelex.system.data_inclusion_config import DataInclusionConfig+from pipelex.system.pipe_run_mode import PipeRunModefrom pipelex.system.trace_context import TraceContext@@ -57,6 +58,7 @@ def make_trace_context(stuff_json_content=stuff_json_content,error_stack_traces=error_stack_traces,),+ run_mode=PipeRunMode.LIVE,)diff --git a/tests/unit/pipelex/graph/test_graph_tracer.py b/tests/unit/pipelex/graph/test_graph_tracer.pyindex 0e0af3954..b7f35d480 100644--- a/tests/unit/pipelex/graph/test_graph_tracer.py+++ b/tests/unit/pipelex/graph/test_graph_tracer.py@@ -4,6 +4,7 @@ from datetime import UTC, datetime, timedeltafrom pipelex.graph.graph_tracer import GraphTracerfrom pipelex.graph.graphspec import EdgeKind, GraphSpecMode, IOSpec, NodeKind, NodeStatus+from pipelex.system.pipe_run_mode import PipeRunModefrom tests.unit.pipelex.graph.conftest import make_defaulted_data_inclusion_config@@ -13,7 +14,7 @@ class TestGraphTracer:def test_setup_returns_initial_context(self) -> None:"""Test that setup returns a properly initialized TraceContext."""tracer = GraphTracer()- context = tracer.setup(graph_id="test-graph-001", data_inclusion=make_defaulted_data_inclusion_config())+ context = tracer.setup(graph_id="test-graph-001", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)assert context.graph_id == "test-graph-001"assert context.parent_node_id is None@@ -27,6 +28,7 @@ class TestGraphTracer:data_inclusion=make_defaulted_data_inclusion_config(),pipeline_ref_domain="test.domain",pipeline_ref_main_pipe="main_pipe",+ run_mode=PipeRunMode.LIVE,)assert context.graph_id == "test-graph-002"@@ -39,11 +41,12 @@ class TestGraphTracer:def test_setup_with_dry_mode_stamps_graphspec_meta(self) -> None:tracer = GraphTracer()- tracer.setup(+ context = tracer.setup(graph_id="test-graph-dry",data_inclusion=make_defaulted_data_inclusion_config(),- mode=GraphSpecMode.DRY,+ run_mode=PipeRunMode.DRY,)+ assert context.run_mode == PipeRunMode.DRYgraph_spec = tracer.teardown()@@ -61,7 +64,7 @@ class TestGraphTracer:def test_full_lifecycle_single_node(self) -> None:"""Test tracking a single pipe execution."""tracer = GraphTracer()- context = tracer.setup(graph_id="lifecycle-test", data_inclusion=make_defaulted_data_inclusion_config())+ context = tracer.setup(graph_id="lifecycle-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)started_at = datetime.now(UTC)node_id, child_context = tracer.on_pipe_start(@@ -103,7 +106,7 @@ class TestGraphTracer:def test_nested_pipe_execution(self) -> None:"""Test tracking nested pipe execution with containment edges."""tracer = GraphTracer()- context = tracer.setup(graph_id="nested-test", data_inclusion=make_defaulted_data_inclusion_config())+ context = tracer.setup(graph_id="nested-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)# Start parent (sequence controller)started_at = datetime.now(UTC)@@ -164,7 +167,7 @@ class TestGraphTracer:def test_error_tracking(self) -> None:"""Test tracking failed pipe execution."""tracer = GraphTracer()- context = tracer.setup(graph_id="error-test", data_inclusion=make_defaulted_data_inclusion_config())+ context = tracer.setup(graph_id="error-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)started_at = datetime.now(UTC)node_id, _ = tracer.on_pipe_start(@@ -196,7 +199,7 @@ class TestGraphTracer:def test_skipped_tracking(self) -> None:"""A lifted (skipped) pipe ends its node in the SKIPPED state with the skip reason."""tracer = GraphTracer()- context = tracer.setup(graph_id="skip-test", data_inclusion=make_defaulted_data_inclusion_config())+ context = tracer.setup(graph_id="skip-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)started_at = datetime.now(UTC)node_id, _ = tracer.on_pipe_start(@@ -227,7 +230,7 @@ class TestGraphTracer:rides the skip and registers in the producer map, so the downstream DATA edge resolves."""tracer = GraphTracer()- context = tracer.setup(graph_id="skip-plural-edge-test", data_inclusion=make_defaulted_data_inclusion_config())+ context = tracer.setup(graph_id="skip-plural-edge-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)started_at = datetime.now(UTC)producer_id, _ = tracer.on_pipe_start(@@ -269,7 +272,7 @@ class TestGraphTracer:def test_data_edge_from_optional_output_carries_marker(self) -> None:"""A DATA edge whose producer output is declared optional (`?`) reports optional=True."""tracer = GraphTracer()- context = tracer.setup(graph_id="optional-edge-test", data_inclusion=make_defaulted_data_inclusion_config())+ context = tracer.setup(graph_id="optional-edge-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)started_at = datetime.now(UTC)producer_id, _ = tracer.on_pipe_start(@@ -307,7 +310,7 @@ class TestGraphTracer:def test_running_nodes_marked_canceled_on_teardown(self) -> None:"""Test that running nodes are marked as canceled on teardown."""tracer = GraphTracer()- context = tracer.setup(graph_id="cancel-test", data_inclusion=make_defaulted_data_inclusion_config())+ context = tracer.setup(graph_id="cancel-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)started_at = datetime.now(UTC)_node_id, _ = tracer.on_pipe_start(@@ -330,7 +333,7 @@ class TestGraphTracer:def test_custom_edges(self) -> None:"""Test adding custom edges between nodes."""tracer = GraphTracer()- context = tracer.setup(graph_id="edge-test", data_inclusion=make_defaulted_data_inclusion_config())+ context = tracer.setup(graph_id="edge-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)started_at = datetime.now(UTC)node1_id, _ = tracer.on_pipe_start(@@ -372,7 +375,7 @@ class TestGraphTracer:def test_selected_outcome_edge(self) -> None:"""Test adding selected outcome edge for conditions."""tracer = GraphTracer()- context = tracer.setup(graph_id="condition-test", data_inclusion=make_defaulted_data_inclusion_config())+ context = tracer.setup(graph_id="condition-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)started_at = datetime.now(UTC)condition_id, cond_ctx = tracer.on_pipe_start(@@ -396,7 +399,7 @@ class TestGraphTracer:tracer.add_selected_outcome_edge(condition_node_id=condition_id,outcome_node_id=outcome_id,- outcome_value="true",+ selected_outcome="true",)tracer.on_pipe_end_success(node_id=condition_id, ended_at=started_at + timedelta(milliseconds=25))@@ -411,7 +414,7 @@ class TestGraphTracer:def test_input_specs_captured(self) -> None:"""Test that input IOSpecs are captured and stored in node_io."""tracer = GraphTracer()- context = tracer.setup(graph_id="io-test", data_inclusion=make_defaulted_data_inclusion_config())+ context = tracer.setup(graph_id="io-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)input_specs = [IOSpec(name="document", concept="Text", digest="abc12"),@@ -442,7 +445,7 @@ class TestGraphTracer:def test_output_spec_captured(self) -> None:"""Test that output IOSpec is captured and stored in node_io."""tracer = GraphTracer()- context = tracer.setup(graph_id="io-test", data_inclusion=make_defaulted_data_inclusion_config())+ context = tracer.setup(graph_id="io-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)output_spec = IOSpec(name="summary",@@ -477,7 +480,7 @@ class TestGraphTracer:def test_data_edge_generation_from_stuff_codes(self) -> None:"""Test that DATA edges are created when stuff_codes match between output and input."""tracer = GraphTracer()- context = tracer.setup(graph_id="data-flow-test", data_inclusion=make_defaulted_data_inclusion_config())+ context = tracer.setup(graph_id="data-flow-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)started_at = datetime.now(UTC)@@ -521,7 +524,7 @@ class TestGraphTracer:def test_data_edge_not_created_for_unknown_producer(self) -> None:"""Test that no DATA edge is created if the producer is unknown (initial pipeline input)."""tracer = GraphTracer()- context = tracer.setup(graph_id="no-producer-test", data_inclusion=make_defaulted_data_inclusion_config())+ context = tracer.setup(graph_id="no-producer-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)started_at = datetime.now(UTC)@@ -548,7 +551,7 @@ class TestGraphTracer:def test_no_self_loop_data_edges(self) -> None:"""Test that DATA edges are not created as self-loops."""tracer = GraphTracer()- context = tracer.setup(graph_id="no-self-loop-test", data_inclusion=make_defaulted_data_inclusion_config())+ context = tracer.setup(graph_id="no-self-loop-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)started_at = datetime.now(UTC)@@ -577,7 +580,7 @@ class TestGraphTracer:def test_multiple_consumers_same_stuff(self) -> None:"""Test DATA edges when multiple pipes consume the same stuff."""tracer = GraphTracer()- context = tracer.setup(graph_id="multi-consumer-test", data_inclusion=make_defaulted_data_inclusion_config())+ context = tracer.setup(graph_id="multi-consumer-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)started_at = datetime.now(UTC)@@ -635,7 +638,7 @@ class TestGraphTracer:created from the list consumer to the item consumers."""tracer = GraphTracer()- context = tracer.setup(graph_id="batch-item-test", data_inclusion=make_defaulted_data_inclusion_config())+ context = tracer.setup(graph_id="batch-item-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)started_at = datetime.now(UTC)@@ -719,7 +722,7 @@ class TestGraphTracer:created from the item producers to the list producer."""tracer = GraphTracer()- context = tracer.setup(graph_id="batch-aggregate-test", data_inclusion=make_defaulted_data_inclusion_config())+ context = tracer.setup(graph_id="batch-aggregate-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)started_at = datetime.now(UTC)@@ -797,7 +800,7 @@ class TestGraphTracer:def test_batch_edges_combined(self) -> None:"""Test both BATCH_ITEM and BATCH_AGGREGATE edges in a complete batch scenario."""tracer = GraphTracer()- context = tracer.setup(graph_id="batch-combined-test", data_inclusion=make_defaulted_data_inclusion_config())+ context = tracer.setup(graph_id="batch-combined-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)started_at = datetime.now(UTC)@@ -891,7 +894,7 @@ class TestGraphTracer:and target_stuff_digest should be the item stuff code."""tracer = GraphTracer()- context = tracer.setup(graph_id="batch-digest-test", data_inclusion=make_defaulted_data_inclusion_config())+ context = tracer.setup(graph_id="batch-digest-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)started_at = datetime.now(UTC)@@ -948,7 +951,7 @@ class TestGraphTracer:and target_stuff_digest should be the output list stuff code."""tracer = GraphTracer()- context = tracer.setup(graph_id="batch-agg-digest-test", data_inclusion=make_defaulted_data_inclusion_config())+ context = tracer.setup(graph_id="batch-agg-digest-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)started_at = datetime.now(UTC)@@ -1006,7 +1009,7 @@ class TestGraphTracer:the controller node to consumers of those outputs."""tracer = GraphTracer()- context = tracer.setup(graph_id="controller-output-test", data_inclusion=make_defaulted_data_inclusion_config())+ context = tracer.setup(graph_id="controller-output-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)started_at = datetime.now(UTC)@@ -1103,7 +1106,7 @@ class TestGraphTracer:the output should not be registered to avoid corrupting data edges."""tracer = GraphTracer()- context = tracer.setup(graph_id="passthrough-test", data_inclusion=make_defaulted_data_inclusion_config())+ context = tracer.setup(graph_id="passthrough-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)started_at = datetime.now(UTC)@@ -1174,7 +1177,7 @@ class TestGraphTracer:All registered outputs should produce correct DATA edges to their consumers."""tracer = GraphTracer()- context = tracer.setup(graph_id="multi-output-test", data_inclusion=make_defaulted_data_inclusion_config())+ context = tracer.setup(graph_id="multi-output-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)started_at = datetime.now(UTC)diff --git a/tests/unit/pipelex/graph/test_graph_tracer_decision_events.py b/tests/unit/pipelex/graph/test_graph_tracer_decision_events.pynew file mode 100644index 000000000..738669a6d--- /dev/null+++ b/tests/unit/pipelex/graph/test_graph_tracer_decision_events.py@@ -0,0 +1,334 @@+"""Unit tests for the decisions a controller records as it makes them, and for the run mode every trace event states."""++from datetime import UTC, datetime, timedelta++import pytest+from pydantic import ValidationError++from pipelex.graph.graph_tracer import GraphTracer+from pipelex.graph.graph_tracer_manager import GraphTracerManager+from pipelex.graph.graph_tracer_protocol import GraphTracerNoOp+from pipelex.graph.graphspec import IOSpec, NodeKind+from pipelex.system.pipe_run_mode import PipeRunMode+from pipelex.system.trace_context import TraceContext+from pipelex.tracing.in_memory_event_log import InMemoryEventLog+from pipelex.tracing.trace_events import (+ BatchFannedOutEvent,+ ConditionResolvedEvent,+ PipeStartEvent,+ TraceEvent,+)+from tests.unit.pipelex.graph.conftest import make_defaulted_data_inclusion_config++_PIPELINE_RUN_ID = "run_decisions_001"+_GRAPH_ID = "graph-decisions"+_T0 = datetime(2026, 10, 13, 9, 0, 0, tzinfo=UTC)+++def _make_tracer(*, run_mode: PipeRunMode) -> tuple[GraphTracer, TraceContext, InMemoryEventLog]:+ event_log = InMemoryEventLog()+ tracer = GraphTracer()+ context = tracer.setup(+ graph_id=_GRAPH_ID,+ data_inclusion=make_defaulted_data_inclusion_config(),+ event_log=event_log,+ pipeline_run_id=_PIPELINE_RUN_ID,+ run_mode=run_mode,+ )+ return tracer, context, event_log+++def _start_of(*, events: list[TraceEvent], node_id: str) -> PipeStartEvent:+ return next(event for event in events if isinstance(event, PipeStartEvent) and event.node_id == node_id)+++class TestGraphTracerDecisionEvents:+ @pytest.mark.parametrize(+ ("rendered_value", "selected_outcome"),+ [("urgent", "urgent"), ("unmatched", None)],+ ids=["matched_key", "default_outcome"],+ )+ def test_condition_resolution_precedes_the_chosen_pipe(self, rendered_value: str, selected_outcome: str | None) -> None:+ """The resolution is recorded before the chosen pipe starts, and the graph's SELECTED_OUTCOME edge to it is labelled with the key."""+ tracer, context, event_log = _make_tracer(run_mode=PipeRunMode.LIVE)+ condition_id, condition_ctx = tracer.on_pipe_start(+ trace_context=context, pipe_code="route", pipe_type="PipeCondition", node_kind=NodeKind.CONTROLLER, started_at=_T0+ )++ tracer.register_condition_resolution(condition_node_id=condition_id, rendered_value=rendered_value, selected_outcome=selected_outcome)+ chosen_id, _ = tracer.on_pipe_start(+ trace_context=condition_ctx,+ pipe_code="handle",+ pipe_type="PipeCompose",+ node_kind=NodeKind.OPERATOR,+ started_at=_T0 + timedelta(seconds=1),+ )+ tracer.on_pipe_end_success(node_id=chosen_id, ended_at=_T0 + timedelta(seconds=2))+ tracer.on_pipe_end_success(node_id=condition_id, ended_at=_T0 + timedelta(seconds=3))+ graph_spec = tracer.teardown()++ events = event_log.read_events(_PIPELINE_RUN_ID)+ resolutions = [event for event in events if isinstance(event, ConditionResolvedEvent)]+ assert len(resolutions) == 1+ resolution = resolutions[0]+ assert resolution.node_id == condition_id+ assert resolution.rendered_value == rendered_value+ assert resolution.selected_outcome == selected_outcome+ assert resolution.sequence < _start_of(events=events, node_id=chosen_id).sequence++ assert graph_spec is not None+ graph_edges = [edge for edge in graph_spec.edges if edge.kind.is_selected_outcome]+ assert [(edge.source, edge.target, edge.label) for edge in graph_edges] == [(condition_id, chosen_id, selected_outcome)]++ def test_condition_resolution_without_a_pipe_draws_no_edge(self) -> None:+ """An outcome mapped to `continue` or `fail` starts no pipe: the resolution is recorded, and the graph has no edge from it."""+ tracer, context, event_log = _make_tracer(run_mode=PipeRunMode.LIVE)+ sequence_id, sequence_ctx = tracer.on_pipe_start(+ trace_context=context, pipe_code="flow", pipe_type="PipeSequence", node_kind=NodeKind.CONTROLLER, started_at=_T0, step_count=2+ )+ condition_id, _ = tracer.on_pipe_start(+ trace_context=sequence_ctx, pipe_code="route", pipe_type="PipeCondition", node_kind=NodeKind.CONTROLLER, started_at=_T0+ )++ tracer.register_condition_resolution(condition_node_id=condition_id, rendered_value="skip", selected_outcome="skip")+ tracer.on_pipe_end_success(node_id=condition_id, ended_at=_T0 + timedelta(seconds=1))+ # The next step starts under the sequence, never under the condition+ next_step_id, _ = tracer.on_pipe_start(+ trace_context=sequence_ctx,+ pipe_code="later",+ pipe_type="PipeCompose",+ node_kind=NodeKind.OPERATOR,+ started_at=_T0 + timedelta(seconds=2),+ )+ tracer.on_pipe_end_success(node_id=next_step_id, ended_at=_T0 + timedelta(seconds=3))+ tracer.on_pipe_end_success(node_id=sequence_id, ended_at=_T0 + timedelta(seconds=4))+ graph_spec = tracer.teardown()++ events = event_log.read_events(_PIPELINE_RUN_ID)+ assert [event.selected_outcome for event in events if isinstance(event, ConditionResolvedEvent)] == ["skip"]+ assert graph_spec is not None+ assert [edge for edge in graph_spec.edges if edge.kind.is_selected_outcome] == []++ def test_batch_fan_out_precedes_the_first_item(self) -> None:+ tracer, context, event_log = _make_tracer(run_mode=PipeRunMode.LIVE)+ batch_id, batch_ctx = tracer.on_pipe_start(+ trace_context=context,+ pipe_code="per_item",+ pipe_type="PipeBatch",+ node_kind=NodeKind.CONTROLLER,+ started_at=_T0,+ input_specs=[IOSpec(name="items", digest="digest_items", multiplicity=True)],+ )++ tracer.register_batch_fan_out(+ batch_node_id=batch_id, item_count=2, branch_pipe_code="process", branch_pipe_ref="flows.process", branch_pipe_type="PipeCompose"+ )+ item_ids: list[str] = []+ for index_item in range(2):+ item_id, _ = tracer.on_pipe_start(+ trace_context=batch_ctx,+ pipe_code="process",+ pipe_type="PipeCompose",+ node_kind=NodeKind.OPERATOR,+ started_at=_T0 + timedelta(seconds=1 + index_item),+ )+ item_ids.append(item_id)++ events = event_log.read_events(_PIPELINE_RUN_ID)+ fan_outs = [event for event in events if isinstance(event, BatchFannedOutEvent)]+ assert len(fan_outs) == 1+ fan_out = fan_outs[0]+ assert fan_out.node_id == batch_id+ assert fan_out.item_count == 2+ assert fan_out.branch_pipe_code == "process"+ assert fan_out.branch_pipe_ref == "flows.process"+ assert fan_out.branch_pipe_type == "PipeCompose"+ first_item_start = _start_of(events=events, node_id=item_ids[0])+ assert fan_out.sequence < first_item_start.sequence++ def test_controllers_state_their_width_as_they_start(self) -> None:+ tracer, context, event_log = _make_tracer(run_mode=PipeRunMode.LIVE)+ sequence_id, sequence_ctx = tracer.on_pipe_start(+ trace_context=context, pipe_code="flow", pipe_type="PipeSequence", node_kind=NodeKind.CONTROLLER, started_at=_T0, step_count=3+ )+ parallel_id, _ = tracer.on_pipe_start(+ trace_context=sequence_ctx,+ pipe_code="fan",+ pipe_type="PipeParallel",+ node_kind=NodeKind.CONTROLLER,+ started_at=_T0 + timedelta(seconds=1),+ branch_count=2,+ )++ events = event_log.read_events(_PIPELINE_RUN_ID)+ sequence_start = _start_of(events=events, node_id=sequence_id)+ parallel_start = _start_of(events=events, node_id=parallel_id)+ assert (sequence_start.step_count, sequence_start.branch_count) == (3, None)+ assert (parallel_start.step_count, parallel_start.branch_count) == (None, 2)++ @pytest.mark.parametrize("run_mode", [PipeRunMode.LIVE, PipeRunMode.DRY], ids=["live", "dry"])+ def test_every_event_states_the_run_mode(self, run_mode: PipeRunMode) -> None:+ """The tracer stamps its run's mode on every event it writes, and its contexts carry the mode down."""+ tracer, context, event_log = _make_tracer(run_mode=run_mode)+ assert context.run_mode == run_mode++ batch_id, batch_ctx = tracer.on_pipe_start(+ trace_context=context, pipe_code="per_item", pipe_type="PipeBatch", node_kind=NodeKind.CONTROLLER, started_at=_T0+ )+ assert batch_ctx.run_mode == run_mode+ tracer.register_batch_fan_out(+ batch_node_id=batch_id, item_count=1, branch_pipe_code="route", branch_pipe_ref="flows.route", branch_pipe_type="PipeCondition"+ )+ tracer.register_batch_item_extraction(list_stuff_code="list", item_stuff_code="item", item_index=0, batch_controller_node_id=batch_id)+ condition_id, condition_ctx = tracer.on_pipe_start(+ trace_context=batch_ctx,+ pipe_code="route",+ pipe_type="PipeCondition",+ node_kind=NodeKind.CONTROLLER,+ started_at=_T0 + timedelta(seconds=1),+ )+ tracer.register_condition_resolution(condition_node_id=condition_id, rendered_value="yes", selected_outcome="yes")+ chosen_id, _ = tracer.on_pipe_start(+ trace_context=condition_ctx,+ pipe_code="handle",+ pipe_type="PipeCompose",+ node_kind=NodeKind.OPERATOR,+ started_at=_T0 + timedelta(seconds=2),+ )+ tracer.register_execution_data(node_id=chosen_id, execution_data={"rendered": "text"})+ tracer.on_pipe_end_success(node_id=chosen_id, ended_at=_T0 + timedelta(seconds=3), output_spec=IOSpec(name="result", digest="result"))+ tracer.on_pipe_end_success(node_id=condition_id, ended_at=_T0 + timedelta(seconds=4))+ tracer.register_batch_aggregation(output_list_stuff_code="out", item_stuff_code="result", item_index=0, batch_controller_node_id=batch_id)+ tracer.on_pipe_end_success(node_id=batch_id, ended_at=_T0 + timedelta(seconds=5))+ tracer.teardown()++ events = event_log.read_events(_PIPELINE_RUN_ID)+ kinds = {event.model_dump()["event_kind"] for event in events}+ assert {"pipe_start", "pipe_end_success", "edge", "batch_fanned_out", "condition_resolved", "batch_item", "batch_aggregate"} <= kinds+ assert {event.mode for event in events} == {run_mode}++ def test_an_inactive_tracer_records_no_decision(self) -> None:+ tracer, _, event_log = _make_tracer(run_mode=PipeRunMode.LIVE)+ tracer.teardown()++ tracer.register_batch_fan_out(+ batch_node_id="gone", item_count=1, branch_pipe_code="process", branch_pipe_ref="flows.process", branch_pipe_type="PipeCompose"+ )+ tracer.register_condition_resolution(condition_node_id="gone", rendered_value="yes", selected_outcome="yes")++ assert event_log.read_events(_PIPELINE_RUN_ID) == []++ def test_a_costs_only_run_records_no_decision(self) -> None:+ """A run that emits no graph events records no decision, even on a tracer it hands an event log."""+ event_log = InMemoryEventLog()+ tracer = GraphTracer()+ context = tracer.setup(+ graph_id=_GRAPH_ID,+ data_inclusion=make_defaulted_data_inclusion_config(),+ event_log=event_log,+ pipeline_run_id=_PIPELINE_RUN_ID,+ emit_graph_events=False,+ run_mode=PipeRunMode.LIVE,+ )+ batch_id, batch_ctx = tracer.on_pipe_start(+ trace_context=context, pipe_code="per_item", pipe_type="PipeBatch", node_kind=NodeKind.CONTROLLER, started_at=_T0+ )+ tracer.register_batch_fan_out(+ batch_node_id=batch_id, item_count=1, branch_pipe_code="route", branch_pipe_ref="flows.route", branch_pipe_type="PipeCondition"+ )+ condition_id, _ = tracer.on_pipe_start(+ trace_context=batch_ctx,+ pipe_code="route",+ pipe_type="PipeCondition",+ node_kind=NodeKind.CONTROLLER,+ started_at=_T0 + timedelta(seconds=1),+ )+ tracer.register_condition_resolution(condition_node_id=condition_id, rendered_value="yes", selected_outcome="yes")+ assert tracer.teardown() is None++ events = event_log.read_events(_PIPELINE_RUN_ID)+ assert [event for event in events if isinstance(event, (BatchFannedOutEvent, ConditionResolvedEvent))] == []++ def test_a_tracer_records_no_decision_on_a_node_it_never_started(self) -> None:+ """A decision looked up on a tracer that does not hold the controller's node is dropped, as its execution data is.++ The pipe started under that node, as a distributed run's child workflow starts its first pipe under its parent's+ controller, gets no SELECTED_OUTCOME edge from a node absent from this tracer's graph.+ """+ tracer, context, event_log = _make_tracer(run_mode=PipeRunMode.LIVE)+ foreign_node_id = "graph-decisions:other_workflow:node_0"+ foreign_ctx = context.copy_for_child(child_node_id=foreign_node_id, next_sequence=0)++ tracer.register_batch_fan_out(+ batch_node_id=foreign_node_id,+ item_count=1,+ branch_pipe_code="process",+ branch_pipe_ref="flows.process",+ branch_pipe_type="PipeCompose",+ )+ tracer.register_condition_resolution(condition_node_id=foreign_node_id, rendered_value="yes", selected_outcome="yes")+ chosen_id, _ = tracer.on_pipe_start(+ trace_context=foreign_ctx, pipe_code="handle", pipe_type="PipeCompose", node_kind=NodeKind.OPERATOR, started_at=_T0+ )+ tracer.on_pipe_end_success(node_id=chosen_id, ended_at=_T0 + timedelta(seconds=1))+ graph_spec = tracer.teardown()++ events = event_log.read_events(_PIPELINE_RUN_ID)+ assert [event for event in events if isinstance(event, (BatchFannedOutEvent, ConditionResolvedEvent))] == []+ assert graph_spec is not None+ assert [edge for edge in graph_spec.edges if edge.kind.is_selected_outcome] == []++ def test_a_trace_context_states_its_run_mode(self) -> None:+ """A context has no default mode: one built without it would stamp a dry run's events as live."""+ with pytest.raises(ValidationError, match="run_mode"):+ TraceContext(graph_id=_GRAPH_ID, data_inclusion=make_defaulted_data_inclusion_config()) # type: ignore[call-arg] # pyright: ignore[reportCallIssue] # the static error is the runtime contract under test++ def test_the_noop_tracer_accepts_decisions_and_carries_the_mode(self) -> None:+ tracer = GraphTracerNoOp()+ context = tracer.setup(graph_id="noop", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.DRY)++ tracer.register_batch_fan_out(+ batch_node_id="node", item_count=1, branch_pipe_code="process", branch_pipe_ref="flows.process", branch_pipe_type="PipeCompose"+ )+ tracer.register_condition_resolution(condition_node_id="node", rendered_value="yes", selected_outcome=None)++ assert context.run_mode == PipeRunMode.DRY+ assert tracer.teardown() is None++ def test_the_manager_routes_decisions_to_the_run_tracer(self) -> None:+ GraphTracerManager.clear_instance()+ manager = GraphTracerManager.get_or_create_instance()+ event_log = InMemoryEventLog()+ try:+ context = manager.open_tracer(+ graph_id=_GRAPH_ID,+ data_inclusion=make_defaulted_data_inclusion_config(),+ event_log=event_log,+ pipeline_run_id=_PIPELINE_RUN_ID,+ run_mode=PipeRunMode.DRY,+ )+ batch_id, _ = manager.on_pipe_start(+ trace_context=context, pipe_code="per_item", pipe_type="PipeBatch", node_kind=NodeKind.CONTROLLER, started_at=_T0+ )+ assert batch_id is not None+ manager.register_batch_fan_out(+ lookup_key=context.lookup_key,+ batch_node_id=batch_id,+ item_count=4,+ branch_pipe_code="process",+ branch_pipe_ref="flows.process",+ branch_pipe_type="PipeCompose",+ )+ manager.register_condition_resolution(+ lookup_key=context.lookup_key, condition_node_id=batch_id, rendered_value="yes", selected_outcome="yes"+ )+ # A key no tracer answers to is a no-op+ manager.register_condition_resolution(lookup_key="unknown", condition_node_id=batch_id, rendered_value="yes", selected_outcome="yes")+ finally:+ manager.close_tracer(_GRAPH_ID)+ GraphTracerManager.clear_instance()++ events = event_log.read_events(_PIPELINE_RUN_ID)+ assert [event.item_count for event in events if isinstance(event, BatchFannedOutEvent)] == [4]+ assert [event.rendered_value for event in events if isinstance(event, ConditionResolvedEvent)] == ["yes"]+ assert {event.mode for event in events} == {PipeRunMode.DRY}diff --git a/tests/unit/pipelex/graph/test_graph_tracer_emit_gating.py b/tests/unit/pipelex/graph/test_graph_tracer_emit_gating.pyindex 3ff068c2c..0c39dfb6d 100644--- a/tests/unit/pipelex/graph/test_graph_tracer_emit_gating.py+++ b/tests/unit/pipelex/graph/test_graph_tracer_emit_gating.py@@ -2,6 +2,7 @@ from datetime import UTC, datetimefrom pipelex.graph.graph_tracer import GraphTracerfrom pipelex.graph.graphspec import NodeKind+from pipelex.system.pipe_run_mode import PipeRunModefrom tests.unit.pipelex.graph.conftest import make_defaulted_data_inclusion_config@@ -24,13 +25,14 @@ class TestGraphTracerEmitGating:data_inclusion=make_defaulted_data_inclusion_config(),emit_graph_events=False,emit_usage_events=True,+ run_mode=PipeRunMode.LIVE,)assert context.emit_graph_events is Falseassert context.emit_usage_events is Truedef test_setup_defaults_emit_flags_true(self) -> None:tracer = GraphTracer()- context = tracer.setup(graph_id="emit-defaults", data_inclusion=make_defaulted_data_inclusion_config())+ context = tracer.setup(graph_id="emit-defaults", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)assert context.emit_graph_events is Trueassert context.emit_usage_events is True@@ -38,9 +40,7 @@ class TestGraphTracerEmitGating:"""Costs-only mode: the tracer still mints node ids, but teardown skips the discarded spec build."""tracer = GraphTracer()context = tracer.setup(- graph_id="costs-only",- data_inclusion=make_defaulted_data_inclusion_config(),- emit_graph_events=False,+ graph_id="costs-only", data_inclusion=make_defaulted_data_inclusion_config(), emit_graph_events=False, run_mode=PipeRunMode.LIVE)self._start_a_node(tracer, context)assert tracer.teardown() is None@@ -49,9 +49,7 @@ class TestGraphTracerEmitGating:def test_teardown_builds_spec_when_graph_events_on(self) -> None:tracer = GraphTracer()context = tracer.setup(- graph_id="graph-on",- data_inclusion=make_defaulted_data_inclusion_config(),- emit_graph_events=True,+ graph_id="graph-on", data_inclusion=make_defaulted_data_inclusion_config(), emit_graph_events=True, run_mode=PipeRunMode.LIVE)self._start_a_node(tracer, context)graph_spec = tracer.teardown()diff --git a/tests/unit/pipelex/graph/test_graph_tracer_event_emission.py b/tests/unit/pipelex/graph/test_graph_tracer_event_emission.pyindex b47ba1e40..c10780b5b 100644--- a/tests/unit/pipelex/graph/test_graph_tracer_event_emission.py+++ b/tests/unit/pipelex/graph/test_graph_tracer_event_emission.py@@ -8,6 +8,7 @@ from datetime import UTC, datetime, timedeltafrom pipelex.graph.graph_tracer import GraphTracerfrom pipelex.graph.graphspec import EdgeKind, IOSpec, NodeKind, NodeStatus+from pipelex.system.pipe_run_mode import PipeRunModefrom pipelex.tracing.in_memory_event_log import InMemoryEventLogfrom pipelex.tracing.trace_events import (BatchAggregateEvent,@@ -42,6 +43,7 @@ class TestGraphTracerEventEmission:event_log=event_log,workflow_id=workflow_id or self.WORKFLOW_ID,pipeline_run_id=self.PIPELINE_RUN_ID,+ run_mode=PipeRunMode.LIVE,)return tracer, event_log@@ -55,6 +57,7 @@ class TestGraphTracerEventEmission:event_log=event_log,workflow_id=self.WORKFLOW_ID,pipeline_run_id=self.PIPELINE_RUN_ID,+ run_mode=PipeRunMode.LIVE,)started_at = datetime(2025, 6, 15, 10, 0, 0, tzinfo=UTC)@@ -89,6 +92,7 @@ class TestGraphTracerEventEmission:event_log=event_log,workflow_id=self.WORKFLOW_ID,pipeline_run_id=self.PIPELINE_RUN_ID,+ run_mode=PipeRunMode.LIVE,)started_at = datetime(2025, 6, 15, 10, 0, 0, tzinfo=UTC)@@ -125,6 +129,7 @@ class TestGraphTracerEventEmission:event_log=event_log,workflow_id=self.WORKFLOW_ID,pipeline_run_id=self.PIPELINE_RUN_ID,+ run_mode=PipeRunMode.LIVE,)started_at = datetime(2025, 6, 15, 10, 0, 0, tzinfo=UTC)@@ -166,6 +171,7 @@ class TestGraphTracerEventEmission:event_log=event_log,workflow_id=self.WORKFLOW_ID,pipeline_run_id=self.PIPELINE_RUN_ID,+ run_mode=PipeRunMode.LIVE,)started_at = datetime(2025, 6, 15, 10, 0, 0, tzinfo=UTC)@@ -206,6 +212,7 @@ class TestGraphTracerEventEmission:event_log=event_log,workflow_id=self.WORKFLOW_ID,pipeline_run_id=self.PIPELINE_RUN_ID,+ run_mode=PipeRunMode.LIVE,)started_at = datetime(2025, 6, 15, 10, 0, 0, tzinfo=UTC)@@ -254,6 +261,7 @@ class TestGraphTracerEventEmission:event_log=event_log,workflow_id=self.WORKFLOW_ID,pipeline_run_id=self.PIPELINE_RUN_ID,+ run_mode=PipeRunMode.LIVE,)started_at = datetime(2025, 6, 15, 10, 0, 0, tzinfo=UTC)@@ -286,6 +294,7 @@ class TestGraphTracerEventEmission:event_log=event_log,workflow_id=self.WORKFLOW_ID,pipeline_run_id=self.PIPELINE_RUN_ID,+ run_mode=PipeRunMode.LIVE,)tracer.register_batch_item_extraction(@@ -315,6 +324,7 @@ class TestGraphTracerEventEmission:event_log=event_log,workflow_id=self.WORKFLOW_ID,pipeline_run_id=self.PIPELINE_RUN_ID,+ run_mode=PipeRunMode.LIVE,)tracer.register_batch_aggregation(@@ -344,6 +354,7 @@ class TestGraphTracerEventEmission:event_log=event_log,workflow_id=self.WORKFLOW_ID,pipeline_run_id=self.PIPELINE_RUN_ID,+ run_mode=PipeRunMode.LIVE,)started_at = datetime(2025, 6, 15, 10, 0, 0, tzinfo=UTC)@@ -406,10 +417,7 @@ class TestGraphTracerEventEmission:def test_no_event_log_works_as_before(self) -> None:"""GraphTracer without event_log (direct mode) works exactly as before."""tracer = GraphTracer()- context = tracer.setup(- graph_id="direct-mode-test",- data_inclusion=make_defaulted_data_inclusion_config(),- )+ context = tracer.setup(graph_id="direct-mode-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)started_at = datetime(2025, 6, 15, 10, 0, 0, tzinfo=UTC)node_id, _child_ctx = tracer.on_pipe_start(@@ -442,6 +450,7 @@ class TestGraphTracerEventEmission:event_log=event_log,workflow_id="wf_run_123",pipeline_run_id=self.PIPELINE_RUN_ID,+ run_mode=PipeRunMode.LIVE,)started_at = datetime(2025, 6, 15, 10, 0, 0, tzinfo=UTC)@@ -466,6 +475,7 @@ class TestGraphTracerEventEmission:event_log=event_log,workflow_id="wf_run_456",pipeline_run_id=self.PIPELINE_RUN_ID,+ run_mode=PipeRunMode.LIVE,)started_at = datetime(2025, 6, 15, 10, 0, 0, tzinfo=UTC)@@ -505,6 +515,7 @@ class TestGraphTracerEventEmission:event_log=event_log,workflow_id=self.WORKFLOW_ID,pipeline_run_id=self.PIPELINE_RUN_ID,+ run_mode=PipeRunMode.LIVE,)started_at = datetime(2025, 6, 15, 10, 0, 0, tzinfo=UTC)@@ -529,10 +540,7 @@ class TestGraphTracerEventEmission:def test_direct_mode_node_id_unchanged(self) -> None:"""In direct mode (no workflow_id), node ID format is unchanged."""tracer = GraphTracer()- context = tracer.setup(- graph_id="direct-graph",- data_inclusion=make_defaulted_data_inclusion_config(),- )+ context = tracer.setup(graph_id="direct-graph", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)started_at = datetime(2025, 6, 15, 10, 0, 0, tzinfo=UTC)node_id, _ = tracer.on_pipe_start(diff --git a/tests/unit/pipelex/graph/test_graph_tracer_manager.py b/tests/unit/pipelex/graph/test_graph_tracer_manager.pyindex 3ce01ed08..f71600a38 100644--- a/tests/unit/pipelex/graph/test_graph_tracer_manager.py+++ b/tests/unit/pipelex/graph/test_graph_tracer_manager.py@@ -6,6 +6,7 @@ from pytest_mock import MockerFixturefrom pipelex.graph.graph_tracer import GraphTracerfrom pipelex.graph.graph_tracer_manager import GraphTracerManagerfrom pipelex.system.data_inclusion_config import DataInclusionConfig+from pipelex.system.pipe_run_mode import PipeRunModedef _make_data_inclusion() -> DataInclusionConfig:@@ -22,7 +23,7 @@ class TestGraphTracerManagerOpenTracer:GraphTracerManager.clear_instance()manager = GraphTracerManager.get_or_create_instance()try:- trace_context = manager.open_tracer(graph_id="lifecycle_graph", data_inclusion=_make_data_inclusion())+ trace_context = manager.open_tracer(graph_id="lifecycle_graph", data_inclusion=_make_data_inclusion(), run_mode=PipeRunMode.LIVE)assert trace_context.graph_id == "lifecycle_graph"assert manager.get_tracer("lifecycle_graph") is not None@@ -39,7 +40,7 @@ class TestGraphTracerManagerOpenTracer:mocker.patch.object(GraphTracer, "setup", side_effect=ValueError("setup blew up"))with pytest.raises(ValueError, match="setup blew up"):- manager.open_tracer(graph_id="leak_check_graph", data_inclusion=_make_data_inclusion())+ manager.open_tracer(graph_id="leak_check_graph", data_inclusion=_make_data_inclusion(), run_mode=PipeRunMode.LIVE)assert manager.get_tracer("leak_check_graph") is Nonefinally:diff --git a/tests/unit/pipelex/graph/test_graph_tracer_manager_stale_key_healing.py b/tests/unit/pipelex/graph/test_graph_tracer_manager_stale_key_healing.pyindex 65f83ebc4..ea3c130c2 100644--- a/tests/unit/pipelex/graph/test_graph_tracer_manager_stale_key_healing.py+++ b/tests/unit/pipelex/graph/test_graph_tracer_manager_stale_key_healing.py@@ -15,6 +15,7 @@ from pytest_mock import MockerFixturefrom pipelex.graph.graph_tracer_manager import GraphTracerManagerfrom pipelex.system.data_inclusion_config import DataInclusionConfig+from pipelex.system.pipe_run_mode import PipeRunMode_DATA_INCLUSION_OFF = DataInclusionConfig(pipe_and_concept_registry=False,@@ -48,6 +49,7 @@ class TestGraphTracerManagerStaleKeyHealing:tracer_key=_TRACER_KEY,emit_graph_events=False,emit_usage_events=True,+ run_mode=PipeRunMode.LIVE,)def test_open_tracer_replaces_stale_tracer_under_same_key(self, caplog: pytest.LogCaptureFixture) -> None:diff --git a/tests/unit/pipelex/graph/test_graph_tracer_node_metadata.py b/tests/unit/pipelex/graph/test_graph_tracer_node_metadata.pyindex 4343cf242..91ab0948c 100644--- a/tests/unit/pipelex/graph/test_graph_tracer_node_metadata.py+++ b/tests/unit/pipelex/graph/test_graph_tracer_node_metadata.py@@ -4,13 +4,14 @@ from datetime import UTC, datetime, timedeltafrom pipelex.graph.graph_tracer import GraphTracerfrom pipelex.graph.graphspec import NodeKind+from pipelex.system.pipe_run_mode import PipeRunModefrom tests.unit.pipelex.graph.conftest import make_defaulted_data_inclusion_configclass TestGraphTracerNodeMetadata:def test_on_pipe_start_persists_description_and_domain_code(self) -> None:tracer = GraphTracer()- context = tracer.setup(graph_id="meta-test", data_inclusion=make_defaulted_data_inclusion_config())+ context = tracer.setup(graph_id="meta-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)started_at = datetime.now(UTC)node_id, _child = tracer.on_pipe_start(@@ -38,7 +39,7 @@ class TestGraphTracerNodeMetadata:def test_on_pipe_start_defaults_metadata_to_none(self) -> None:tracer = GraphTracer()- context = tracer.setup(graph_id="meta-default-test", data_inclusion=make_defaulted_data_inclusion_config())+ context = tracer.setup(graph_id="meta-default-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)started_at = datetime.now(UTC)node_id, _child = tracer.on_pipe_start(diff --git a/tests/unit/pipelex/graph/test_graph_tracer_noop.py b/tests/unit/pipelex/graph/test_graph_tracer_noop.pyindex 803f62e66..5df846054 100644--- a/tests/unit/pipelex/graph/test_graph_tracer_noop.py+++ b/tests/unit/pipelex/graph/test_graph_tracer_noop.py@@ -4,6 +4,7 @@ from datetime import UTC, datetime, timedeltafrom pipelex.graph.graph_tracer_protocol import GraphTracerNoOpfrom pipelex.graph.graphspec import EdgeKind, NodeKind+from pipelex.system.pipe_run_mode import PipeRunModefrom tests.unit.pipelex.graph.conftest import make_defaulted_data_inclusion_config@@ -13,14 +14,14 @@ class TestGraphTracerNoOp:def test_noop_returns_context(self) -> None:"""Test that no-op tracer still returns valid context."""tracer = GraphTracerNoOp()- context = tracer.setup(graph_id="noop-test", data_inclusion=make_defaulted_data_inclusion_config())+ context = tracer.setup(graph_id="noop-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)assert context.graph_id == "noop-test"def test_noop_teardown_returns_none(self) -> None:"""Test that no-op tracer returns None on teardown."""tracer = GraphTracerNoOp()- tracer.setup(graph_id="noop-test", data_inclusion=make_defaulted_data_inclusion_config())+ tracer.setup(graph_id="noop-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)result = tracer.teardown()assert result is None@@ -28,7 +29,7 @@ class TestGraphTracerNoOp:def test_noop_pipe_lifecycle(self) -> None:"""Test that no-op tracer handles pipe lifecycle without errors."""tracer = GraphTracerNoOp()- context = tracer.setup(graph_id="noop-test", data_inclusion=make_defaulted_data_inclusion_config())+ context = tracer.setup(graph_id="noop-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)started_at = datetime.now(UTC)node_id, child_ctx = tracer.on_pipe_start(diff --git a/tests/unit/pipelex/graph/test_trace_context_emit_flags.py b/tests/unit/pipelex/graph/test_trace_context_emit_flags.pyindex df472f6db..c64110895 100644--- a/tests/unit/pipelex/graph/test_trace_context_emit_flags.py+++ b/tests/unit/pipelex/graph/test_trace_context_emit_flags.py@@ -6,6 +6,7 @@ carried — so a costs-only or graph-only run keeps its gating as nested pipes s"""from pipelex.system.data_inclusion_config import DataInclusionConfig+from pipelex.system.pipe_run_mode import PipeRunModefrom pipelex.system.trace_context import TraceContextDATA_INCLUSION_OFF = DataInclusionConfig(@@ -18,7 +19,7 @@ DATA_INCLUSION_OFF = DataInclusionConfig(class TestTraceContextEmitFlags:def test_defaults_emit_both_streams(self) -> None:"""A TraceContext built without explicit flags emits both streams (legacy behavior)."""- context = TraceContext(graph_id="run_1", data_inclusion=DATA_INCLUSION_OFF)+ context = TraceContext(graph_id="run_1", data_inclusion=DATA_INCLUSION_OFF, run_mode=PipeRunMode.LIVE)assert context.emit_graph_events is Trueassert context.emit_usage_events is Trueassert context.describe_pipe_io is True@@ -26,10 +27,7 @@ class TestTraceContextEmitFlags:def test_copy_for_child_preserves_costs_only_flags(self) -> None:"""costs-only flags survive into a child context (graph off, costs on)."""parent = TraceContext(- graph_id="run_1",- data_inclusion=DATA_INCLUSION_OFF,- emit_graph_events=False,- emit_usage_events=True,+ graph_id="run_1", data_inclusion=DATA_INCLUSION_OFF, emit_graph_events=False, emit_usage_events=True, run_mode=PipeRunMode.LIVE)child = parent.copy_for_child(child_node_id="run_1:node_0", next_sequence=1)assert child.parent_node_id == "run_1:node_0"@@ -40,10 +38,7 @@ class TestTraceContextEmitFlags:def test_copy_for_child_preserves_graph_only_flags(self) -> None:"""graph-only flags survive into a child context (graph on, costs off)."""parent = TraceContext(- graph_id="run_1",- data_inclusion=DATA_INCLUSION_OFF,- emit_graph_events=True,- emit_usage_events=False,+ graph_id="run_1", data_inclusion=DATA_INCLUSION_OFF, emit_graph_events=True, emit_usage_events=False, run_mode=PipeRunMode.LIVE)child = parent.copy_for_child(child_node_id="run_1:node_0", next_sequence=2)assert child.emit_graph_events is True@@ -51,6 +46,6 @@ class TestTraceContextEmitFlags:def test_copy_for_child_preserves_describe_pipe_io_off(self) -> None:"""A run that will write no graphspec keeps its I/O description off in every child context."""- parent = TraceContext(graph_id="run_1", data_inclusion=DATA_INCLUSION_OFF, describe_pipe_io=False)+ parent = TraceContext(graph_id="run_1", data_inclusion=DATA_INCLUSION_OFF, describe_pipe_io=False, run_mode=PipeRunMode.LIVE)child = parent.copy_for_child(child_node_id="run_1:node_0", next_sequence=1)assert child.describe_pipe_io is Falsediff --git a/tests/unit/pipelex/kernel/test_pipelex_kernel_run_state.py b/tests/unit/pipelex/kernel/test_pipelex_kernel_run_state.pyindex 27cd2e33b..b17645248 100644--- a/tests/unit/pipelex/kernel/test_pipelex_kernel_run_state.py+++ b/tests/unit/pipelex/kernel/test_pipelex_kernel_run_state.py@@ -20,6 +20,7 @@ from pipelex.core.memory.working_memory_factory import WorkingMemoryFactoryfrom pipelex.core.stuffs.exceptions import StuffContentTypeErrorfrom pipelex.core.stuffs.list_content import ListContentfrom pipelex.core.stuffs.text_content import TextContent+from pipelex.kernel.exceptions import TraceContextRunModeErrorfrom pipelex.kernel.memory_ops import (extract_main_content,extract_main_content_as_list,@@ -39,7 +40,7 @@ if TYPE_CHECKING:_GRAPH_ID = "kernel-run-state"-def _trace_context() -> TraceContext:+def _trace_context(*, run_mode: PipeRunMode = PipeRunMode.DRY) -> TraceContext:return TraceContext(graph_id=_GRAPH_ID,data_inclusion=DataInclusionConfig(@@ -49,6 +50,7 @@ def _trace_context() -> TraceContext:),emit_graph_events=False,emit_usage_events=True,+ run_mode=run_mode,)@@ -124,6 +126,51 @@ class TestPipelexKernelRunState:pipeline_run_id="plr-elsewhere",)+ @pytest.mark.parametrize(+ ("is_dry_run_forced", "requested_run_mode", "context_run_mode"),+ [+ (True, PipeRunMode.LIVE, PipeRunMode.LIVE),+ (False, PipeRunMode.DRY, PipeRunMode.LIVE),+ (False, PipeRunMode.LIVE, PipeRunMode.DRY),+ ],+ ids=["keyless_boot_forces_the_live_request_dry", "dry_request_live_context", "live_request_dry_context"],+ )+ def test_a_trace_context_stating_another_mode_than_the_run_takes_is_refused(+ self, mocker: MockerFixture, is_dry_run_forced: bool, requested_run_mode: PipeRunMode, context_run_mode: PipeRunMode+ ) -> None:+ """Every usage event states the context's mode, so a context saying live for a run whose steps run dry is refused.++ The keyless arm is the one a caller cannot see in its own arguments: it requests a live run and opens a live+ trace, and the boot runs the steps dry.+ """+ mocker.patch("pipelex.runtime_hub.is_dry_run_forced", return_value=is_dry_run_forced)++ with pytest.raises(TraceContextRunModeError, match="trace context stating a"):+ PipelexKernel.make(+ storage_scope="test/scope",+ read_scope=None,+ run_mode=requested_run_mode,+ user_id="test-user",+ trace_context=_trace_context(run_mode=context_run_mode),+ )++ def test_on_a_keyless_boot_a_dry_trace_context_carries_a_live_request(self, mocker: MockerFixture) -> None:+ """A caller that opens its trace with the mode the boot gives the run gets a dry run whose every step states dry."""+ mocker.patch("pipelex.runtime_hub.is_dry_run_forced", return_value=True)++ kernel = PipelexKernel.make(+ storage_scope="test/scope",+ read_scope=None,+ run_mode=PipeRunMode.LIVE,+ user_id="test-user",+ trace_context=_trace_context(run_mode=PipeRunMode.DRY),+ )++ assert kernel.cogt_run_params.run_mode.is_dry+ step_trace_context = kernel.make_step_metadata().trace_context+ assert step_trace_context is not None+ assert step_trace_context.run_mode.is_dry+def test_an_empty_pipeline_run_id_is_refused(self) -> None:"""An empty id is a caller's bug, and minting one in its place would hide it from a replaying host."""with pytest.raises(ValueError, match="empty pipeline_run_id"):diff --git a/tests/unit/pipelex/pipe_run/test_pipe_run_usage_stats.py b/tests/unit/pipelex/pipe_run/test_pipe_run_usage_stats.pyindex b1320d2cf..3ef809b7e 100644--- a/tests/unit/pipelex/pipe_run/test_pipe_run_usage_stats.py+++ b/tests/unit/pipelex/pipe_run/test_pipe_run_usage_stats.py@@ -35,6 +35,7 @@ def _job(*, mocker: MockerFixture, is_traced: bool) -> Any:data_inclusion=DataInclusionConfig(pipe_and_concept_registry=False, stuff_json_content=False, error_stack_traces=False),emit_graph_events=False,emit_usage_events=True,+ run_mode=PipeRunMode.LIVE,)job.job_metadata = JobMetadata(run_metadata=RunMetadata(user_id="pytest", pipeline_run_id=USAGE_STATS_RUN_ID, storage_scope="test/scope", read_scope=None),diff --git a/tests/unit/pipelex/pipe_run/test_tracing_assembly.py b/tests/unit/pipelex/pipe_run/test_tracing_assembly.pyindex 347c00a86..948622b46 100644--- a/tests/unit/pipelex/pipe_run/test_tracing_assembly.py+++ b/tests/unit/pipelex/pipe_run/test_tracing_assembly.py@@ -11,7 +11,6 @@ from pipelex.cogt.llm.llm_report import LLMTokensUsagefrom pipelex.cogt.usage.cost_category import CostCategoryfrom pipelex.cogt.usage.token_category import TokenCategoryfrom pipelex.core.pipes.pipe_output import PipeOutput-from pipelex.graph.graphspec import GraphSpecModefrom pipelex.pipe_run.tracing_assembly import TracingAssembly, assemble_tracing, assemble_tracing_on_outputfrom pipelex.system.exceptions import MissingDependencyErrorfrom pipelex.system.job_metadata import JobMetadata, RunMetadata@@ -23,7 +22,7 @@ from pipelex.tracing.trace_events import UsageReportEvent_MODULE = "pipelex.pipe_run.tracing_assembly"-def _make_usage_event(pipeline_run_id: str, node_id: str) -> UsageReportEvent:+def _make_usage_event(pipeline_run_id: str, node_id: str, run_mode: PipeRunMode = PipeRunMode.LIVE) -> UsageReportEvent:"""A UsageReportEvent carrying a real LLMTokensUsage (so the aggregator returns it)."""metadata = JobMetadata(run_metadata=RunMetadata(storage_scope="test/scope", read_scope=None, user_id="tracing-assembly-test", pipeline_run_id=pipeline_run_id)@@ -40,6 +39,7 @@ def _make_usage_event(pipeline_run_id: str, node_id: str) -> UsageReportEvent:workflow_id="direct",timestamp=datetime.now(UTC),sequence=0,+ mode=run_mode,node_id=node_id,tokens_usage=tokens_usage,)@@ -116,13 +116,41 @@ class TestTracingAssembly:assemble_usage=False,domain_code="dom",main_pipe_code="main",- run_mode=PipeRunMode.DRY,)assert result.graph_spec is sentinel_graphassert result.tokens_usages is Noneassemble.assert_called_once()- assert assemble.call_args.kwargs["mode"] == GraphSpecMode.DRY++ @pytest.mark.parametrize("run_mode", [PipeRunMode.LIVE, PipeRunMode.DRY], ids=["live", "dry"])+ def test_the_graph_states_the_mode_its_events_state(self, mocker: MockerFixture, run_mode: PipeRunMode) -> None:+ """No caller passes the graph's mode: the assembled graph takes the one its events state."""+ self._enable_tracing(mocker)+ event_log = mocker.MagicMock()+ event_log.read_events = mocker.MagicMock(return_value=[_make_usage_event("plr-mode", "plr-mode:node_0", run_mode=run_mode)])+ mocker.patch(f"{_MODULE}.make_event_log", return_value=event_log)++ result = assemble_tracing(pipeline_run_id="plr-mode", assemble_graph=True, assemble_usage=False)++ assert result.graph_assembly_error is None+ assert result.graph_spec is not None+ assert result.graph_spec.meta["mode"] == run_mode.graphspec_mode++ def test_events_that_mix_modes_get_no_graph_and_keep_their_usage(self, mocker: MockerFixture) -> None:+ """Events of a dry run and a live run under one pipeline run id describe no one run: no graph, and the reason why."""+ self._enable_tracing(mocker)+ live_event = _make_usage_event("plr-mixed-modes", "plr-mixed-modes:node_0", run_mode=PipeRunMode.LIVE)+ dry_event = _make_usage_event("plr-mixed-modes", "plr-mixed-modes:node_1", run_mode=PipeRunMode.DRY)+ event_log = mocker.MagicMock()+ event_log.read_events = mocker.MagicMock(return_value=[live_event, dry_event])+ mocker.patch(f"{_MODULE}.make_event_log", return_value=event_log)++ result = assemble_tracing(pipeline_run_id="plr-mixed-modes", assemble_graph=True, assemble_usage=True)++ assert result.graph_spec is None+ assert result.graph_assembly_error is not None+ assert "mix run modes (dry, live)" in result.graph_assembly_error+ assert result.tokens_usages == [live_event.tokens_usage, dry_event.tokens_usage]def test_graph_and_usage_both_assembled(self, mocker: MockerFixture) -> None:self._enable_tracing(mocker)diff --git a/tests/unit/pipelex/reporting/test_emit_runner_fallback.py b/tests/unit/pipelex/reporting/test_emit_runner_fallback.pyindex 155814924..d8a8a77df 100644--- a/tests/unit/pipelex/reporting/test_emit_runner_fallback.py+++ b/tests/unit/pipelex/reporting/test_emit_runner_fallback.py@@ -30,6 +30,7 @@ from pipelex.system.configuration.configs import NdjsonTracingConfig, TracingBacfrom pipelex.system.data_inclusion_config import DataInclusionConfigfrom pipelex.system.exceptions import MissingDependencyErrorfrom pipelex.system.job_metadata import JobMetadata, RunMetadata, UnitJobId+from pipelex.system.pipe_run_mode import PipeRunModefrom pipelex.system.trace_context import TraceContextfrom pipelex.tracing import activity_event_logfrom pipelex.tracing.activity_event_log import ActivityEventLogCache@@ -81,6 +82,7 @@ def _make_trace_context(graph_id: str,parent_node_id: str | None = "g:node_0",tracer_key: str | None = None,+ run_mode: PipeRunMode = PipeRunMode.LIVE,) -> TraceContext:return TraceContext(graph_id=graph_id,@@ -88,6 +90,7 @@ def _make_trace_context(parent_node_id=parent_node_id,node_sequence=0,data_inclusion=DATA_INCLUSION_OFF,+ run_mode=run_mode,)@@ -139,6 +142,22 @@ class TestEmitRunnerFallback:assert usage_events[0].node_id == "g:node_2"assert usage_events[0].writer_id.startswith("act_")+ def test_fallback_states_the_run_mode_of_the_trace_context(+ self,+ tmp_path: Path,+ mocker: MockerFixture,+ ) -> None:+ """An activity's usage event states the mode its trace context carries, since it holds nothing else of the run."""+ _enable_ndjson_tracing(mocker, tmp_path)++ manager = ReportingManager()+ manager.setup()+ trace_context = _make_trace_context(graph_id="run_dry", tracer_key="wf_dry", run_mode=PipeRunMode.DRY)+ manager.report_inference_job(_make_llm_job("run_dry", trace_context=trace_context))++ usage_events = [evt for evt in NdjsonEventLog(traces_dir=str(tmp_path)).read_events("run_dry") if isinstance(evt, UsageReportEvent)]+ assert [evt.mode for evt in usage_events] == [PipeRunMode.DRY]+def test_runner_fallback_uses_tracer_key_when_set(self,tmp_path: Path,diff --git a/tests/unit/pipelex/reporting/test_emit_runner_fallback_non_llm.py b/tests/unit/pipelex/reporting/test_emit_runner_fallback_non_llm.pyindex 0cf16d92c..15098d35a 100644--- a/tests/unit/pipelex/reporting/test_emit_runner_fallback_non_llm.py+++ b/tests/unit/pipelex/reporting/test_emit_runner_fallback_non_llm.py@@ -37,6 +37,7 @@ from pipelex.reporting.reporting_manager import ReportingManagerfrom pipelex.system.configuration.configs import NdjsonTracingConfig, TracingBackendfrom pipelex.system.data_inclusion_config import DataInclusionConfigfrom pipelex.system.job_metadata import JobMetadata, RunMetadata+from pipelex.system.pipe_run_mode import PipeRunModefrom pipelex.system.trace_context import TraceContextfrom pipelex.tools.misc.image_utils import ImageFormatfrom pipelex.tracing.activity_event_log import ActivityEventLogCache@@ -72,6 +73,7 @@ def _trace_context(run_id: str, tracer_key: str) -> TraceContext:parent_node_id="g:node_0",node_sequence=0,data_inclusion=_DATA_INCLUSION_OFF,+ run_mode=PipeRunMode.LIVE,)diff --git a/tests/unit/pipelex/reporting/test_emit_usage_event_gating.py b/tests/unit/pipelex/reporting/test_emit_usage_event_gating.pyindex 294339a6d..482ebed37 100644--- a/tests/unit/pipelex/reporting/test_emit_usage_event_gating.py+++ b/tests/unit/pipelex/reporting/test_emit_usage_event_gating.py@@ -23,6 +23,7 @@ from pipelex.reporting.reporting_manager import ReportingManagerfrom pipelex.system.configuration.configs import NdjsonTracingConfig, TracingBackendfrom pipelex.system.data_inclusion_config import DataInclusionConfigfrom pipelex.system.job_metadata import JobMetadata, RunMetadata, UnitJobId+from pipelex.system.pipe_run_mode import PipeRunModefrom pipelex.system.trace_context import TraceContextfrom pipelex.tracing.activity_event_log import ActivityEventLogCachefrom pipelex.tracing.ndjson_event_log import NdjsonEventLog@@ -69,6 +70,7 @@ def _make_trace_context(graph_id: str, *, emit_usage_events: bool, emit_graph_evdata_inclusion=DATA_INCLUSION_OFF,emit_graph_events=emit_graph_events,emit_usage_events=emit_usage_events,+ run_mode=PipeRunMode.LIVE,)diff --git a/tests/unit/pipelex/reporting/test_reporting_event_emission.py b/tests/unit/pipelex/reporting/test_reporting_event_emission.pyindex 56fb4b76a..504411b4e 100644--- a/tests/unit/pipelex/reporting/test_reporting_event_emission.py+++ b/tests/unit/pipelex/reporting/test_reporting_event_emission.py@@ -21,6 +21,7 @@ from pipelex.config import get_configfrom pipelex.reporting.reporting_manager import ReportingManagerfrom pipelex.system.data_inclusion_config import DataInclusionConfigfrom pipelex.system.job_metadata import JobMetadata, RunMetadata, UnitJobId+from pipelex.system.pipe_run_mode import PipeRunModefrom pipelex.system.trace_context import TraceContextfrom pipelex.tracing.in_memory_event_log import InMemoryEventLogfrom pipelex.tracing.trace_events import UsageReportEvent@@ -66,6 +67,7 @@ def _make_trace_context(parent_node_id: str | None = None,tracer_key: str | None = None,node_sequence: int = 0,+ run_mode: PipeRunMode = PipeRunMode.LIVE,) -> TraceContext:"""Create a TraceContext for testing with lookup_key resolution."""return TraceContext(@@ -74,6 +76,7 @@ def _make_trace_context(parent_node_id=parent_node_id,node_sequence=node_sequence,data_inclusion=DATA_INCLUSION_OFF,+ run_mode=run_mode,)@@ -118,6 +121,17 @@ class TestReportingEventEmission:assert usage_events[0].tokens_usage.nb_tokens_by_category[TokenCategory.INPUT] == 100assert usage_events[0].node_id == "unknown" # No trace_context parent set+ @pytest.mark.parametrize("run_mode", [PipeRunMode.LIVE, PipeRunMode.DRY], ids=["live", "dry"])+ def test_usage_event_states_the_run_mode(self, run_mode: PipeRunMode) -> None:+ """A usage event states the mode its trace context carries, the one fact of the run the reporting side holds."""+ manager, event_log = self._make_reporting_manager_with_event_log()++ trace_context = _make_trace_context(graph_id=self.PIPELINE_RUN_ID, run_mode=run_mode)+ manager.report_inference_job(_make_test_llm_job(self.PIPELINE_RUN_ID, trace_context=trace_context))++ usage_events = [evt for evt in event_log.read_events(self.PIPELINE_RUN_ID) if isinstance(evt, UsageReportEvent)]+ assert [evt.mode for evt in usage_events] == [run_mode]+def test_report_inference_job_captures_node_id_from_trace_context(self) -> None:"""UsageReportEvent captures the parent node_id from graph context."""manager, event_log = self._make_reporting_manager_with_event_log()diff --git a/tests/unit/pipelex/tracing/conftest.py b/tests/unit/pipelex/tracing/conftest.pyindex 48aef1fe8..1169e8392 100644--- a/tests/unit/pipelex/tracing/conftest.py+++ b/tests/unit/pipelex/tracing/conftest.py@@ -3,6 +3,7 @@from datetime import UTC, datetimefrom pipelex.graph.graphspec import EdgeKind, NodeKind+from pipelex.system.pipe_run_mode import PipeRunModefrom pipelex.tracing.trace_events import EdgeEvent, PipeStartEvent@@ -17,6 +18,7 @@ def make_trace_event(workflow_id=workflow_id,timestamp=datetime(2025, 6, 15, 10, 30, 0, tzinfo=UTC),sequence=sequence,+ mode=PipeRunMode.LIVE,node_id=f"graph_1:{workflow_id}:node_{sequence}",pipe_code="test_pipe",pipe_type="PipeLLMGenText",@@ -35,6 +37,7 @@ def make_edge_event(workflow_id=workflow_id,timestamp=datetime(2025, 6, 15, 10, 30, 0, tzinfo=UTC),sequence=sequence,+ mode=PipeRunMode.LIVE,edge_id=f"graph_1:{workflow_id}:edge_{sequence}",source_node_id=f"graph_1:{workflow_id}:node_0",target_node_id=f"graph_1:{workflow_id}:node_1",diff --git a/tests/unit/pipelex/tracing/test_assembler_equivalence.py b/tests/unit/pipelex/tracing/test_assembler_equivalence.pyindex c243e25d5..9ccdd19c9 100644--- a/tests/unit/pipelex/tracing/test_assembler_equivalence.py+++ b/tests/unit/pipelex/tracing/test_assembler_equivalence.py@@ -13,6 +13,7 @@ from datetime import UTC, datetime, timedeltafrom typing import Any, Callableimport pytest+from pytest_mock import MockerFixturefrom pipelex.cogt.llm.llm_report import LLMTokensUsagefrom pipelex.cogt.usage.cost_category import CostCategory@@ -21,7 +22,9 @@ from pipelex.graph.condition_output_merge import ConditionOutputMerge, Conditionfrom pipelex.graph.graph_tracer import GraphTracerfrom pipelex.graph.graphspec import EdgeKind, EdgeSpec, GraphSpec, IOSpec, NodeKind, NodeSpecfrom pipelex.system.job_metadata import JobCategory, JobMetadata, RunMetadata, UnitJobId+from pipelex.system.pipe_run_mode import PipeRunModefrom pipelex.system.trace_context import TraceContext+from pipelex.tracing import graphspec_assembler as graphspec_assembler_modulefrom pipelex.tracing.graphspec_assembler import GraphSpecAssemblerfrom pipelex.tracing.in_memory_event_log import InMemoryEventLogfrom pipelex.tracing.trace_events import PipeStartEvent, UsageReportEvent@@ -129,7 +132,7 @@ def _run_both_paths(scenario: ScenarioFn) -> tuple[GraphSpec, GraphSpec]:# Path A: direct mode (no event log)tracer_direct = GraphTracer()- ctx_direct = tracer_direct.setup(graph_id=_GRAPH_ID, data_inclusion=data_inclusion)+ ctx_direct = tracer_direct.setup(graph_id=_GRAPH_ID, data_inclusion=data_inclusion, run_mode=PipeRunMode.LIVE)scenario(tracer_direct, ctx_direct)direct_spec = tracer_direct.teardown()assert direct_spec is not None@@ -143,6 +146,7 @@ def _run_both_paths(scenario: ScenarioFn) -> tuple[GraphSpec, GraphSpec]:event_log=event_log,workflow_id=_WORKFLOW_ID,pipeline_run_id=_PIPELINE_RUN_ID,+ run_mode=PipeRunMode.LIVE,)scenario(tracer_event, ctx_event)# Don't call teardown on the event-log tracer — we want to assemble from events@@ -474,6 +478,7 @@ def _usage_events_for(*, assembled_child_node_id: str, event_log: InMemoryEventLwriter_id=event_log.writer_id,timestamp=_T0 + timedelta(seconds=2),sequence=event_log.next_sequence(),+ mode=PipeRunMode.LIVE,node_id=assembled_child_node_id,tokens_usage=LLMTokensUsage(job_metadata=JobMetadata(@@ -492,7 +497,7 @@ def _usage_events_for(*, assembled_child_node_id: str, event_log: InMemoryEventLdef _scenario_condition_selected_outcome(tracer: GraphTracer, context: TraceContext) -> None:- """Condition pipe with SELECTED_OUTCOME edge."""+ """Condition pipe that resolves to a key, then runs that outcome's pipe: both builders derive the SELECTED_OUTCOME edge."""started_at = _T0cond_id, cond_ctx = tracer.on_pipe_start(@@ -502,6 +507,7 @@ def _scenario_condition_selected_outcome(tracer: GraphTracer, context: TraceContnode_kind=NodeKind.CONTROLLER,started_at=started_at,)+ tracer.register_condition_resolution(condition_node_id=cond_id, rendered_value="true", selected_outcome="true")outcome_id, _ = tracer.on_pipe_start(trace_context=cond_ctx,@@ -511,13 +517,6 @@ def _scenario_condition_selected_outcome(tracer: GraphTracer, context: TraceContstarted_at=started_at + timedelta(seconds=1),)- tracer.add_edge(- source_node_id=cond_id,- target_node_id=outcome_id,- edge_kind=EdgeKind.SELECTED_OUTCOME,- label="true",- )-tracer.on_pipe_end_success(node_id=outcome_id,ended_at=started_at + timedelta(seconds=2),@@ -528,6 +527,72 @@ def _scenario_condition_selected_outcome(tracer: GraphTracer, context: TraceCont)+def _scenario_decisions_at_decision_time(tracer: GraphTracer, context: TraceContext) -> None:+ """A sequence whose steps record each decision as it is made: a condition's resolution, then a batch's fan-out.++ The condition takes its default outcome, so its SELECTED_OUTCOME edge is unlabelled; the batch states its width+ before its first item starts.+ """+ started_at = _T0++ seq_id, seq_ctx = tracer.on_pipe_start(+ trace_context=context,+ pipe_code="flow",+ pipe_type="PipeSequence",+ node_kind=NodeKind.CONTROLLER,+ started_at=started_at,+ step_count=2,+ )++ cond_id, cond_ctx = tracer.on_pipe_start(+ trace_context=seq_ctx,+ pipe_code="route",+ pipe_type="PipeCondition",+ node_kind=NodeKind.CONTROLLER,+ started_at=started_at + timedelta(seconds=1),+ )+ tracer.register_condition_resolution(condition_node_id=cond_id, rendered_value="unmatched", selected_outcome=None)+ outcome_id, _ = tracer.on_pipe_start(+ trace_context=cond_ctx,+ pipe_code="fallback",+ pipe_type="PipeCompose",+ node_kind=NodeKind.OPERATOR,+ started_at=started_at + timedelta(seconds=2),+ )+ tracer.on_pipe_end_success(+ node_id=outcome_id, ended_at=started_at + timedelta(seconds=3), output_spec=IOSpec(name="routed", digest="digest_routed")+ )+ tracer.on_pipe_end_success(node_id=cond_id, ended_at=started_at + timedelta(seconds=4))++ batch_id, batch_ctx = tracer.on_pipe_start(+ trace_context=seq_ctx,+ pipe_code="per_item",+ pipe_type="PipeBatch",+ node_kind=NodeKind.CONTROLLER,+ started_at=started_at + timedelta(seconds=5),+ input_specs=[IOSpec(name="items", digest="digest_items", multiplicity=True)],+ )+ tracer.register_batch_fan_out(+ batch_node_id=batch_id, item_count=1, branch_pipe_code="process", branch_pipe_ref="flows.process", branch_pipe_type="PipeCompose"+ )+ tracer.register_batch_item_extraction(+ list_stuff_code="digest_items", item_stuff_code="digest_item_0", item_index=0, batch_controller_node_id=batch_id+ )+ item_id, _ = tracer.on_pipe_start(+ trace_context=batch_ctx,+ pipe_code="process",+ pipe_type="PipeCompose",+ node_kind=NodeKind.OPERATOR,+ started_at=started_at + timedelta(seconds=6),+ input_specs=[IOSpec(name="item", digest="digest_item_0")],+ )+ tracer.on_pipe_end_success(+ node_id=item_id, ended_at=started_at + timedelta(seconds=7), output_spec=IOSpec(name="result", digest="digest_result_0")+ )+ tracer.on_pipe_end_success(node_id=batch_id, ended_at=started_at + timedelta(seconds=8))+ tracer.on_pipe_end_success(node_id=seq_id, ended_at=started_at + timedelta(seconds=9))++def _scenario_dry_condition_merge(tracer: GraphTracer, context: TraceContext) -> None:"""Dry-run condition whose two outcomes write different concepts into the slot a consumer reads.@@ -660,6 +725,7 @@ class TestAssemblerEquivalence:pytest.param(_scenario_pass_through, id="pass_through"),pytest.param(_scenario_condition_selected_outcome, id="condition_selected_outcome"),pytest.param(_scenario_dry_condition_merge, id="dry_condition_merge"),+ pytest.param(_scenario_decisions_at_decision_time, id="decisions_at_decision_time"),],)def test_equivalence(self, scenario_fn: ScenarioFn) -> None:@@ -667,6 +733,98 @@ class TestAssemblerEquivalence:direct_spec, assembled_spec = _run_both_paths(scenario_fn)_assert_graphs_equivalent(direct_spec, assembled_spec)+ def test_decision_events_give_both_graphs_the_choice_edge(self, mocker: MockerFixture) -> None:+ """The decision events are read without a warning, and both graphs derive the condition's choice as its edge."""+ warning_spy = mocker.spy(graphspec_assembler_module.log, "warning")+ direct_spec, assembled_spec = _run_both_paths(_scenario_decisions_at_decision_time)++ assert warning_spy.call_count == 0+ for spec in (direct_spec, assembled_spec):+ pipe_code_by_node_id = {node.node_id: node.pipe_code for node in spec.nodes}+ selected_edges = [edge for edge in spec.edges if edge.kind.is_selected_outcome]+ assert [(pipe_code_by_node_id[edge.source], pipe_code_by_node_id[edge.target], edge.label) for edge in selected_edges] == [+ ("route", "fallback", None)+ ]++ def test_selected_outcome_edge_when_the_chosen_pipe_starts_under_another_tracer(self) -> None:+ """A distributed run: the condition resolves under its workflow's tracer, and the chosen pipe starts under another.++ A sub-pipe dispatched to a workflow of its own runs under a tracer that workflow opens, which keeps the+ condition as the parent of its first node and never saw the resolution. The graph assembled from the run's+ events still draws the SELECTED_OUTCOME edge, as the in-process graph of the same run does.+ """+ data_inclusion = make_defaulted_data_inclusion_config()+ event_log = InMemoryEventLog()++ parent_tracer = GraphTracer()+ parent_ctx = parent_tracer.setup(+ graph_id=_GRAPH_ID,+ data_inclusion=data_inclusion,+ event_log=event_log,+ workflow_id="wf_parent",+ pipeline_run_id=_PIPELINE_RUN_ID,+ run_mode=PipeRunMode.LIVE,+ )+ cond_id, cond_ctx = parent_tracer.on_pipe_start(+ trace_context=parent_ctx, pipe_code="route", pipe_type="PipeCondition", node_kind=NodeKind.CONTROLLER, started_at=_T0+ )+ parent_tracer.register_condition_resolution(condition_node_id=cond_id, rendered_value="urgent", selected_outcome="urgent")++ # The dispatched workflow opens its own tracer, and keeps the parent node of the context it received.+ child_tracer = GraphTracer()+ child_ctx = child_tracer.setup(+ graph_id=_GRAPH_ID,+ data_inclusion=data_inclusion,+ event_log=event_log,+ workflow_id="wf_child",+ pipeline_run_id=_PIPELINE_RUN_ID,+ run_mode=cond_ctx.run_mode,+ ).model_copy(update={"parent_node_id": cond_ctx.parent_node_id})+ chosen_id, _ = child_tracer.on_pipe_start(+ trace_context=child_ctx,+ pipe_code="escalate",+ pipe_type="PipeCompose",+ node_kind=NodeKind.OPERATOR,+ started_at=_T0 + timedelta(seconds=1),+ )+ child_tracer.on_pipe_end_success(node_id=chosen_id, ended_at=_T0 + timedelta(seconds=2))+ child_tracer.teardown()+ parent_tracer.on_pipe_end_success(node_id=cond_id, ended_at=_T0 + timedelta(seconds=3))+ parent_tracer.teardown()+ assembled_spec = GraphSpecAssembler.assemble(events=event_log.read_events(_PIPELINE_RUN_ID), graph_id=_GRAPH_ID)++ # The same run, in process: one tracer sees both the resolution and the chosen pipe's start.+ direct_tracer = GraphTracer()+ direct_ctx = direct_tracer.setup(graph_id=_GRAPH_ID, data_inclusion=data_inclusion, run_mode=PipeRunMode.LIVE)+ direct_cond_id, direct_cond_ctx = direct_tracer.on_pipe_start(+ trace_context=direct_ctx, pipe_code="route", pipe_type="PipeCondition", node_kind=NodeKind.CONTROLLER, started_at=_T0+ )+ direct_tracer.register_condition_resolution(condition_node_id=direct_cond_id, rendered_value="urgent", selected_outcome="urgent")+ direct_chosen_id, _ = direct_tracer.on_pipe_start(+ trace_context=direct_cond_ctx,+ pipe_code="escalate",+ pipe_type="PipeCompose",+ node_kind=NodeKind.OPERATOR,+ started_at=_T0 + timedelta(seconds=1),+ )+ direct_tracer.on_pipe_end_success(node_id=direct_chosen_id, ended_at=_T0 + timedelta(seconds=2))+ direct_tracer.on_pipe_end_success(node_id=direct_cond_id, ended_at=_T0 + timedelta(seconds=3))+ direct_spec = direct_tracer.teardown()+ assert direct_spec is not None++ assert [(edge.source, edge.target, edge.label) for edge in assembled_spec.edges if edge.kind.is_selected_outcome] == [+ (cond_id, chosen_id, "urgent")+ ]+ for spec in (direct_spec, assembled_spec):+ pipe_code_by_node_id = {node.node_id: node.pipe_code for node in spec.nodes}+ edges_by_codes = sorted(+ (pipe_code_by_node_id[edge.source], pipe_code_by_node_id[edge.target], edge.kind, edge.label) for edge in spec.edges+ )+ assert edges_by_codes == [+ ("route", "escalate", EdgeKind.CONTAINS, None),+ ("route", "escalate", EdgeKind.SELECTED_OUTCOME, "urgent"),+ ]+def test_multiplicity_survives_both_builders(self) -> None:"""Agreement alone would pass if both builders dropped the marker, so pin its value too:the batch's list io items stay plural and its items stay single, on both routes.@@ -709,7 +867,7 @@ class TestAssemblerEquivalence:data_inclusion = make_defaulted_data_inclusion_config()tracer_direct = GraphTracer()- ctx_direct = tracer_direct.setup(graph_id=_GRAPH_ID, data_inclusion=data_inclusion)+ ctx_direct = tracer_direct.setup(graph_id=_GRAPH_ID, data_inclusion=data_inclusion, run_mode=PipeRunMode.LIVE)_scenario_with_usage(tracer_direct, ctx_direct)direct_spec = tracer_direct.teardown()assert direct_spec is not None@@ -722,6 +880,7 @@ class TestAssemblerEquivalence:event_log=event_log,workflow_id=_WORKFLOW_ID,pipeline_run_id=_PIPELINE_RUN_ID,+ run_mode=PipeRunMode.LIVE,)_scenario_with_usage(tracer_event, ctx_event)tracer_event.teardown()diff --git a/tests/unit/pipelex/tracing/test_assembler_usage_attribution.py b/tests/unit/pipelex/tracing/test_assembler_usage_attribution.pyindex c5493ac54..3fa374ece 100644--- a/tests/unit/pipelex/tracing/test_assembler_usage_attribution.py+++ b/tests/unit/pipelex/tracing/test_assembler_usage_attribution.py@@ -10,6 +10,7 @@ from pipelex.cogt.usage.token_category import TokenCategoryfrom pipelex.graph.graphspec import GraphSpec, NodeKind, NodeSpecfrom pipelex.reporting.reporting_types import AnyTokensUsagefrom pipelex.system.job_metadata import JobCategory, JobMetadata, RunMetadata, UnitJobId+from pipelex.system.pipe_run_mode import PipeRunModefrom pipelex.tracing.graphspec_assembler import GraphSpecAssemblerfrom pipelex.tracing.trace_events import (UNATTRIBUTED_NODE_ID,@@ -39,6 +40,7 @@ def _base(sequence: int) -> dict[str, Any]:"workflow_id": _WORKFLOW_ID,"timestamp": _T0 + timedelta(seconds=sequence),"sequence": sequence,+ "mode": PipeRunMode.LIVE,}diff --git a/tests/unit/pipelex/tracing/test_dynamodb_event_log.py b/tests/unit/pipelex/tracing/test_dynamodb_event_log.pyindex c266ce8d5..05122105f 100644--- a/tests/unit/pipelex/tracing/test_dynamodb_event_log.py+++ b/tests/unit/pipelex/tracing/test_dynamodb_event_log.py@@ -1,4 +1,6 @@+import jsonimport time+from datetime import UTC, datetimefrom typing import Anyimport pytest@@ -6,14 +8,45 @@ from botocore.exceptions import BotoCoreError, ClientError, EndpointConnectionErfrom pydantic import ValidationErrorfrom pytest_mock import MockerFixture+from pipelex.graph.graphspec import NodeKindfrom pipelex.system.configuration.configs import DynamoDBTracingConfig, TracingConfig+from pipelex.system.pipe_run_mode import PipeRunMode+from pipelex.tracing.buffering_event_log import BufferingEventLogfrom pipelex.tracing.dynamodb_event_log import DynamoDBEventLogfrom pipelex.tracing.event_log_factory import make_event_logfrom pipelex.tracing.event_log_protocol import EventLogProtocol-from pipelex.tracing.exceptions import EventLogReadError, EventLogSetupError+from pipelex.tracing.exceptions import EventLogReadError, EventLogSchemaMismatchError, EventLogSetupError+from pipelex.tracing.trace_events import BatchFannedOutEvent, ConditionResolvedEvent, PipeStartEvent, TraceEventfrom tests.unit.pipelex.tracing.conftest import make_trace_event_SECONDS_PER_DAY = 86_400+_RUN_ID = "plr-decisions"+_T0 = datetime(2026, 10, 13, 9, 0, 0, tzinfo=UTC)+++def _decision_events(*, mode: PipeRunMode) -> list[TraceEvent]:+ base: dict[str, Any] = {"pipeline_run_id": _RUN_ID, "workflow_id": "wf_flow", "timestamp": _T0, "mode": mode}+ return [+ PipeStartEvent(+ **base,+ sequence=0,+ node_id="g:wf_flow:node_0",+ pipe_code="flow",+ pipe_type="PipeSequence",+ node_kind=NodeKind.CONTROLLER,+ step_count=3,+ ),+ ConditionResolvedEvent(**base, sequence=1, node_id="g:wf_flow:node_1", rendered_value="unmatched", selected_outcome=None),+ BatchFannedOutEvent(+ **base,+ sequence=2,+ node_id="g:wf_flow:node_2",+ item_count=5,+ branch_pipe_code="process",+ branch_pipe_ref="flows.process",+ branch_pipe_type="PipeCompose",+ ),+ ]class TestDynamoDBEventLog:@@ -109,6 +142,33 @@ class TestDynamoDBEventLog:with pytest.raises(EventLogReadError):event_log.read_events("plr-paginated")+ def test_decision_events_survive_the_buffer_and_the_table(self, mocker: MockerFixture) -> None:+ """The Temporal path: a workflow buffers the events, an activity writes them to DynamoDB, a reader reads them back whole."""+ event_log, mock_table = self._make_event_log_with_table(mocker)+ buffer = BufferingEventLog()+ events = _decision_events(mode=PipeRunMode.DRY)+ for event in events:+ buffer.emit(event)+ for event in buffer.drain():+ event_log.emit(event)+ stored_items = [call.kwargs["Item"] for call in mock_table.put_item.call_args_list]+ assert all(isinstance(item["ttl"], int) for item in stored_items)+ mock_table.query.return_value = {"Items": stored_items}++ assert event_log.read_events(_RUN_ID) == events++ def test_an_event_stored_without_a_mode_is_refused(self, mocker: MockerFixture) -> None:+ """A payload written before every event stated its mode is in a shape this version refuses, never read as live."""+ event_log, mock_table = self._make_event_log_with_table(mocker)+ payload = json.loads(_decision_events(mode=PipeRunMode.LIVE)[0].model_dump_json())+ del payload["mode"]+ mock_table.query.return_value = {+ "Items": [{"PK": f"PIPELINE_RUN#{_RUN_ID}", "SK": "EVENT#wf_flow#primary#0000000000", "payload": json.dumps(payload)}]+ }++ with pytest.raises(EventLogSchemaMismatchError):+ event_log.read_events(_RUN_ID)+def test_init_translates_construction_error(self, mocker: MockerFixture) -> None:# A botocore failure while building the boto3 client (e.g. a misconfigured region) must surface as# the domain EventLogSetupError so best-effort callers (tracing assembly) degrade rather than abort.diff --git a/tests/unit/pipelex/tracing/test_graphspec_assembler.py b/tests/unit/pipelex/tracing/test_graphspec_assembler.pyindex 8d2e2c916..c243fd638 100644--- a/tests/unit/pipelex/tracing/test_graphspec_assembler.py+++ b/tests/unit/pipelex/tracing/test_graphspec_assembler.py@@ -3,11 +3,16 @@from datetime import UTC, datetime, timedeltafrom typing import Any+import pytest+from pipelex.graph.graphspec import EdgeKind, ErrorSpec, GraphSpecMode, IOSpec, NodeKind, NodeStatus+from pipelex.system.pipe_run_mode import PipeRunMode+from pipelex.tracing.exceptions import TraceEventModeErrorfrom pipelex.tracing.graphspec_assembler import GraphSpecAssemblerfrom pipelex.tracing.trace_events import (BatchAggregateEvent,BatchItemEvent,+ ConditionResolvedEvent,ControllerOutputEvent,EdgeEvent,ParallelCombineEvent,@@ -41,13 +46,14 @@ def _edge_id(workflow_id: str, index: int) -> str:return f"{_GRAPH_ID}:{workflow_id}:edge_{index}"-def _base(workflow_id: str, sequence: int, timestamp_offset: int = 0) -> dict[str, Any]:+def _base(workflow_id: str, sequence: int, timestamp_offset: int = 0, run_mode: PipeRunMode = PipeRunMode.LIVE) -> dict[str, Any]:"""Base fields for event construction."""return {"pipeline_run_id": _PIPELINE_RUN_ID,"workflow_id": workflow_id,"timestamp": _time_at(timestamp_offset or sequence),"sequence": sequence,+ "mode": run_mode,}@@ -63,19 +69,48 @@ def _io_spec(name: str, digest: str) -> IOSpec:class TestGraphSpecAssembler:"""Tests for GraphSpecAssembler.assemble()."""- def test_empty_events(self) -> None:- """Empty event list produces empty GraphSpec."""- result = GraphSpecAssembler.assemble(events=[], graph_id=_GRAPH_ID)- assert result.graph_id == _GRAPH_ID- assert result.nodes == []- assert result.edges == []- assert result.meta["mode"] == GraphSpecMode.LIVE-- def test_explicit_mode(self) -> None:- """Assembler stamps the requested GraphSpec provenance mode."""- result = GraphSpecAssembler.assemble(events=[], graph_id=_GRAPH_ID, mode=GraphSpecMode.DRY)+ @pytest.mark.parametrize(+ ("run_mode", "graphspec_mode"),+ [(PipeRunMode.LIVE, GraphSpecMode.LIVE), (PipeRunMode.DRY, GraphSpecMode.DRY)],+ ids=["live", "dry"],+ )+ def test_the_graph_takes_the_mode_its_events_state(self, run_mode: PipeRunMode, graphspec_mode: GraphSpecMode) -> None:+ node_id = _node_id(_WF_A, 0)+ result = GraphSpecAssembler.assemble(+ events=[+ PipeStartEvent(+ **_base(_WF_A, 0, run_mode=run_mode), node_id=node_id, pipe_code="only", pipe_type="PipeLLM", node_kind=NodeKind.OPERATOR+ ),+ PipeEndSuccessEvent(**_base(_WF_A, 1, run_mode=run_mode), node_id=node_id, ended_at=_time_at(1)),+ ],+ graph_id=_GRAPH_ID,+ )assert result.meta["format"] == "mthds"- assert result.meta["mode"] == GraphSpecMode.DRY+ assert result.meta["mode"] == graphspec_mode++ @pytest.mark.parametrize(+ ("run_modes", "expected_message"),+ [+ ([], "No trace events to assemble graph 'test_graph' from"),+ ([PipeRunMode.LIVE, PipeRunMode.DRY], "The trace events of graph 'test_graph' mix run modes (dry, live)"),+ ],+ ids=["no_events", "mixed_modes"],+ )+ def test_events_stating_no_single_mode_are_refused(self, run_modes: list[PipeRunMode], expected_message: str) -> None:+ """A set of events that mixes modes holds more than one run, and an empty one states no mode: neither gets a graph."""+ events = [+ PipeStartEvent(+ **_base(_WF_A, index_event, run_mode=run_mode),+ node_id=_node_id(_WF_A, index_event),+ pipe_code=f"pipe_{index_event}",+ pipe_type="PipeLLM",+ node_kind=NodeKind.OPERATOR,+ )+ for index_event, run_mode in enumerate(run_modes)+ ]+ with pytest.raises(TraceEventModeError) as exc_info:+ GraphSpecAssembler.assemble(events=events, graph_id=_GRAPH_ID)+ assert str(exc_info.value).startswith(expected_message)def test_registry_source_payloads_survive_assembly(self) -> None:"""PipeStartEvent and PipeEndSuccessEvent registry payloads are copied unchanged."""@@ -505,20 +540,17 @@ class TestGraphSpecAssembler:assert (branch_1, controller) in batch_agg_pairsdef test_condition_selected_outcome(self) -> None:- """Condition pipe with SELECTED_OUTCOME edge."""- condition = _node_id(_WF_A, 0)- outcome = _node_id(_WF_A, 1)+ """The SELECTED_OUTCOME edge is derived from the condition's resolution and the start of the pipe it chose.++ The chosen pipe starts in another workflow than the condition, whose events sort first, as a dispatched sub-pipe+ does in a distributed run; an edge event of the same kind a tracer drew is regenerated, never duplicated.+ """+ condition = _node_id(_WF_B, 0)+ outcome = _node_id(_WF_A, 0)events = [PipeStartEvent(**_base(_WF_A, 0),- node_id=condition,- pipe_code="check",- pipe_type="PipeCondition",- node_kind=NodeKind.CONTROLLER,- ),- PipeStartEvent(- **_base(_WF_A, 1),node_id=outcome,parent_node_id=condition,pipe_code="branch_true",@@ -526,27 +558,40 @@ class TestGraphSpecAssembler:node_kind=NodeKind.OPERATOR,),EdgeEvent(- **_base(_WF_A, 2),+ **_base(_WF_A, 1),edge_id=_edge_id(_WF_A, 0),source_node_id=condition,target_node_id=outcome,edge_kind=EdgeKind.CONTAINS,),+ PipeEndSuccessEvent(+ **_base(_WF_A, 2),+ node_id=outcome,+ ended_at=_time_at(4),+ ),+ PipeStartEvent(+ **_base(_WF_B, 0),+ node_id=condition,+ pipe_code="check",+ pipe_type="PipeCondition",+ node_kind=NodeKind.CONTROLLER,+ ),+ ConditionResolvedEvent(+ **_base(_WF_B, 1),+ node_id=condition,+ rendered_value="true",+ selected_outcome="true",+ ),EdgeEvent(- **_base(_WF_A, 3),- edge_id=_edge_id(_WF_A, 1),+ **_base(_WF_B, 2),+ edge_id=_edge_id(_WF_B, 0),source_node_id=condition,target_node_id=outcome,edge_kind=EdgeKind.SELECTED_OUTCOME,label="true",),PipeEndSuccessEvent(- **_base(_WF_A, 4),- node_id=outcome,- ended_at=_time_at(4),- ),- PipeEndSuccessEvent(- **_base(_WF_A, 5),+ **_base(_WF_B, 3),node_id=condition,ended_at=_time_at(5),),@@ -555,10 +600,39 @@ class TestGraphSpecAssembler:result = GraphSpecAssembler.assemble(events=events, graph_id=_GRAPH_ID)selected_edges = [edge for edge in result.edges if edge.kind == EdgeKind.SELECTED_OUTCOME]- assert len(selected_edges) == 1- assert selected_edges[0].source == condition- assert selected_edges[0].target == outcome- assert selected_edges[0].label == "true"+ assert [(edge.source, edge.target, edge.label) for edge in selected_edges] == [(condition, outcome, "true")]++ def test_a_resolution_of_a_condition_absent_from_the_graph_draws_no_edge(self) -> None:+ """A resolution whose condition node no event started draws no SELECTED_OUTCOME edge from a node the graph lacks."""+ absent_condition = _node_id(_WF_B, 0)+ outcome = _node_id(_WF_A, 0)++ events = [+ PipeStartEvent(+ **_base(_WF_A, 0),+ node_id=outcome,+ parent_node_id=absent_condition,+ pipe_code="branch_true",+ pipe_type="PipeLLM",+ node_kind=NodeKind.OPERATOR,+ ),+ PipeEndSuccessEvent(+ **_base(_WF_A, 1),+ node_id=outcome,+ ended_at=_time_at(4),+ ),+ ConditionResolvedEvent(+ **_base(_WF_B, 0),+ node_id=absent_condition,+ rendered_value="true",+ selected_outcome="true",+ ),+ ]++ result = GraphSpecAssembler.assemble(events=events, graph_id=_GRAPH_ID)++ assert [node.node_id for node in result.nodes] == [outcome]+ assert [edge for edge in result.edges if edge.kind == EdgeKind.SELECTED_OUTCOME] == []def test_partial_failure(self) -> None:"""Child fails with error, parent has no end event → FAILED + CANCELED."""diff --git a/tests/unit/pipelex/tracing/test_ndjson_dedup_assembly.py b/tests/unit/pipelex/tracing/test_ndjson_dedup_assembly.pyindex 901635531..cf8b2bdc3 100644--- a/tests/unit/pipelex/tracing/test_ndjson_dedup_assembly.py+++ b/tests/unit/pipelex/tracing/test_ndjson_dedup_assembly.py@@ -10,6 +10,7 @@ from typing import Anyimport pytestfrom pipelex.graph.graphspec import EdgeKind, IOSpec, NodeKind, NodeStatus+from pipelex.system.pipe_run_mode import PipeRunModefrom pipelex.tracing.graphspec_assembler import GraphSpecAssemblerfrom pipelex.tracing.ndjson_event_log import NdjsonEventLogfrom pipelex.tracing.trace_events import (@@ -35,6 +36,7 @@ def _base(sequence: int) -> dict[str, Any]:"workflow_id": _WF_ID,"timestamp": _time_at(sequence),"sequence": sequence,+ "mode": PipeRunMode.LIVE,}diff --git a/tests/unit/pipelex/tracing/test_non_llm_usage_aggregation.py b/tests/unit/pipelex/tracing/test_non_llm_usage_aggregation.pyindex 3cb51081d..943643eea 100644--- a/tests/unit/pipelex/tracing/test_non_llm_usage_aggregation.py+++ b/tests/unit/pipelex/tracing/test_non_llm_usage_aggregation.py@@ -21,6 +21,7 @@ from pipelex.cogt.usage.cost_registry import CostRegistry, ModelUsageKeyfrom pipelex.cogt.usage.token_category import TokenCategoryfrom pipelex.reporting.reporting_types import AnyTokensUsagefrom pipelex.system.job_metadata import JobMetadata, RunMetadata+from pipelex.system.pipe_run_mode import PipeRunModefrom pipelex.tracing.trace_events import UsageReportEventfrom pipelex.tracing.usage_aggregator import UsageAggregator@@ -36,6 +37,7 @@ def _usage_event(tokens_usage: AnyTokensUsage, sequence: int) -> UsageReportEvenwriter_id="act_test",timestamp=datetime.now(UTC),sequence=sequence,+ mode=PipeRunMode.LIVE,node_id=f"g:node_{sequence}",tokens_usage=tokens_usage,)diff --git a/tests/unit/pipelex/tracing/test_trace_events.py b/tests/unit/pipelex/tracing/test_trace_events.pyindex 9219da969..f3565deb1 100644--- a/tests/unit/pipelex/tracing/test_trace_events.py+++ b/tests/unit/pipelex/tracing/test_trace_events.py@@ -4,7 +4,7 @@ from datetime import UTC, datetimefrom typing import Any, ClassVarimport pytest-from pydantic import TypeAdapter+from pydantic import TypeAdapter, ValidationErrorfrom pipelex.cogt.llm.llm_report import LLMTokensUsagefrom pipelex.cogt.usage.cost_category import CostCategory@@ -12,11 +12,14 @@ from pipelex.cogt.usage.token_category import TokenCategoryfrom pipelex.graph.condition_output_merge import ConditionOutputMerge, ConditionOutputTypingfrom pipelex.graph.graphspec import EdgeKind, ErrorSpec, IOSpec, NodeKindfrom pipelex.system.job_metadata import JobCategory, JobMetadata, RunMetadata, UnitJobId+from pipelex.system.pipe_run_mode import PipeRunModefrom pipelex.tracing.trace_events import (AnyTraceEvent,BatchAggregateEvent,+ BatchFannedOutEvent,BatchItemEvent,ConditionOutputMergeEvent,+ ConditionResolvedEvent,ControllerOutputEvent,EdgeEvent,ExecutionDataEvent,@@ -47,6 +50,7 @@ class _Shared:"workflow_id": _Shared.WORKFLOW_ID,"timestamp": _Shared.TIMESTAMP,"sequence": sequence,+ "mode": PipeRunMode.LIVE,}@staticmethod@@ -199,6 +203,26 @@ class TestTraceEvents:),},),+ (+ "batch_fanned_out",+ TraceEventKind.BATCH_FANNED_OUT,+ {+ "node_id": _Shared.NODE_ID,+ "item_count": 3,+ "branch_pipe_code": "summarize_record",+ "branch_pipe_ref": "records.summarize_record",+ "branch_pipe_type": "PipeLLM",+ },+ ),+ (+ "condition_resolved",+ TraceEventKind.CONDITION_RESOLVED,+ {+ "node_id": _Shared.NODE_ID,+ "rendered_value": "urgent",+ "selected_outcome": "urgent",+ },+ ),("execution_data",TraceEventKind.EXECUTION_DATA,@@ -228,6 +252,8 @@ class TestTraceEvents:TraceEventKind.BATCH_AGGREGATE: BatchAggregateEvent,TraceEventKind.PARALLEL_COMBINE: ParallelCombineEvent,TraceEventKind.CONDITION_OUTPUT_MERGE: ConditionOutputMergeEvent,+ TraceEventKind.BATCH_FANNED_OUT: BatchFannedOutEvent,+ TraceEventKind.CONDITION_RESOLVED: ConditionResolvedEvent,TraceEventKind.EXECUTION_DATA: ExecutionDataEvent,TraceEventKind.USAGE_REPORT: UsageReportEvent,}@@ -407,6 +433,86 @@ class TestTraceEvents:assert restored.description == "Summarize the input document."assert restored.domain_code == "summarization"+ @pytest.mark.parametrize(+ ("step_count", "branch_count"),+ [(None, None), (3, None), (None, 2)],+ ids=["operator", "sequence", "parallel"],+ )+ def test_pipe_start_carries_step_and_branch_counts(self, step_count: int | None, branch_count: int | None) -> None:+ """A sequence states its authored step count and a parallel its branch count as they start; any other pipe states neither."""+ event = PipeStartEvent(+ **_Shared.make_base_fields(),+ node_id=_Shared.NODE_ID,+ pipe_code="controller",+ pipe_type="PipeSequence",+ node_kind=NodeKind.CONTROLLER,+ step_count=step_count,+ branch_count=branch_count,+ )++ payload = event.model_dump(mode="json")+ restored = _any_trace_event_adapter.validate_json(event.model_dump_json())++ assert payload["step_count"] == step_count+ assert payload["branch_count"] == branch_count+ assert isinstance(restored, PipeStartEvent)+ assert restored.step_count == step_count+ assert restored.branch_count == branch_count++ def test_condition_resolved_default_outcome_reads_null(self) -> None:+ """A condition that took its default outcome states `selected_outcome` as null, with the value it rendered."""+ event = ConditionResolvedEvent(+ **_Shared.make_base_fields(),+ node_id=_Shared.NODE_ID,+ rendered_value="unmatched",+ selected_outcome=None,+ )++ payload = event.model_dump(mode="json")+ restored = _any_trace_event_adapter.validate_json(event.model_dump_json())++ assert payload["event_kind"] == "condition_resolved"+ assert payload["selected_outcome"] is None+ assert payload["rendered_value"] == "unmatched"+ assert restored == event++ def test_batch_fanned_out_refuses_a_negative_item_count(self) -> None:+ with pytest.raises(ValidationError):+ BatchFannedOutEvent(+ **_Shared.make_base_fields(),+ node_id=_Shared.NODE_ID,+ item_count=-1,+ branch_pipe_code="summarize_record",+ branch_pipe_ref="records.summarize_record",+ branch_pipe_type="PipeLLM",+ )++ @pytest.mark.parametrize("mode", [PipeRunMode.LIVE, PipeRunMode.DRY], ids=["live", "dry"])+ def test_every_event_states_its_mode(self, mode: PipeRunMode) -> None:+ """Every event kind serializes the run mode it was written in, and reads it back."""+ for _, event_kind, extra_fields in self.EVENTS_AND_KINDS:+ event_cls = self.EVENT_CLASSES[event_kind]+ event = event_cls(**{**_Shared.make_base_fields(), "mode": mode}, **extra_fields)++ payload = event.model_dump(mode="json")+ restored = _any_trace_event_adapter.validate_json(event.model_dump_json())++ assert payload["mode"] == mode, event_kind+ assert restored.mode == mode, event_kind++ def test_an_event_without_a_mode_is_refused(self) -> None:+ """The mode is required: a line written before it existed fails to load rather than reading as live."""+ base_fields = _Shared.make_base_fields()+ del base_fields["mode"]+ with pytest.raises(ValidationError):+ PipeStartEvent(+ **base_fields,+ node_id=_Shared.NODE_ID,+ pipe_code="noop_pipe",+ pipe_type="PipeNoop",+ node_kind=NodeKind.OPERATOR,+ )+def test_all_event_kinds_covered(self) -> None:"""Every TraceEventKind has a corresponding test case and event class."""tested_kinds = {entry[1] for entry in self.EVENTS_AND_KINDS}diff --git a/tests/unit/pipelex/tracing/test_usage_aggregator.py b/tests/unit/pipelex/tracing/test_usage_aggregator.pyindex 13f1a44ac..79076e8c2 100644--- a/tests/unit/pipelex/tracing/test_usage_aggregator.py+++ b/tests/unit/pipelex/tracing/test_usage_aggregator.py@@ -8,6 +8,7 @@ from pipelex.cogt.usage.cost_category import CostCategoryfrom pipelex.cogt.usage.token_category import TokenCategoryfrom pipelex.graph.graphspec import NodeKindfrom pipelex.system.job_metadata import JobCategory, JobMetadata, RunMetadata, UnitJobId+from pipelex.system.pipe_run_mode import PipeRunModefrom pipelex.tracing.trace_events import PipeStartEvent, TraceEvent, UsageReportEventfrom pipelex.tracing.usage_aggregator import UsageAggregator@@ -37,6 +38,7 @@ def _make_base_fields(sequence: int = 0) -> dict[str, Any]:"workflow_id": _WORKFLOW_ID,"timestamp": _TIMESTAMP,"sequence": sequence,+ "mode": PipeRunMode.LIVE,}diff --git a/tests/unit/pipelex/tracing/test_writer_id_schema.py b/tests/unit/pipelex/tracing/test_writer_id_schema.pyindex 7ef62145b..c532b46a8 100644--- a/tests/unit/pipelex/tracing/test_writer_id_schema.py+++ b/tests/unit/pipelex/tracing/test_writer_id_schema.py@@ -13,6 +13,7 @@ from typing import Anyfrom pytest_mock import MockerFixturefrom pipelex.graph.graphspec import NodeKind+from pipelex.system.pipe_run_mode import PipeRunModefrom pipelex.tracing.buffering_event_log import BufferingEventLogfrom pipelex.tracing.dynamodb_event_log import DynamoDBEventLogfrom pipelex.tracing.in_memory_event_log import InMemoryEventLog@@ -31,6 +32,7 @@ def _make_pipe_start(sequence: int, writer_id: str = "primary", workflow_id: strwriter_id=writer_id,timestamp=_TS,sequence=sequence,+ mode=PipeRunMode.LIVE,node_id=f"g:{workflow_id}:node_{sequence}",pipe_code="pipe_x",pipe_type="PipeLLM",@@ -48,6 +50,7 @@ class TestWriterIdSchema:workflow_id=_WORKFLOW_ID,timestamp=_TS,sequence=0,+ mode=PipeRunMode.LIVE,node_id=f"g:{_WORKFLOW_ID}:node_0",pipe_code="pipe_x",pipe_type="PipeLLM",@@ -75,6 +78,7 @@ class TestWriterIdSchema:"workflow_id": _WORKFLOW_ID,"timestamp": _TS.isoformat(),"sequence": 0,+ "mode": "live","event_kind": "pipe_start","node_id": f"g:{_WORKFLOW_ID}:node_0","pipe_code": "pipe_x",