Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

feature/Decisions-at-decision-time · L-260927-638e5f (#1427)

Louis Choquel1f2d210e82fc
diff --git a/.drift/acks/hub-layering-convention.toml b/.drift/acks/hub-layering-convention.toml
index e6fdf4150..a5ac63e3b 100644
--- a/.drift/acks/hub-layering-convention.toml
+++ b/.drift/acks/hub-layering-convention.toml
@@ -1,10 +1,10 @@
contract = "hub-layering-convention"
-digest = "sha256:a936b7ba24211f227bcc1b60046075b56522c24b7f6de366ee2fa091ce821267"
+digest = "sha256:558d2a12672e159edf2827791bcb5370781df2b576b958e0ea48989833087f8b"
reviewed_by = "Louis Choquel"
-reviewed_at = "2026-10-12T02:37:48Z"
-rationale = "real-catch: interpreter_hub.py gains the module-level does_pipe_run_leave_process (and InterpreterHub.set_pipe_run takes leaves_process, stored beside the pipe run), so the run row of the partition table now names it. No import was added to either hub, KERNEL_LAYER_PACKAGES is unchanged, and every public module-level function and class of both hubs is in the tables."
+reviewed_at = "2026-10-13T02:26:31Z"
+rationale = "real-catch: runtime_hub.py gains the module-level resolve_effective_run_mode, moved from the pipe tier so the kernel layer can reach it; the run-mode row of the partition table now names it and the resolve_run_mode_for_boot paragraph says why it sits beside it. No import crosses the boundary: runtime_hub imports cogt_run_params, which is kernel-layer, and KERNEL_LAYER_PACKAGES is unchanged."
[trigger_files]
"pipelex/cli/dev_cli/commands/hub_layering_guard.py" = "blob:fda2237457fe0b6830ff2ae8f720698735bec4c3"
"pipelex/interpreter_hub.py" = "blob:023f8452daf76dde409afc7c0966b149b11edabb"
-"pipelex/runtime_hub.py" = "blob:65beba670c857897858b75fd17bbbc03ba84e750"
+"pipelex/runtime_hub.py" = "blob:8269c5d9d549dd06eb8329e3763efed403f95fed"
diff --git a/.drift/acks/pipelex-kernel-docs.toml b/.drift/acks/pipelex-kernel-docs.toml
index 3b94fed28..bc914050b 100644
--- a/.drift/acks/pipelex-kernel-docs.toml
+++ b/.drift/acks/pipelex-kernel-docs.toml
@@ -1,14 +1,14 @@
contract = "pipelex-kernel-docs"
-digest = "sha256:f6449898b2c6db663c5a11f287652c487620dd24dec35654a0883fb62c2fd91d"
+digest = "sha256:1ed250b7f7895e5a94b70b4329c99cef9f1f2c41803c69494deeacbce044392c"
reviewed_by = "Louis Choquel"
-reviewed_at = "2026-10-10T20:46:23Z"
-rationale = "real-catch: run_multi_judgment now hands the verdicts over by field name, so an aliased field keeps its verdict, and decides a refusal by class_field_may_hold_nothing, the predicate the load's concept walk reads a class field by, storing None outright when the field's type admits it. pipelex-kernel.md said a refused field was left absent when neither required nor defaulted; the judgment paragraph now states the shared rule and the by-name handover, and gains the PipeJudge execution-data paragraph moved out of the building-methods page."
+reviewed_at = "2026-10-13T02:26:32Z"
+rationale = "real-catch: PipelexKernel.make now resolves its mode through runtime_hub.resolve_effective_run_mode and refuses a trace_context stating another mode with the new TraceContextRunModeError in kernel/exceptions.py. pipelex-kernel.md's usage example built a TraceContext without the now-required run_mode; it now states the effective mode on the context and on make, a paragraph explains the refusal and why the kernel refuses rather than corrects its copy, and the keyless-boot sentence names resolve_effective_run_mode."
[trigger_files]
"pipelex/kernel/__init__.py" = "blob:c4f1b0fe2ccc17db9110733f71b612f4ac1ba4ce"
"pipelex/kernel/compose_ops.py" = "blob:ccd8ba141246380eb5e9d55e693fa6fedf38233a"
"pipelex/kernel/compose_results.py" = "blob:a837ae9680c7d4d0eb4dee37d4657c410e5b5cea"
-"pipelex/kernel/exceptions.py" = "blob:e6c9fc0ddae10e533f8aaf2fd0f8afcb46881461"
+"pipelex/kernel/exceptions.py" = "blob:ec784b58d732cca19b9278c83afd009422792e81"
"pipelex/kernel/extract_ops.py" = "blob:19d556f0254017d7d52c30f4f1a46e2235289042"
"pipelex/kernel/extract_results.py" = "blob:79405a56760f3d4b483660d76441fff95f58c64d"
"pipelex/kernel/func_ops.py" = "blob:97c68c977ddc260c9b36b7eda5f4c7964e956a27"
@@ -22,7 +22,7 @@ rationale = "real-catch: run_multi_judgment now hands the verdicts over by field
"pipelex/kernel/llm_prompt_content.py" = "blob:f996cead6e7789494f01f69c128ae71d6dcf8466"
"pipelex/kernel/llm_results.py" = "blob:6e3b8a2878b5bb284ece5fac10262fb59ac2029d"
"pipelex/kernel/memory_ops.py" = "blob:7fbbfaaec99f9712d8e215520cb02e26aad3fb40"
-"pipelex/kernel/pipelex_kernel.py" = "blob:5dec4e68d517d5ab1d7e9cbb91d836f37b10e713"
+"pipelex/kernel/pipelex_kernel.py" = "blob:a923ec3c37769d0eac1b37be075971464c27c06c"
"pipelex/kernel/prompt_assembly.py" = "blob:7559725bc0d813cd136bf258b8178d96a8f49194"
"pipelex/kernel/prompt_references.py" = "blob:a4a8c5535c7e51a16ac6c8d1296439b331192a7f"
"pipelex/kernel/search_ops.py" = "blob:b7de7ff62c45dde7f7aa8a9f409aecf3b664cce5"
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 89e50d6d6..de86ddb9d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,16 @@
## [Unreleased]
+### Added
+
+- **`condition_resolved` and `batch_fanned_out` trace events**: a live `PipeCondition` writes `condition_resolved` the moment its choice is final, before anything runs under it, with the `rendered_value` its expression rendered and the `selected_outcome` key of the outcome map it matched, `null` when it took the default outcome, and a chosen pipe whose inputs are missing fails the condition with no outcome recorded; both graph builders derive from that event and the start of the chosen pipe the `selected_outcome` edge from the condition, labelled with that key, which no run drew before, whichever workflow the chosen pipe started in. A `PipeBatch` writes `batch_fanned_out` once it knows its width, before any item starts, with `item_count`, `branch_pipe_code`, the bare code of the pipe each item runs, `branch_pipe_ref`, its qualified reference, and `branch_pipe_type`. A dry run resolves no condition, so it writes no `condition_resolved`, and both graph builders build the same graph from a run's events.
+- **`step_count` and `branch_count` on `pipe_start`**: a `PipeSequence`'s start event states the number of steps its author wrote, the binding of a dotted `batch_over` not among them, and a `PipeParallel`'s the number of its branches; both are `null` on every other pipe.
+
+### Changed
+
+- **Every trace event states its `mode` (Breaking)**: each trace event carries `mode`, `live` or `dry`, the mode the run's pipes run in, which the run's `TraceContext` carries as the new `run_mode`, so a usage event an activity writes states it too, and a live run a keyless boot (`needs_inference=False`) forces dry states `dry` on its trace context, its events and its graph, and, like every dry run, opens no OpenTelemetry trace; an event stored without `mode` by an earlier version is refused with `EventLogSchemaMismatchError` rather than read as live, and a consumer that builds trace events passes it. Callers follow new signatures: `TraceContext.run_mode` is required, with no default, and so is `run_mode` on `GraphTracerManager.open_tracer` and on the tracers' `setup` (`GraphTracerProtocol`, `GraphTracer` and `GraphTracerNoOp`), where it replaces `mode: GraphSpecMode`, so a workflow opening a tracer of its own for part of a run passes the `run_mode` of the context it received, and `PipelexKernel.make` refuses, with `TraceContextRunModeError`, a `trace_context` stating another mode than the one its run takes, which the new `runtime_hub.resolve_effective_run_mode` gives, dry on a keyless boot. `GraphSpecAssembler.assemble` no longer takes `mode` and raises the new `TraceEventModeError` on a set of events that mixes modes or holds none, and `assemble_tracing` and `assemble_tracing_on_output` no longer take `run_mode`: the graph states the mode its events state, so a dry run assembled from its events in a durable workflow gets a dry graph where it got a live one, and `assemble_tracing` reports a `TraceEventModeError` as the run's `graph_assembly_error`.
+- **Controllers register their execution data as they decide (Breaking)**: a `PipeSequence` registers `step_count` as it starts, now the number of steps its author wrote, and the new `current_step` as each step starts; a `PipeParallel` registers its `branch_count`, `add_each_output` and `combined_output_concept` before its branches run, a `PipeBatch` its `item_count` and `branch_pipe_code` before its items run, and a live `PipeCondition` its `evaluated_expression` and `selected_outcome` once its choice is final, before the chosen pipe runs, where each used to register them once it had finished. `GraphTracerProtocol` adds `register_batch_fan_out` and `register_condition_resolution`, and `add_selected_outcome_edge` takes `selected_outcome: str | None` in place of `outcome_value`.
+
### Fixed
- **Trace events written to DynamoDB expire (Breaking)**: the DynamoDB event log now writes every event with a `ttl` in epoch seconds, its write time plus the new `runtime.tracing.dynamodb.retention_days` (30 by default, at least 1), so a table that expires items on `ttl` deletes a run's events, full output data included, where it used to keep them forever. Only events written from this version on carry a `ttl`, so the rows an earlier version wrote stay until they are deleted or given a `ttl` by hand. `DynamoDBEventLog` takes `retention_days` and now takes all its arguments by keyword; `make_event_log` reads the setting from the configuration.
diff --git a/docs/contribute/hub-layering.md b/docs/contribute/hub-layering.md
index c305e09cf..31addb953 100644
--- a/docs/contribute/hub-layering.md
+++ b/docs/contribute/hub-layering.md
@@ -45,7 +45,7 @@ There is deliberately **no** `pipelex.hub`. It was deleted rather than kept as a
| inference | `get_inference_manager`, `get_llm_worker`, `get_img_gen_worker`, `get_extract_worker` |
| content generation | `get_content_generator`, `scoped_content_generator` |
| reporting | `get_report_delegate`, `is_in_isolated_execution` |
-| run mode | `is_dry_run_forced`, `resolve_run_mode_for_boot` |
+| run mode | `is_dry_run_forced`, `resolve_run_mode_for_boot`, `resolve_effective_run_mode` |
| boot orchestrator | `get_boot_orchestrator` |
| tracing | `scoped_event_log`, `get_event_log_override` |
| plugin registries | `get_inference_backend_registry`, `get_model_lister_registry`, `get_orchestrator_registry`, `get_bundle_validator_registry`, `get_storage_provider_registry`, `get_secrets_provider_registry`, `get_optional_service_error_vocabulary` |
@@ -78,7 +78,7 @@ Placement is decided by *kind*, not by who currently imports it. Several accesso
`get_pipe_func_executor_registry` is the one placement worth flagging: it is a plugin registry by kind (which would put it in the kernel layer) but its protocol lives in `pipe_operators/func/` (which puts it in the interpreter layer). It is **interpreter-layer**. See [Known inversions](#known-inversions) for the underlying wart.
-Not everything a hub module exports is an accessor. `resolve_run_mode_for_boot` *applies* the keyless-boot forced-DRY flag rather than reading it, and it lives beside `is_dry_run_forced`, the accessor it consumes. The lifecycle question above still decides placement — the flag is set once at boot and identical for every method the process runs, so the rule that interprets it is kernel-layer by kind. What makes the *module* the right home rather than either caller is that there are now two: the pipe tier's `PipeRunParamsFactory.make_run_params` and the kernel tier's `PipelexKernel.make` both mint run params, and a boot-scoped invariant copied into each is one that drifts. When a rule over hub state acquires a second caller, move the rule to the hub — do not let the second caller reimplement it.
+Not everything a hub module exports is an accessor. `resolve_run_mode_for_boot` *applies* the keyless-boot forced-DRY flag rather than reading it, and it lives beside `is_dry_run_forced`, the accessor it consumes. The lifecycle question above still decides placement — the flag is set once at boot and identical for every method the process runs, so the rule that interprets it is kernel-layer by kind. What makes the *module* the right home rather than either caller is that there are now two: the pipe tier's `PipeRunParamsFactory.make_run_params` and the kernel tier's `PipelexKernel.make` both mint run params, and a boot-scoped invariant copied into each is one that drifts. When a rule over hub state acquires a second caller, move the rule to the hub — do not let the second caller reimplement it. `resolve_effective_run_mode` sits beside it for the same reason: it validates the requested mode against `is_mock_usage` before applying the flag, and its callers are `make_run_params`, `pipeline_run_setup`, which needs the mode before it opens the run's tracer, and `PipelexKernel.make`, which is kernel-layer and so could not reach it in the pipe tier.
## Why the boundary exists
diff --git a/docs/errors/execution-and-runtime.md b/docs/errors/execution-and-runtime.md
index 5154d7282..3e713272d 100644
--- a/docs/errors/execution-and-runtime.md
+++ b/docs/errors/execution-and-runtime.md
@@ -57,5 +57,6 @@ own page. Classes are grouped by subsystem.
- [`EventLogReadError`](event-log-read-error.md) — Event log read
- [`EventLogSchemaMismatchError`](event-log-schema-mismatch-error.md) — Event log schema mismatch
- [`EventLogSetupError`](event-log-setup-error.md) — Event log setup
+- [`TraceEventModeError`](trace-event-mode-error.md) — Trace event mode
[Back to Error Reference](index.md)
diff --git a/docs/errors/trace-event-mode-error.md b/docs/errors/trace-event-mode-error.md
new file mode 100644
index 000000000..0f7073a5d
--- /dev/null
+++ b/docs/errors/trace-event-mode-error.md
@@ -0,0 +1,21 @@
+---
+title: "Trace event mode"
+description: "Reference for the `TraceEventModeError` Pipelex error class."
+---
+
+<!-- pipelex:generated -->
+
+# Trace event mode
+
+Raised when a set of trace events states no single run mode to build a graph in.
+
+| Field | Value |
+|---|---|
+| `error_type` | `TraceEventModeError` |
+| `title` | Trace event mode |
+| `type_uri` | `https://docs.pipelex.com/latest/errors/trace-event-mode-error/` |
+| `error_domain` | _(inherited from parent)_ |
+| Defined in | `pipelex.tracing.exceptions` |
+| Parent class | [`PipelexError`](pipelex-error.md) |
+
+[Back to Error Reference](index.md)
diff --git a/docs/under-the-hood/execution-graph-tracing.md b/docs/under-the-hood/execution-graph-tracing.md
index ceaba87e8..b92979b1b 100644
--- a/docs/under-the-hood/execution-graph-tracing.md
+++ b/docs/under-the-hood/execution-graph-tracing.md
@@ -183,6 +183,8 @@ class GraphSpec(BaseModel):
`meta.format` is always `"mthds"` on newly emitted Pipelex graphs. `meta.mode` records provenance for renderers and shared tooling: Pipelex execution graphs emit `"dry"` for dry-run/mock execution and `"live"` for real execution. The shared renderer also accepts `"static"` for graphs produced by a static MTHDS graph builder.
+Each builder takes the mode from the run itself, never from a default. The in-process tracer takes the `run_mode` its opener states. The assembler takes the `mode` its events state: it is given no mode of its own, and it refuses, with `TraceEventModeError`, a set of events that states more than one mode, which holds more than one run, and an empty set, which states none. `assemble_tracing` records that refusal as the run's `graph_assembly_error` and keeps the run's usage.
+
### Node Types
| NodeKind | Description |
@@ -203,7 +205,7 @@ class GraphSpec(BaseModel):
| `CONTROL` | Execution flow between pipes |
| `DATA` | Data flow (stuff passed between pipes) |
| `CONTAINS` | Parent-child containment (controller → children) |
-| `SELECTED_OUTCOME` | Condition outcome selection |
+| `SELECTED_OUTCOME` | From a live `PipeCondition` to the pipe of the outcome it chose, labelled with the outcome-map key the value matched, unlabelled when the default outcome was taken. Both builders derive it from the condition's `condition_resolved` event and the start of the chosen pipe (see [Decisions at Decision Time](#decisions-at-decision-time)) |
| `BATCH_ITEM` | Batch fan-out: input list → item extracted for each batch iteration |
| `BATCH_AGGREGATE` | Batch fan-in: item outputs → aggregated output list |
| `PARALLEL_COMBINE` | Branch outputs → combined output in PipeParallel |
@@ -237,6 +239,7 @@ trace_context = manager.open_tracer(
data_inclusion=config.data_inclusion,
pipeline_ref_domain="my_domain",
pipeline_ref_main_pipe="my_pipe",
+ run_mode=PipeRunMode.LIVE, # required: whoever opens the trace states the run's mode
)
# 2. Context flows through JobMetadata to each pipe
@@ -293,6 +296,49 @@ Semantics:
This is what lets a graph-producing dry-run trace entirely in memory (no NDJSON file, no DynamoDB round-trip). Both dry-run entrypoints rely on it: `dry_run_pipe_in_process` (`pipelex/pipe_run/dry_run_in_process.py` — the graph arm of protocol `validate` and of the single Temporal validation activity) and `dry_run_pipeline` (`pipelex/pipeline/dry_run_pipeline.py`) itself — these functions exist to produce a graph, so they install their own scoped `InMemoryEventLog` rather than depending on the host having tracing configured (a host with `runtime.tracing.is_enabled = false`, like pipelex-api's `/validate` in direct mode, still gets its graph). A run that nonetheless finishes without a graph raises the typed `DryRunGraphNotProducedError`.
+### Trace Events
+
+The tracer writes one event for each thing that happens in the run, in the order it happens (`pipelex/tracing/trace_events.py`). Every backend carries the events whole: the NDJSON and DynamoDB logs store each one as its JSON and read it back through the discriminated union `AnyTraceEvent`, and the in-memory and buffering logs hold the objects themselves. Every event carries these fields:
+
+- `pipeline_run_id`, `workflow_id` and `writer_id`, which name the run, the workflow and the process that wrote it;
+- `timestamp`, and `sequence`, which rises with each event one writer writes;
+- `event_kind`, which tells the event's kind;
+- `mode`, `live` or `dry`, the mode of the run that wrote it.
+
+The run's `TraceContext` carries the mode as `run_mode`, so a usage event that an activity writes from the context alone states it too. `mode` is required: an event stored by a version that did not write it is refused with `EventLogSchemaMismatchError` rather than read as live, and running the pipeline again writes a log in the current shape. `run_mode` is required too, with no default, on `TraceContext`, on `GraphTracerManager.open_tracer` and on the tracer's `setup`, since a default would stamp a dry run's events as live: whoever opens a trace states the mode, and a workflow that opens a tracer of its own for part of a run passes the `run_mode` of the context it received.
+
+The mode a trace states is the one the run's pipes run in, which is not always the one requested: a keyless boot (`needs_inference=False`) runs a requested live run dry. `pipeline_run_setup` therefore resolves the run's mode once, before it opens the tracer, through `resolve_effective_run_mode` (`pipelex/runtime_hub.py`), which first refuses `is_mock_usage` on a live request and then applies the keyless coercion. The tracer, the trace context, the OpenTelemetry context and the pipe job all take that one value, so a forced dry run states `dry` on every event and on its graph, and opens no OpenTelemetry trace, as no dry run does. The kernel tier holds to the same rule from the other side: its caller opens the trace, so `PipelexKernel.make` refuses, with `TraceContextRunModeError`, a trace context stating another mode than the one `resolve_effective_run_mode` gives the run (see [The Pipelex Kernel](pipelex-kernel.md)).
+
+| `event_kind` | Written when | What it carries |
+|---|---|---|
+| `pipe_start` | A pipe starts | The node, its parent, the pipe's code, type, description and domain, its node kind and its inputs; a `PipeSequence` also states `step_count`, the number of steps its author wrote, and a `PipeParallel` states `branch_count`, the number of its branches, both `null` on every other pipe |
+| `pipe_end_success` | A pipe ends with an output | The node, its end time, its output and its metrics |
+| `pipe_end_error` | A pipe fails | The node, its end time and the error |
+| `pipe_end_skipped` | A pipe is lifted because a plain input is absent | The node, its end time, the reason, and the empty list a plural output still writes |
+| `edge` | The tracer draws an edge | Its source, target, kind, label and stuff digests |
+| `controller_output` | A controller registers an output it produced | The node and the output |
+| `batch_item` | A `PipeBatch` extracts an item from its input list | The list and item digests, the item index and the batch's node |
+| `batch_aggregate` | A `PipeBatch` adds an item's result to its output list | The output list and item digests, the item index and the batch's node |
+| `parallel_combine` | A `PipeParallel` combines its branch outputs | The combined digest, the branch digests and their producers |
+| `condition_output_merge` | A dry-run `PipeCondition` merges its outcomes' outputs (see [A Dry-Run Condition's Output](#a-dry-run-conditions-output)) | The merge record |
+| `condition_resolved` | A live `PipeCondition`'s choice is final, before anything runs under it | The condition's node, `rendered_value`, the value its expression rendered, and `selected_outcome`, the outcome-map key the value matched, or `null` when the default outcome was taken |
+| `batch_fanned_out` | A `PipeBatch` knows how many items it runs, before any item starts | The batch's node, `item_count`, and the pipe each item runs: `branch_pipe_code`, its bare code, `branch_pipe_ref`, its domain-qualified reference, and `branch_pipe_type` |
+| `execution_data` | A pipe records details of its execution | The node, and the entries merged into its `execution_data` |
+| `usage_report` | An inference job reports what it used | The node it is attributed to and its usage |
+
+#### Decisions at Decision Time
+
+A controller records each decision the moment it is final, so a reader following the trace while the run goes on knows a condition's outcome and a batch's width before the work under them runs. A live `PipeCondition` writes `condition_resolved` once its choice is final: at once for an outcome mapped to `continue` or `fail`, which runs nothing, and, for an outcome that runs a pipe, once it has found that pipe's inputs in working memory. A chosen pipe whose inputs are missing fails the condition before it runs, so the trace records no outcome taken, neither the event nor the execution data. A `PipeBatch` writes `batch_fanned_out` before its first item starts. A dry run runs every outcome of a condition and chooses none, so it writes no `condition_resolved` and no `SELECTED_OUTCOME` edge; a dry batch still writes `batch_fanned_out` for the items it runs. Both are graph events about a node of the tracer's graph: a run that emits no graph events, in costs-only mode, writes neither, and a tracer records a decision only on a node it started itself, as it records execution data, so a decision looked up on a tracer that does not hold the controller's node is dropped rather than written for a node that tracer's graph lacks.
+
+The controllers register their execution data at the same moments:
+
+- A `PipeSequence` registers `step_count` as it starts, and `current_step`, the step's number among the steps its author wrote, as each step starts. The binding written for a dotted `batch_over` shares the number of the batch step after it.
+- A `PipeParallel` registers `branch_count`, `add_each_output` and `combined_output_concept` before any branch runs.
+- A `PipeBatch` registers `item_count` and `branch_pipe_code` with its fan-out. The two spellings of `branch_pipe_code` differ: the execution data carries the blueprint's field as its author wrote it, which may be domain-qualified, such as `my_domain.summarize`, while the `batch_fanned_out` event's `branch_pipe_code` is always the bare code, `summarize`, with `branch_pipe_ref` the qualified reference, `my_domain.summarize`.
+- A live `PipeCondition` registers `evaluated_expression` and `selected_outcome` with its resolution. Here `selected_outcome` is the outcome itself, the code of the pipe it runs or `continue` or `fail`, where the `condition_resolved` event names the outcome-map key.
+
+Both graph builders derive the `SELECTED_OUTCOME` edge from two facts of the run, by the one rule in `pipelex/graph/selected_outcome.py`: a condition's `condition_resolved`, and the start of a pipe whose parent is that condition's node, which can only be the pipe of the outcome it chose. The in-process tracer applies the rule at teardown to the resolutions it recorded and the nodes it holds; the assembler applies it after reading every event, whichever tracer wrote them. The edge is never drawn when the chosen pipe starts, because in a distributed run that pipe can start in another workflow, under a tracer of its own that never saw the resolution, so the edge would be missing from one graph of the run and present in the other. An outcome mapped to `continue` or `fail` starts no pipe under the condition, so it has no edge, and a resolution whose condition node is not in the graph draws none either, since the edge would have no source. The tracer still writes the edge as an `edge` event at teardown, as it does every derived edge, and the assembler regenerates it rather than reading that event, so it is never doubled. Neither builder reads `batch_fanned_out`, because a batch's items are already its children. Both builders therefore build the same graph from the same events.
+
### TraceContext Propagation
TraceContext is a serializable context that flows through pipe execution:
@@ -303,6 +349,7 @@ class TraceContext(BaseModel):
parent_node_id: str | None # Parent pipe's node ID
node_sequence: int # Counter for generating node IDs
data_inclusion: DataInclusionConfig # What data to capture
+ run_mode: PipeRunMode # Live or dry, stated on every trace event
def copy_for_child(self, child_node_id: str, *, next_sequence: int) -> TraceContext:
"""Create context for nested pipe execution."""
@@ -311,6 +358,7 @@ class TraceContext(BaseModel):
parent_node_id=child_node_id,
node_sequence=next_sequence,
data_inclusion=self.data_inclusion,
+ run_mode=self.run_mode,
)
```
@@ -564,6 +612,8 @@ validate_graphspec(graph_spec)
| File | Purpose |
|------|---------|
| `pipelex/graph/graphspec.py` | Canonical GraphSpec model |
+| `pipelex/tracing/trace_events.py` | The trace events and their discriminated union |
+| `pipelex/tracing/graphspec_assembler.py` | Builds the GraphSpec from a run's trace events |
| `pipelex/tracing/usage_attribution.py` | Per-node usage accumulation and subtree rollup |
| `pipelex/graph/graph_tracer.py` | GraphTracer implementation |
| `pipelex/graph/graph_tracer_manager.py` | Singleton manager for tracers |
@@ -572,6 +622,7 @@ validate_graphspec(graph_spec)
| `pipelex/system/data_inclusion_config.py` | Data-capture flags carried by the trace context, surfaced in the TOML under `interpreter.pipeline_execution.graph.data_inclusion` |
| `pipelex/graph/graph_analysis.py` | Pre-computed graph analysis |
| `pipelex/graph/condition_output_merge.py` | The merge record of a dry-run condition and the rewrite both graph builders apply |
+| `pipelex/graph/selected_outcome.py` | The rule both graph builders derive a live condition's `SELECTED_OUTCOME` edges by |
| `pipelex/graph/graph_factory.py` | Output generation factory |
| `pipelex/graph/graph_config.py` | Configuration models |
| `pipelex/graph/validation.py` | GraphSpec validation |
diff --git a/docs/under-the-hood/pipelex-kernel.md b/docs/under-the-hood/pipelex-kernel.md
index f4ac015dd..f080d3d63 100644
--- a/docs/under-the-hood/pipelex-kernel.md
+++ b/docs/under-the-hood/pipelex-kernel.md
@@ -34,7 +34,7 @@ That boot stands up the model deck, the content generator, the class registry, t
**`read_scope` is required too, and `None` is a value you pass, not a default.** It bounds what the run may read: every `pipelex-storage://` key a step reads must lie under it, compared segment by segment, and no step reads a bare path or a `file://` URI from the local disk. `https://` and `data:` URLs are untouched. A value can carry any URL its method or its model chose, so the check runs where values are read, in every content-generation leaf before the dry-run branch, and `shape_inputs` applies it to a table given as a CSV input, which is always a local file. The storage scope must lie under the read scope, since a run reads its own outputs back, and a mismatch is refused at construction. A multi-tenant host passes its tenant's prefix (the hosted platform passes the organization id); `None` is the explicit statement that the run is unscoped, which is the truth for a single-tenant service or a laptop. It has no default for the storage scope's reason: a host that forgot it would run unscoped, and an unscoped run reads every key the process's credentials reach. A refused read raises `UriReadRefusedError`, a caller-facing input error that names where the URL sat and never quotes it.
-`needs_inference=False` boots keyless (every enabled backend and model loaded, no credential resolved, no model-deck validation) and sets the forced-DRY flag: every run the process initiates is coerced to `run_mode=DRY`, so the leaves mock instead of calling a provider. `PipelexKernel.make` applies that rule through the same `runtime_hub.resolve_run_mode_for_boot` the pipe tier uses — a second copy of the rule at a second factory is how the two would drift apart.
+`needs_inference=False` boots keyless (every enabled backend and model loaded, no credential resolved, no model-deck validation) and sets the forced-DRY flag: every run the process initiates is coerced to `run_mode=DRY`, so the leaves mock instead of calling a provider. `PipelexKernel.make` applies that rule through the same `runtime_hub.resolve_effective_run_mode` the pipe tier uses, which refuses `is_mock_usage` on a live request before it coerces — a second copy of the rule at a second factory is how the two would drift apart.
---
@@ -149,22 +149,27 @@ Nothing derived from config or the model deck is cached on the instance — reso
Everything else is yours:
```python
-from pipelex.runtime_hub import get_report_delegate
+from pipelex.runtime_hub import get_report_delegate, resolve_effective_run_mode
+from pipelex.system.pipe_run_mode import PipeRunMode
from pipelex.system.trace_context import TraceContext
from pipelex.tracing.in_memory_event_log import InMemoryEventLog
from pipelex.tracing.usage_aggregator import UsageAggregator
+# The mode the run takes on this boot: a boot without inference runs a live request dry.
+run_mode = resolve_effective_run_mode(requested=PipeRunMode.LIVE, is_mock_usage=False)
event_log = InMemoryEventLog()
-trace_context = TraceContext(graph_id=run_id, data_inclusion=data_inclusion, emit_graph_events=False, emit_usage_events=True)
+trace_context = TraceContext(graph_id=run_id, data_inclusion=data_inclusion, emit_graph_events=False, emit_usage_events=True, run_mode=run_mode)
get_report_delegate().set_event_log(context_key=trace_context.lookup_key, event_log=event_log, workflow_id="direct", pipeline_run_id=run_id)
try:
- kernel = PipelexKernel.make(user_id="my-service", storage_scope="my-service", read_scope=None, trace_context=trace_context)
+ kernel = PipelexKernel.make(user_id="my-service", storage_scope="my-service", read_scope=None, run_mode=run_mode, trace_context=trace_context)
...
tokens_usages = UsageAggregator.aggregate(event_log.read_events(run_id))
finally:
get_report_delegate().clear_event_log(context_key=trace_context.lookup_key)
```
+The trace context states the mode the run takes, which `resolve_effective_run_mode` gives: every usage event a step writes states the context's `run_mode`, so a context saying live for a run whose steps run dry would stamp the whole run live. `PipelexKernel.make` refuses a context stating another mode with `TraceContextRunModeError` (`pipelex/kernel/exceptions.py`), a `ValueError` raised before any step runs. It refuses rather than correct its own copy because the trace is the caller's, possibly shared with writers the kernel never sees, which a corrected copy would leave disagreeing with the kernel's events.
+
Passing a `trace_context` adopts its `graph_id` as the run's `pipeline_run_id`, and passing a `pipeline_run_id` that differs from it raises `ValueError` rather than letting either win. The two are one identity: letting them diverge would scatter a single run's usage events across two ids, because the registered-context emit path stamps the event log's id while the runner fallback stamps the metadata's — and a read-back keyed on either would silently miss the other's.
`pipelex.tracing` holds both halves a caller needs (`make_event_log` for a configured backend, `UsageAggregator` for the read-back) and is kernel-layer, so none of this costs the boot contract. The records that come out are the same [`TokensUsage` wire records](./tokens-usage-wire-records.md) an `/execute` response carries — pinned by an integration test that runs the same step through both callers and compares them.
diff --git a/pipelex/graph/graph_tracer.py b/pipelex/graph/graph_tracer.py
index 407598d89..a79ce6f7f 100644
--- a/pipelex/graph/graph_tracer.py
+++ b/pipelex/graph/graph_tracer.py
@@ -12,7 +12,6 @@ from pipelex.graph.graphspec import (
EdgeSpec,
ErrorSpec,
GraphSpec,
- GraphSpecMode,
IOSpec,
NodeIOSpec,
NodeKind,
@@ -23,13 +22,17 @@ from pipelex.graph.graphspec import (
make_graphspec_meta,
output_digest_is_optional,
)
+from pipelex.graph.selected_outcome import derive_selected_outcome_edges
from pipelex.system.data_inclusion_config import DataInclusionConfig
+from pipelex.system.pipe_run_mode import PipeRunMode
from pipelex.system.trace_context import TraceContext
from pipelex.tracing.event_log_protocol import EventLogProtocol # ruff: ignore[typing-only-first-party-import] - used in __init__ annotations
from pipelex.tracing.trace_events import (
BatchAggregateEvent,
+ BatchFannedOutEvent,
BatchItemEvent,
ConditionOutputMergeEvent,
+ ConditionResolvedEvent,
ControllerOutputEvent,
EdgeEvent,
ExecutionDataEvent,
@@ -161,7 +164,10 @@ class GraphTracer(GraphTracerProtocol):
# Registries for pipe and concept data (keyed by pipe_ref and concept_ref)
self._pipe_registry: dict[str, dict[str, Any]] = {}
self._concept_registry: dict[str, dict[str, Any]] = {}
- self._mode: GraphSpecMode = GraphSpecMode.LIVE
+ self._run_mode: PipeRunMode = PipeRunMode.LIVE
+ # The live conditions that have chosen their outcome, by condition node id, each with the outcome-map key it
+ # matched (None for the default outcome): teardown derives the SELECTED_OUTCOME edges from them
+ self._condition_resolutions: dict[str, str | None] = {}
@property
def is_active(self) -> bool:
@@ -235,7 +241,7 @@ class GraphTracer(GraphTracerProtocol):
pipeline_run_id: str | None = None,
emit_graph_events: bool = True,
emit_usage_events: bool = True,
- mode: GraphSpecMode = GraphSpecMode.LIVE,
+ run_mode: PipeRunMode,
) -> TraceContext:
"""Initialize tracing for a new pipeline run.
@@ -253,7 +259,8 @@ class GraphTracer(GraphTracerProtocol):
teardown skips the discarded spec build; the returned TraceContext carries the flag.
emit_usage_events: Whether this run emits usage (cost) events. Stamped onto the returned
TraceContext so it is born with the correct flag.
- mode: Provenance mode to stamp onto generated GraphSpecs.
+ run_mode: Whether the run is live or a dry run: stamped on every trace event and on the returned
+ TraceContext, and turned into the provenance mode of the generated GraphSpec.
"""
self._is_active = True
self._emit_graph_events = emit_graph_events
@@ -278,7 +285,8 @@ class GraphTracer(GraphTracerProtocol):
self._event_sequence = 0
self._pipe_registry = {}
self._concept_registry = {}
- self._mode = mode
+ self._run_mode = run_mode
+ self._condition_resolutions = {}
return TraceContext(
graph_id=graph_id,
@@ -287,6 +295,7 @@ class GraphTracer(GraphTracerProtocol):
data_inclusion=data_inclusion,
emit_graph_events=emit_graph_events,
emit_usage_events=emit_usage_events,
+ run_mode=run_mode,
)
@override
@@ -314,6 +323,7 @@ class GraphTracer(GraphTracerProtocol):
self._generate_batch_item_edges()
self._generate_batch_aggregate_edges()
self._generate_parallel_combine_edges()
+ self._generate_selected_outcome_edges()
self._is_active = False
@@ -326,7 +336,7 @@ class GraphTracer(GraphTracerProtocol):
pipeline_ref=self._pipeline_ref or PipelineRef(),
nodes=nodes,
edges=self._edges,
- meta=make_graphspec_meta(mode=self._mode),
+ meta=make_graphspec_meta(mode=self._run_mode.graphspec_mode),
pipe_registry=dict(self._pipe_registry),
concept_registry=dict(self._concept_registry),
)
@@ -353,7 +363,8 @@ class GraphTracer(GraphTracerProtocol):
self._event_sequence = 0
self._pipe_registry = {}
self._concept_registry = {}
- self._mode = GraphSpecMode.LIVE
+ self._run_mode = PipeRunMode.LIVE
+ self._condition_resolutions = {}
return graph
@@ -494,6 +505,20 @@ class GraphTracer(GraphTracerProtocol):
target_stuff_digest=combined_stuff_code,
)
+ def _generate_selected_outcome_edges(self) -> None:
+ """Generate the SELECTED_OUTCOME edges from each resolved condition to the pipe it then started.
+
+ The rule is the one the event-replay assembler applies (`derive_selected_outcome_edges`), so both graphs of a run
+ carry the same edges.
+ """
+ parent_by_node = {node_id: node_data.parent_node_id for node_id, node_data in self._nodes.items()}
+ for edge in derive_selected_outcome_edges(resolutions=self._condition_resolutions, parent_by_node=parent_by_node):
+ self.add_selected_outcome_edge(
+ condition_node_id=edge.condition_node_id,
+ outcome_node_id=edge.outcome_node_id,
+ selected_outcome=edge.selected_outcome,
+ )
+
@override
def register_batch_item_extraction(
self,
@@ -529,6 +554,7 @@ class GraphTracer(GraphTracerProtocol):
pipeline_run_id=self._event_pipeline_run_id,
writer_id=self._event_writer_id(),
workflow_id=self._workflow_id,
+ mode=self._run_mode,
timestamp=datetime.now(UTC),
sequence=self._next_event_sequence(),
list_stuff_code=list_stuff_code,
@@ -573,6 +599,7 @@ class GraphTracer(GraphTracerProtocol):
pipeline_run_id=self._event_pipeline_run_id,
writer_id=self._event_writer_id(),
workflow_id=self._workflow_id,
+ mode=self._run_mode,
timestamp=datetime.now(UTC),
sequence=self._next_event_sequence(),
output_list_stuff_code=output_list_stuff_code,
@@ -616,6 +643,7 @@ class GraphTracer(GraphTracerProtocol):
pipeline_run_id=self._event_pipeline_run_id,
writer_id=self._event_writer_id(),
workflow_id=self._workflow_id,
+ mode=self._run_mode,
timestamp=datetime.now(UTC),
sequence=self._next_event_sequence(),
combined_stuff_code=combined_stuff_code,
@@ -647,12 +675,99 @@ class GraphTracer(GraphTracerProtocol):
pipeline_run_id=self._event_pipeline_run_id,
writer_id=self._event_writer_id(),
workflow_id=self._workflow_id,
+ mode=self._run_mode,
timestamp=datetime.now(UTC),
sequence=self._next_event_sequence(),
merge=merge,
)
)
+ @override
+ def register_batch_fan_out(
+ self,
+ *,
+ batch_node_id: str,
+ item_count: int,
+ branch_pipe_code: str,
+ branch_pipe_ref: str,
+ branch_pipe_type: str,
+ ) -> None:
+ """Record that a PipeBatch knows how many items it runs, before it starts any of them.
+
+ Nothing of the graph depends on it: the event is for whoever reads the trace while the run goes on. It is a graph
+ event, recorded on a node of this tracer's graph: a run that emits no graph events (costs-only) records none, and
+ neither does a tracer that never started the batch's node.
+
+ Args:
+ batch_node_id: The PipeBatch's node ID.
+ item_count: The number of items the batch runs.
+ branch_pipe_code: The bare code of the pipe each item runs.
+ branch_pipe_ref: The `domain.code` reference of the pipe each item runs.
+ branch_pipe_type: The type of the pipe each item runs.
+ """
+ if not self._is_active or not self._emit_graph_events:
+ return
+ if batch_node_id not in self._nodes:
+ return
+ if self._event_log is not None:
+ self._emit_event(
+ BatchFannedOutEvent(
+ pipeline_run_id=self._event_pipeline_run_id,
+ writer_id=self._event_writer_id(),
+ workflow_id=self._workflow_id,
+ mode=self._run_mode,
+ timestamp=datetime.now(UTC),
+ sequence=self._next_event_sequence(),
+ node_id=batch_node_id,
+ item_count=item_count,
+ branch_pipe_code=branch_pipe_code,
+ branch_pipe_ref=branch_pipe_ref,
+ branch_pipe_type=branch_pipe_type,
+ )
+ )
+
+ @override
+ def register_condition_resolution(
+ self,
+ *,
+ condition_node_id: str,
+ rendered_value: str,
+ selected_outcome: str | None,
+ ) -> None:
+ """Record that a live PipeCondition has chosen its outcome, before the chosen pipe starts.
+
+ The graph's SELECTED_OUTCOME edge runs from the condition to the pipe it then starts under its node, derived at
+ teardown, as the event-replay assembler derives it from this resolution's event and that pipe's start, whichever
+ tracer wrote the start. An outcome mapped to `continue` or `fail` starts no pipe, and so draws no edge. A run that
+ emits no graph events (costs-only) records no resolution, and neither does a tracer that never started the
+ condition's node, which would otherwise write an event the assembler could draw an edge from a node absent from
+ the graph with.
+
+ Args:
+ condition_node_id: The PipeCondition's node ID.
+ rendered_value: The value the condition's expression rendered.
+ selected_outcome: The outcome-map key the value matched, or None when the default outcome was taken.
+ """
+ if not self._is_active or not self._emit_graph_events:
+ return
+ if condition_node_id not in self._nodes:
+ return
+ self._condition_resolutions[condition_node_id] = selected_outcome
+ if self._event_log is not None:
+ self._emit_event(
+ ConditionResolvedEvent(
+ pipeline_run_id=self._event_pipeline_run_id,
+ writer_id=self._event_writer_id(),
+ workflow_id=self._workflow_id,
+ mode=self._run_mode,
+ timestamp=datetime.now(UTC),
+ sequence=self._next_event_sequence(),
+ node_id=condition_node_id,
+ rendered_value=rendered_value,
+ selected_outcome=selected_outcome,
+ )
+ )
+
@override
def on_pipe_start(
self,
@@ -667,6 +782,8 @@ class GraphTracer(GraphTracerProtocol):
concept_data: list[dict[str, Any]] | None = None,
description: str | None = None,
domain_code: str | None = None,
+ step_count: int | None = None,
+ branch_count: int | None = None,
) -> tuple[str, TraceContext]:
"""Record the start of a pipe execution."""
if not self._is_active:
@@ -711,6 +828,7 @@ class GraphTracer(GraphTracerProtocol):
pipeline_run_id=self._event_pipeline_run_id,
writer_id=self._event_writer_id(),
workflow_id=self._workflow_id,
+ mode=self._run_mode,
timestamp=started_at,
sequence=self._next_event_sequence(),
node_id=node_id,
@@ -723,6 +841,8 @@ class GraphTracer(GraphTracerProtocol):
input_specs=input_specs or [],
pipe_data=pipe_data or {},
concept_data=concept_data or [],
+ step_count=step_count,
+ branch_count=branch_count,
)
)
@@ -764,6 +884,7 @@ class GraphTracer(GraphTracerProtocol):
pipeline_run_id=self._event_pipeline_run_id,
writer_id=self._event_writer_id(),
workflow_id=self._workflow_id,
+ mode=self._run_mode,
timestamp=datetime.now(UTC),
sequence=self._next_event_sequence(),
node_id=node_id,
@@ -824,6 +945,7 @@ class GraphTracer(GraphTracerProtocol):
pipeline_run_id=self._event_pipeline_run_id,
writer_id=self._event_writer_id(),
workflow_id=self._workflow_id,
+ mode=self._run_mode,
timestamp=ended_at,
sequence=self._next_event_sequence(),
node_id=node_id,
@@ -869,6 +991,7 @@ class GraphTracer(GraphTracerProtocol):
pipeline_run_id=self._event_pipeline_run_id,
writer_id=self._event_writer_id(),
workflow_id=self._workflow_id,
+ mode=self._run_mode,
timestamp=datetime.now(UTC),
sequence=self._next_event_sequence(),
node_id=node_id,
@@ -910,6 +1033,7 @@ class GraphTracer(GraphTracerProtocol):
pipeline_run_id=self._event_pipeline_run_id,
writer_id=self._event_writer_id(),
workflow_id=self._workflow_id,
+ mode=self._run_mode,
timestamp=ended_at,
sequence=self._next_event_sequence(),
node_id=node_id,
@@ -967,6 +1091,7 @@ class GraphTracer(GraphTracerProtocol):
pipeline_run_id=self._event_pipeline_run_id,
writer_id=self._event_writer_id(),
workflow_id=self._workflow_id,
+ mode=self._run_mode,
timestamp=ended_at,
sequence=self._next_event_sequence(),
node_id=node_id,
@@ -1014,6 +1139,7 @@ class GraphTracer(GraphTracerProtocol):
pipeline_run_id=self._event_pipeline_run_id,
writer_id=self._event_writer_id(),
workflow_id=self._workflow_id,
+ mode=self._run_mode,
timestamp=datetime.now(UTC),
sequence=self._next_event_sequence(),
edge_id=edge_id,
@@ -1032,18 +1158,19 @@ class GraphTracer(GraphTracerProtocol):
*,
condition_node_id: str,
outcome_node_id: str,
- outcome_value: str,
+ selected_outcome: str | None,
) -> None:
- """Add an edge indicating a selected condition outcome.
+ """Add an edge from a condition to the pipe of the outcome it chose.
Args:
condition_node_id: The condition pipe's node ID.
- outcome_node_id: The selected outcome pipe's node ID.
- outcome_value: The value that was selected.
+ outcome_node_id: The node ID of the pipe the chosen outcome runs.
+ selected_outcome: The outcome-map key the condition matched, which labels the edge, or None when it took
+ its default outcome, which leaves the edge unlabelled.
"""
self.add_edge(
source_node_id=condition_node_id,
target_node_id=outcome_node_id,
edge_kind=EdgeKind.SELECTED_OUTCOME,
- label=outcome_value,
+ label=selected_outcome,
)
diff --git a/pipelex/graph/graph_tracer_manager.py b/pipelex/graph/graph_tracer_manager.py
index a08f9f2ac..3312e7553 100644
--- a/pipelex/graph/graph_tracer_manager.py
+++ b/pipelex/graph/graph_tracer_manager.py
@@ -7,8 +7,9 @@ from pipelex import log
from pipelex.graph.condition_output_merge import ConditionOutputMerge
from pipelex.graph.graph_tracer import GraphTracer
from pipelex.graph.graph_tracer_protocol import GraphTracerProtocol
-from pipelex.graph.graphspec import GraphSpec, GraphSpecMode, IOSpec, NodeKind
+from pipelex.graph.graphspec import GraphSpec, IOSpec, NodeKind
from pipelex.system.data_inclusion_config import DataInclusionConfig
+from pipelex.system.pipe_run_mode import PipeRunMode
from pipelex.system.registries.singleton import ABCSingletonMeta, MetaSingleton
from pipelex.system.trace_context import TraceContext
from pipelex.tools.log.error_fields import error_fields
@@ -107,7 +108,7 @@ class GraphTracerManager(metaclass=ABCSingletonMeta):
emit_graph_events: bool = True,
emit_usage_events: bool = True,
describe_pipe_io: bool = True,
- mode: GraphSpecMode = GraphSpecMode.LIVE,
+ run_mode: PipeRunMode,
) -> TraceContext:
"""Create and initialize a new tracer for a pipeline run.
@@ -131,7 +132,9 @@ class GraphTracerManager(metaclass=ABCSingletonMeta):
describe_pipe_io: Whether the run builds the I/O artifacts describing its graphspec's
data. Read by the run, never by the tracer, so it is stamped on the context here
rather than threaded into setup.
- mode: Provenance mode to stamp onto generated GraphSpecs.
+ run_mode: Whether the run is live or a dry run: stamped on every trace event and on the returned
+ TraceContext, and turned into the provenance mode of the generated GraphSpec. A workflow that opens
+ a tracer of its own for part of a run passes the mode of the context it received.
Returns:
Initial TraceContext to pass through JobMetadata.
@@ -172,7 +175,7 @@ class GraphTracerManager(metaclass=ABCSingletonMeta):
pipeline_run_id=pipeline_run_id,
emit_graph_events=emit_graph_events,
emit_usage_events=emit_usage_events,
- mode=mode,
+ run_mode=run_mode,
)
# Set the tracer_key on the TraceContext so downstream lookups use the same key
if tracer_key is not None:
@@ -242,6 +245,8 @@ class GraphTracerManager(metaclass=ABCSingletonMeta):
concept_data: list[dict[str, Any]] | None = None,
description: str | None = None,
domain_code: str | None = None,
+ step_count: int | None = None,
+ branch_count: int | None = None,
) -> tuple[str | None, TraceContext | None]:
"""Record the start of a pipe execution.
@@ -256,6 +261,8 @@ class GraphTracerManager(metaclass=ABCSingletonMeta):
concept_data: Optional list of serialized concept dicts for the concept registry.
description: Optional human-readable pipe description (mirrored onto NodeSpec).
domain_code: Optional domain code of the pipe (mirrored onto NodeSpec).
+ step_count: On a PipeSequence, the number of steps its author wrote.
+ branch_count: On a PipeParallel, the number of its branches.
Returns:
Tuple of (node_id, child_trace_context) if tracing is active, (None, None) otherwise.
@@ -275,6 +282,8 @@ class GraphTracerManager(metaclass=ABCSingletonMeta):
concept_data=concept_data,
description=description,
domain_code=domain_code,
+ step_count=step_count,
+ branch_count=branch_count,
)
def on_pipe_end_success(
@@ -528,3 +537,59 @@ class GraphTracerManager(metaclass=ABCSingletonMeta):
if tracer is None:
return
tracer.register_condition_output_merge(merge=merge)
+
+ def register_batch_fan_out(
+ self,
+ *,
+ lookup_key: str,
+ batch_node_id: str,
+ item_count: int,
+ branch_pipe_code: str,
+ branch_pipe_ref: str,
+ branch_pipe_type: str,
+ ) -> None:
+ """Register that a PipeBatch knows how many items it runs, before it starts any of them.
+
+ Args:
+ lookup_key: The tracer lookup key.
+ batch_node_id: The PipeBatch's node ID.
+ item_count: The number of items the batch runs.
+ branch_pipe_code: The bare code of the pipe each item runs.
+ branch_pipe_ref: The `domain.code` reference of the pipe each item runs.
+ branch_pipe_type: The type of the pipe each item runs.
+ """
+ tracer = self._get_tracer(lookup_key)
+ if tracer is None:
+ return
+ tracer.register_batch_fan_out(
+ batch_node_id=batch_node_id,
+ item_count=item_count,
+ branch_pipe_code=branch_pipe_code,
+ branch_pipe_ref=branch_pipe_ref,
+ branch_pipe_type=branch_pipe_type,
+ )
+
+ def register_condition_resolution(
+ self,
+ *,
+ lookup_key: str,
+ condition_node_id: str,
+ rendered_value: str,
+ selected_outcome: str | None,
+ ) -> None:
+ """Register that a live PipeCondition has chosen its outcome, before the chosen pipe starts.
+
+ Args:
+ lookup_key: The tracer lookup key.
+ condition_node_id: The PipeCondition's node ID.
+ rendered_value: The value the condition's expression rendered.
+ selected_outcome: The outcome-map key the value matched, or None when the default outcome was taken.
+ """
+ tracer = self._get_tracer(lookup_key)
+ if tracer is None:
+ return
+ tracer.register_condition_resolution(
+ condition_node_id=condition_node_id,
+ rendered_value=rendered_value,
+ selected_outcome=selected_outcome,
+ )
diff --git a/pipelex/graph/graph_tracer_protocol.py b/pipelex/graph/graph_tracer_protocol.py
index fd9bce8d7..0f3b7fc35 100644
--- a/pipelex/graph/graph_tracer_protocol.py
+++ b/pipelex/graph/graph_tracer_protocol.py
@@ -4,8 +4,9 @@ from typing import Any, Protocol
from typing_extensions import override
from pipelex.graph.condition_output_merge import ConditionOutputMerge
-from pipelex.graph.graphspec import EdgeKind, GraphSpec, GraphSpecMode, IOSpec, NodeKind
+from pipelex.graph.graphspec import EdgeKind, GraphSpec, IOSpec, NodeKind
from pipelex.system.data_inclusion_config import DataInclusionConfig
+from pipelex.system.pipe_run_mode import PipeRunMode
from pipelex.system.trace_context import TraceContext
from pipelex.tracing.event_log_protocol import EventLogProtocol # ruff: ignore[typing-only-first-party-import] - used in setup signature
@@ -29,7 +30,7 @@ class GraphTracerProtocol(Protocol):
pipeline_run_id: str | None = None,
emit_graph_events: bool = True,
emit_usage_events: bool = True,
- mode: GraphSpecMode = GraphSpecMode.LIVE,
+ run_mode: PipeRunMode,
) -> TraceContext:
"""Initialize tracing for a new pipeline run.
@@ -45,7 +46,8 @@ class GraphTracerProtocol(Protocol):
Stamped onto the returned TraceContext so it is born with the correct flag.
emit_usage_events: Whether usage (cost) trace events should be emitted for this run.
Stamped onto the returned TraceContext so it is born with the correct flag.
- mode: Provenance mode to stamp onto generated GraphSpecs.
+ run_mode: Whether the run is live or a dry run: stamped on every trace event and on the returned
+ TraceContext, and turned into the provenance mode of the generated GraphSpec.
Returns:
Initial TraceContext to pass through JobMetadata.
@@ -73,6 +75,8 @@ class GraphTracerProtocol(Protocol):
concept_data: list[dict[str, Any]] | None = None,
description: str | None = None,
domain_code: str | None = None,
+ step_count: int | None = None,
+ branch_count: int | None = None,
) -> tuple[str, TraceContext]:
"""Record the start of a pipe execution.
@@ -87,6 +91,8 @@ class GraphTracerProtocol(Protocol):
concept_data: Optional list of serialized concept dicts for the concept registry.
description: Optional human-readable pipe description (mirrored onto NodeSpec).
domain_code: Optional domain code of the pipe (mirrored onto NodeSpec).
+ step_count: On a PipeSequence, the number of steps its author wrote.
+ branch_count: On a PipeParallel, the number of its branches.
Returns:
Tuple of (node_id for this pipe, updated TraceContext for children).
@@ -276,6 +282,45 @@ class GraphTracerProtocol(Protocol):
"""
...
+ def register_batch_fan_out(
+ self,
+ *,
+ batch_node_id: str,
+ item_count: int,
+ branch_pipe_code: str,
+ branch_pipe_ref: str,
+ branch_pipe_type: str,
+ ) -> None:
+ """Register that a PipeBatch knows how many items it runs, before it starts any of them.
+
+ Args:
+ batch_node_id: The PipeBatch's node ID.
+ item_count: The number of items the batch runs.
+ branch_pipe_code: The bare code of the pipe each item runs.
+ branch_pipe_ref: The `domain.code` reference of the pipe each item runs.
+ branch_pipe_type: The type of the pipe each item runs.
+ """
+ ...
+
+ def register_condition_resolution(
+ self,
+ *,
+ condition_node_id: str,
+ rendered_value: str,
+ selected_outcome: str | None,
+ ) -> None:
+ """Register that a live PipeCondition has chosen its outcome, before the chosen pipe starts.
+
+ A graph builder derives the SELECTED_OUTCOME edge from this resolution and the start of the pipe it then runs
+ under the condition's node (see `pipelex.graph.selected_outcome`).
+
+ Args:
+ condition_node_id: The PipeCondition's node ID.
+ rendered_value: The value the condition's expression rendered.
+ selected_outcome: The outcome-map key the value matched, or None when the default outcome was taken.
+ """
+ ...
+
def register_execution_data(
self,
*,
@@ -310,13 +355,14 @@ class GraphTracerNoOp(GraphTracerProtocol):
pipeline_run_id: str | None = None,
emit_graph_events: bool = True,
emit_usage_events: bool = True,
- mode: GraphSpecMode = GraphSpecMode.LIVE,
+ run_mode: PipeRunMode,
) -> TraceContext:
return TraceContext(
graph_id=graph_id,
data_inclusion=data_inclusion,
emit_graph_events=emit_graph_events,
emit_usage_events=emit_usage_events,
+ run_mode=run_mode,
)
@override
@@ -337,6 +383,8 @@ class GraphTracerNoOp(GraphTracerProtocol):
concept_data: list[dict[str, Any]] | None = None,
description: str | None = None,
domain_code: str | None = None,
+ step_count: int | None = None,
+ branch_count: int | None = None,
) -> tuple[str, TraceContext]:
node_id = trace_context.make_node_id()
child_context = trace_context.copy_for_child(node_id, next_sequence=trace_context.node_sequence + 1)
@@ -450,3 +498,25 @@ class GraphTracerNoOp(GraphTracerProtocol):
merge: ConditionOutputMerge,
) -> None:
pass
+
+ @override
+ def register_batch_fan_out(
+ self,
+ *,
+ batch_node_id: str,
+ item_count: int,
+ branch_pipe_code: str,
+ branch_pipe_ref: str,
+ branch_pipe_type: str,
+ ) -> None:
+ pass
+
+ @override
+ def register_condition_resolution(
+ self,
+ *,
+ condition_node_id: str,
+ rendered_value: str,
+ selected_outcome: str | None,
+ ) -> None:
+ pass
diff --git a/pipelex/graph/selected_outcome.py b/pipelex/graph/selected_outcome.py
new file mode 100644
index 000000000..c9e32d293
--- /dev/null
+++ b/pipelex/graph/selected_outcome.py
@@ -0,0 +1,57 @@
+"""The SELECTED_OUTCOME edge of a live condition, which both graph builders derive by this one rule.
+
+A live `PipeCondition` records its resolution, and then starts the pipe of the outcome it chose under its own node. The
+edge from the condition to that pipe is derived from those two facts once the graph is built, never drawn by the tracer
+the chosen pipe starts under: in a distributed run that pipe can start in another workflow, under a tracer of its own
+that never saw the resolution, and the event-replay assembler reads both facts from the run's events whichever tracer
+wrote them. The in-process tracer and the assembler derive the edge through `derive_selected_outcome_edges`, so the two
+graphs of one run carry the same edges.
+"""
+
+from collections.abc import Mapping
+from typing import NamedTuple
+
+
+class SelectedOutcomeEdge(NamedTuple):
+ """An edge from a live condition to a pipe it ran, the pipe of the outcome it chose."""
+
+ condition_node_id: str
+ outcome_node_id: str
+ selected_outcome: str | None
+ """The outcome-map key the condition matched, which labels the edge, or None when it took its default outcome."""
+
+
+def derive_selected_outcome_edges(
+ *,
+ resolutions: Mapping[str, str | None],
+ parent_by_node: Mapping[str, str | None],
+) -> list[SelectedOutcomeEdge]:
+ """The SELECTED_OUTCOME edges of a graph, one to each pipe started under a resolved condition.
+
+ A live condition starts no pipe but the one of the outcome it chose, and an outcome mapped to `continue` or `fail`
+ starts none, so every pipe whose parent is a resolved condition is the pipe of its chosen outcome. A dry run resolves
+ no condition, so its graph has no such edge. A resolution whose condition node is not in the graph draws no edge,
+ since the edge would have no source: the event-replay assembler reads resolutions from events whichever writer wrote
+ them, and a pipe can name as its parent a node whose start is not among the events it reads.
+
+ Args:
+ resolutions: The selected outcome of each resolved condition, by the condition's node ID: the outcome-map key
+ the value it rendered matched, or None when it took its default outcome.
+ parent_by_node: The parent node ID of every node of the graph, None for a root, in the order the builder holds
+ its nodes.
+
+ Returns:
+ The edges, in the order of their target nodes in `parent_by_node`.
+ """
+ edges: list[SelectedOutcomeEdge] = []
+ for node_id, parent_node_id in parent_by_node.items():
+ if parent_node_id is None or parent_node_id not in resolutions or parent_node_id not in parent_by_node:
+ continue
+ edges.append(
+ SelectedOutcomeEdge(
+ condition_node_id=parent_node_id,
+ outcome_node_id=node_id,
+ selected_outcome=resolutions[parent_node_id],
+ )
+ )
+ return edges
diff --git a/pipelex/kernel/exceptions.py b/pipelex/kernel/exceptions.py
index e6c9fc0dd..ec784b58d 100644
--- a/pipelex/kernel/exceptions.py
+++ b/pipelex/kernel/exceptions.py
@@ -16,3 +16,15 @@ class JudgmentOutputFieldsError(ValueError):
it is raised before the judging model is called. A `ValueError`, as `PromptContentError` is: the
caller handed the kernel a value it cannot work with.
"""
+
+
+class TraceContextRunModeError(ValueError):
+ """A kernel run was handed a trace context stating a run mode other than the one the run takes.
+
+ Every trace event of a run states the mode its steps run in, and the usage events of a kernel run state the mode
+ of the trace context its caller supplied. The caller opened that trace, and may share it with writers the kernel
+ never sees, so the kernel cannot correct it: it refuses the context before any step runs, rather than let the run
+ write events that say live while its steps run dry, as a keyless boot forces them to. The mode a run takes is the
+ one `resolve_effective_run_mode` returns, which is what the caller states on the context. A `ValueError`, as the
+ other refusals of `PipelexKernel.make` are: the caller handed the kernel a value it cannot work with.
+ """
diff --git a/pipelex/kernel/pipelex_kernel.py b/pipelex/kernel/pipelex_kernel.py
index 5dec4e68d..a923ec3c3 100644
--- a/pipelex/kernel/pipelex_kernel.py
+++ b/pipelex/kernel/pipelex_kernel.py
@@ -42,7 +42,7 @@ from contextlib import AbstractContextManager
from typing import Any, Self
from uuid import uuid4
-from pipelex.cogt.content_generation.cogt_run_params import CogtRunParams, check_mock_usage_requires_dry
+from pipelex.cogt.content_generation.cogt_run_params import CogtRunParams
from pipelex.cogt.llm.llm_setting import LLMModelChoice
from pipelex.core.concepts.concept import Concept
from pipelex.core.concepts.concept_factory import ConceptFactory
@@ -50,6 +50,7 @@ from pipelex.core.concepts.native.concept_native import NativeConceptCode
from pipelex.core.memory.working_memory import WorkingMemory
from pipelex.core.stuffs.stuff_content import StuffContent
from pipelex.core.stuffs.text_content import TextContent
+from pipelex.kernel.exceptions import TraceContextRunModeError
from pipelex.kernel.llm_ops import (
resolve_llm_setting_for_object,
resolve_llm_setting_for_text,
@@ -59,7 +60,7 @@ from pipelex.kernel.llm_ops import (
from pipelex.kernel.llm_prompt_content import LlmPromptContent
from pipelex.kernel.llm_results import LlmObjectResult, LlmTextResult
from pipelex.kernel.templating_style_ops import resolve_templating_style
-from pipelex.runtime_hub import resolve_run_mode_for_boot
+from pipelex.runtime_hub import resolve_effective_run_mode
from pipelex.system.job_metadata import JobMetadata, RunMetadata
from pipelex.system.pipe_run_mode import PipeRunMode
from pipelex.system.trace_context import TraceContext
@@ -106,12 +107,19 @@ class PipelexKernel:
This is a run-params factory, so it owes the same keyless-boot contract as the pipe tier's:
a process booted with ``needs_inference=False`` forces every run it initiates to DRY, and a
kernel that skipped that would spend real money on the exact boot this package documents as
- its target. Applied through the shared ``resolve_run_mode_for_boot`` — a second copy of the
+ its target. Applied through the shared ``resolve_effective_run_mode`` — a second copy of the
rule at a second factory is how the two would drift. And, like the pipe tier's factory, the
REQUESTED mode is validated before that coercion: ``is_mock_usage`` is a DRY-only sub-flag,
so an illegal LIVE request must fail loud on every boot rather than be silently legalised by
a keyless process forcing it to DRY.
+ A ``trace_context`` must state the mode the run takes, the coerced one on a keyless boot: every
+ usage event a step writes states the context's ``run_mode``, so a context saying live for a run
+ whose steps run dry would stamp the whole run live. The caller opened that trace and may share it
+ with writers the kernel never sees, so the kernel refuses a context stating another mode with
+ ``TraceContextRunModeError``, before any step runs, rather than correct its own copy and leave
+ the others disagreeing.
+
``request_id`` is the inbound request this run serves, the identifier a hosted deployment
filters its logs on. It rides the run metadata beside ``user_id``, reaches every step's
metadata and the log context each step binds, and is validated by ``RunMetadata`` exactly as
@@ -157,7 +165,15 @@ class PipelexKernel:
replay-safe source here; the kernel stays ignorant of it, which is the point of this tier —
nothing about a durable backend reaches the runtime or the code it runs.
"""
- check_mock_usage_requires_dry(run_mode=run_mode, is_mock_usage=is_mock_usage)
+ effective_run_mode = resolve_effective_run_mode(requested=run_mode, is_mock_usage=is_mock_usage)
+ if trace_context is not None and trace_context.run_mode != effective_run_mode:
+ msg = (
+ f"PipelexKernel.make was given a trace context stating a {trace_context.run_mode} run for a run that "
+ f"runs {effective_run_mode}: every event of the run would state a mode its steps do not run in. Open "
+ f"the trace with run_mode={effective_run_mode}, the mode resolve_effective_run_mode gives this request "
+ "on this boot; a boot without inference runs every run dry."
+ )
+ raise TraceContextRunModeError(msg)
return cls(
job_metadata=JobMetadata(
run_metadata=RunMetadata(
@@ -170,7 +186,7 @@ class PipelexKernel:
),
trace_context=trace_context,
),
- cogt_run_params=CogtRunParams(run_mode=resolve_run_mode_for_boot(requested=run_mode), is_mock_usage=is_mock_usage),
+ cogt_run_params=CogtRunParams(run_mode=effective_run_mode, is_mock_usage=is_mock_usage),
step_id_source=step_id_source,
)
diff --git a/pipelex/pipe_controllers/batch/pipe_batch.py b/pipelex/pipe_controllers/batch/pipe_batch.py
index 28429d0d0..6408b100f 100644
--- a/pipelex/pipe_controllers/batch/pipe_batch.py
+++ b/pipelex/pipe_controllers/batch/pipe_batch.py
@@ -27,6 +27,7 @@ if TYPE_CHECKING:
from pipelex.core.stuffs.stuff_content import StuffContent
from pipelex.libraries.library_crate import LibraryCrate
+ from pipelex.pipe_machinery.pipe_abstract import PipeAbstract
# When a single PipeBatch fans out over more than this many items, log a one-time advisory pointing at
# durable execution — bounded fan-out is a basic backpressure effort, not durable, rate-limited execution.
@@ -148,6 +149,8 @@ class PipeBatch(PipeController):
batch_output_stuff_code = StuffFactory.make_stuff_code()
item_count = len(input_content.items)
+ # The batch records its width the moment it knows it, before it starts any item.
+ self._register_fan_out(job_metadata=job_metadata, item_count=item_count, branch_pipe=sub_pipe)
# Read off the payload, never off live config: the bound was frozen into the run params at
# submit time precisely so the fan-out shape stays a pure function of the run. See
# `PipeRunParams.batch_max_concurrency`.
@@ -275,16 +278,28 @@ class PipeBatch(PipeController):
name=output_name,
)
- # Capture execution data for the graph tracer
+ return PipeOutput(
+ working_memory=working_memory,
+ pipeline_run_id=job_metadata.run_metadata.pipeline_run_id,
+ )
+
+ def _register_fan_out(self, *, job_metadata: JobMetadata, item_count: int, branch_pipe: "PipeAbstract") -> None:
+ """Record the batch's width and the pipe each item runs, in its execution data and as its fan-out trace event."""
execution_data_dict: dict[str, Any] = {
- "item_count": len(input_content.items),
+ "item_count": item_count,
"branch_pipe_code": self.branch_pipe_code,
}
self._register_execution_data(job_metadata=job_metadata, execution_data=execution_data_dict)
-
- return PipeOutput(
- working_memory=working_memory,
- pipeline_run_id=job_metadata.run_metadata.pipeline_run_id,
+ trace_node = self._trace_node(job_metadata=job_metadata)
+ if trace_node is None:
+ return
+ trace_node.tracer_manager.register_batch_fan_out(
+ lookup_key=trace_node.lookup_key,
+ batch_node_id=trace_node.node_id,
+ item_count=item_count,
+ branch_pipe_code=branch_pipe.code,
+ branch_pipe_ref=branch_pipe.pipe_ref,
+ branch_pipe_type=branch_pipe.type,
)
@override
diff --git a/pipelex/pipe_controllers/condition/pipe_condition.py b/pipelex/pipe_controllers/condition/pipe_condition.py
index d35d3016f..c067a898a 100644
--- a/pipelex/pipe_controllers/condition/pipe_condition.py
+++ b/pipelex/pipe_controllers/condition/pipe_condition.py
@@ -473,18 +473,15 @@ class PipeCondition(PipeController):
# Select the outcome based on the evaluated expression
outcome = self.outcome_map.get(evaluated_expression, self.default_outcome)
- # Capture execution data for the graph tracer
- execution_data_dict: dict[str, Any] = {
- "evaluated_expression": evaluated_expression,
- "selected_outcome": str(outcome),
- }
+ # The condition records its choice the moment the choice is final, before anything runs under it: at once for
+ # `continue` and `fail`, which run nothing, and for a pipe once its inputs are known to be there to run it.
# Handle continue case (phase 1): `continue` resolves the declared output as
# ABSENT — a declared-absent record with provenance, memory otherwise unchanged. A previous
# main stuff stays under its own name (the migration idiom: consume it explicitly
# downstream); it no longer passes through as this pipe's output.
if SpecialOutcome.is_continue(outcome):
- self._register_execution_data(job_metadata=job_metadata, execution_data=execution_data_dict)
+ self._record_resolution(job_metadata=job_metadata, evaluated_expression=evaluated_expression, outcome=outcome)
# The reason names only the pipe and the outcome: a pipe that force-unwraps this output quotes it to the caller
# under STRICT disclosure, and the value the expression rendered can be literal text of a condition a host
# library declared. The execution data above keeps that value.
@@ -496,7 +493,7 @@ class PipeCondition(PipeController):
return PipeOutput(working_memory=working_memory, pipeline_run_id=job_metadata.run_metadata.pipeline_run_id)
if SpecialOutcome.is_fail(outcome):
- self._register_execution_data(job_metadata=job_metadata, execution_data=execution_data_dict)
+ self._record_resolution(job_metadata=job_metadata, evaluated_expression=evaluated_expression, outcome=outcome)
# The caller's method maps this outcome to 'fail' on purpose, a refusal of the run it was given. The
# message names only the pipe: the value the expression rendered can be literal text of the expression,
# or an outcome key, of a condition a host library declared. The execution data above keeps that value.
@@ -540,9 +537,11 @@ class PipeCondition(PipeController):
)
)
- pipe_output = await get_pipe_router().run(
+ # Recorded only now: a choice whose pipe cannot run is a failure of the condition, never an outcome it took.
+ self._record_resolution(job_metadata=job_metadata, evaluated_expression=evaluated_expression, outcome=outcome)
+ return await get_pipe_router().run(
pipe_job=PipeJobFactory.make_pipe_job(
- pipe=get_required_pipe(pipe_code=outcome),
+ pipe=chosen_pipe,
job_metadata=job_metadata,
working_memory=working_memory,
pipe_run_params=pipe_run_params,
@@ -550,8 +549,33 @@ class PipeCondition(PipeController):
library_crate=library_crate,
),
)
+
+ def _record_resolution(self, *, job_metadata: JobMetadata, evaluated_expression: str, outcome: str) -> None:
+ """Record the outcome a live run chose, in the execution data and as the condition's resolution trace event.
+
+ The execution data keeps the value the expression rendered and the outcome it selected, a pipe code or a special
+ outcome. The trace event names the outcome by the key of the outcome map the value matched, or by None when it
+ matched none and the default outcome was taken, since `"default"` can itself be a key.
+
+ Args:
+ job_metadata: The condition's job metadata, whose trace context names its graph node.
+ evaluated_expression: The value the condition's expression rendered.
+ outcome: The outcome selected: a pipe code, `continue` or `fail`.
+ """
+ execution_data_dict: dict[str, Any] = {
+ "evaluated_expression": evaluated_expression,
+ "selected_outcome": outcome,
+ }
self._register_execution_data(job_metadata=job_metadata, execution_data=execution_data_dict)
- return pipe_output
+ trace_node = self._trace_node(job_metadata=job_metadata)
+ if trace_node is None:
+ return
+ trace_node.tracer_manager.register_condition_resolution(
+ lookup_key=trace_node.lookup_key,
+ condition_node_id=trace_node.node_id,
+ rendered_value=evaluated_expression,
+ selected_outcome=evaluated_expression if evaluated_expression in self.outcome_map else None,
+ )
@override
async def _dry_run_controller_pipe(
diff --git a/pipelex/pipe_controllers/parallel/pipe_parallel.py b/pipelex/pipe_controllers/parallel/pipe_parallel.py
index d44e217d5..c0ea18729 100644
--- a/pipelex/pipe_controllers/parallel/pipe_parallel.py
+++ b/pipelex/pipe_controllers/parallel/pipe_parallel.py
@@ -438,6 +438,10 @@ class PipeParallel(PipeController):
variable_names=[sub_pipe.output_name],
)
+ @override
+ def traced_branch_count(self) -> int:
+ return len(self.parallel_sub_pipes)
+
@override
def pipe_dependencies(self) -> set[str]:
return {sub_pipe.pipe_code for sub_pipe in self.parallel_sub_pipes}
@@ -463,6 +467,14 @@ class PipeParallel(PipeController):
final_stuff_code = pipe_run_params.final_stuff_code
pipe_run_params.final_stuff_code = None
+ # The parallel records how many branches it runs, and how it combines them, as it starts, before any branch runs.
+ execution_data_dict: dict[str, Any] = {
+ "branch_count": self.traced_branch_count(),
+ "add_each_output": self.add_each_output,
+ "combined_output_concept": self.output.concept.concept_ref,
+ }
+ self._register_execution_data(job_metadata=job_metadata, execution_data=execution_data_dict)
+
tasks: list[Coroutine[Any, Any, PipeOutput]] = []
for sub_pipe in self.parallel_sub_pipes:
@@ -596,14 +608,6 @@ class PipeParallel(PipeController):
output_stuffs=output_stuffs,
)
- # Capture execution data for the graph tracer
- execution_data_dict: dict[str, Any] = {
- "branch_count": len(self.parallel_sub_pipes),
- "add_each_output": self.add_each_output,
- "combined_output_concept": self.output.concept.concept_ref,
- }
- self._register_execution_data(job_metadata=job_metadata, execution_data=execution_data_dict)
-
return PipeOutput(
working_memory=working_memory,
pipeline_run_id=job_metadata.run_metadata.pipeline_run_id,
diff --git a/pipelex/pipe_controllers/sequence/pipe_sequence.py b/pipelex/pipe_controllers/sequence/pipe_sequence.py
index cc78c3b03..926f67c59 100644
--- a/pipelex/pipe_controllers/sequence/pipe_sequence.py
+++ b/pipelex/pipe_controllers/sequence/pipe_sequence.py
@@ -1,5 +1,5 @@
import sys
-from typing import TYPE_CHECKING, Any, Literal
+from typing import TYPE_CHECKING, Literal
from pydantic import PrivateAttr, field_validator
from typing_extensions import override
@@ -37,6 +37,7 @@ from pipelex.pipe_controllers.sequence.sequence_typed_flow import (
SequenceFlowMemo,
SequenceStep,
SequenceTypedFlow,
+ authored_step_count,
authored_step_number,
build_sequence_typed_flow,
derive_binding_spec,
@@ -89,6 +90,11 @@ class PipeSequence(PipeController):
"""The number messages give a step: its place among the steps the author wrote, counting from 1."""
return authored_step_number(steps=self.sequential_sub_pipes, step_index=step_index)
+ @override
+ def traced_step_count(self) -> int:
+ """The number of steps the author wrote: the binding written for a dotted `batch_over` is no step of theirs."""
+ return authored_step_count(steps=self.sequential_sub_pipes)
+
@property
def has_binding_step(self) -> bool:
"""Whether a step binds a value, which only a run of a sequence with one reads from the typed flow.
@@ -1032,9 +1038,17 @@ class PipeSequence(PipeController):
typed_flow: SequenceTypedFlow | None = None
# By step index, each binding the run derived from the value its root held, the flow having no spec for that root.
runtime_bindings: dict[int, DerivedBinding] = {}
+ # The sequence records how many steps it runs as it starts, and the number of each step as that step starts.
+ self._register_execution_data(job_metadata=job_metadata, execution_data={"step_count": self.traced_step_count()})
+ current_step: int | None = None
for sub_pipe_index, sub_pipe in enumerate(self.sequential_sub_pipes):
is_last_step = sub_pipe_index == len(self.sequential_sub_pipes) - 1
+ step_number = self._step_number(step_index=sub_pipe_index)
+ # The binding written for a dotted `batch_over` shares the number of the batch step after it: one record each.
+ if step_number != current_step:
+ current_step = step_number
+ self._register_execution_data(job_metadata=job_metadata, execution_data={"current_step": step_number})
if isinstance(sub_pipe, BindingStep):
if typed_flow is None:
typed_flow = self.build_typed_flow()
@@ -1066,11 +1080,6 @@ class PipeSequence(PipeController):
library_crate=library_crate,
)
evolving_memory = pipe_output.working_memory
- # Capture execution data for the graph tracer
- execution_data_dict: dict[str, Any] = {
- "step_count": len(self.sequential_sub_pipes),
- }
- self._register_execution_data(job_metadata=job_metadata, execution_data=execution_data_dict)
return PipeOutput(
working_memory=evolving_memory,
diff --git a/pipelex/pipe_controllers/sequence/sequence_typed_flow.py b/pipelex/pipe_controllers/sequence/sequence_typed_flow.py
index 0af585a59..11efb1252 100644
--- a/pipelex/pipe_controllers/sequence/sequence_typed_flow.py
+++ b/pipelex/pipe_controllers/sequence/sequence_typed_flow.py
@@ -177,16 +177,26 @@ class DerivedBinding(NamedTuple):
stuff_spec: StuffSpec
+def is_authored_step(*, step: SequenceStep) -> bool:
+ """Whether the author wrote this step: the binding a sequence writes for a dotted `batch_over` is no step of theirs."""
+ return not (isinstance(step, BindingStep) and step.is_dotted_batch_over)
+
+
def authored_step_number(*, steps: list[SequenceStep], step_index: int) -> int:
"""The number of a step, counting from 1, as its author wrote the sequence.
The binding a sequence writes for a dotted `batch_over` is no step of the author's: it shares the number of the pipe
step it was written for, which follows it.
"""
- rewritten_before = sum(1 for step in steps[:step_index] if isinstance(step, BindingStep) and step.is_dotted_batch_over)
+ rewritten_before = sum(1 for step in steps[:step_index] if not is_authored_step(step=step))
return step_index + 1 - rewritten_before
+def authored_step_count(*, steps: list[SequenceStep]) -> int:
+ """The number of steps the author wrote, the number `authored_step_number` gives the last of them."""
+ return sum(1 for step in steps if is_authored_step(step=step))
+
+
def resolve_library_concept(*, concept_library: ConceptLibraryAbstract, concept_ref: str) -> Concept | None:
"""The library's concept for a ref, looked up directly, then among a dependency's aliased keys when one alone matches."""
try:
diff --git a/pipelex/pipe_machinery/pipe_abstract.py b/pipelex/pipe_machinery/pipe_abstract.py
index fb2d86372..9a64351ca 100644
--- a/pipelex/pipe_machinery/pipe_abstract.py
+++ b/pipelex/pipe_machinery/pipe_abstract.py
@@ -115,6 +115,14 @@ class CompanionSlot(NamedTuple):
producing_pipe_code: str
+class PipeTraceNode(NamedTuple):
+ """A running pipe's own node in the run's trace, with the tracer manager that records what happens on it."""
+
+ tracer_manager: GraphTracerManager
+ lookup_key: str
+ node_id: str
+
+
class InputPresenceScan(NamedTuple):
"""The runtime trichotomy (D3) applied to a pipe's needed inputs against working memory.
@@ -170,6 +178,20 @@ class PipeAbstract(ABC, BaseModel):
# overrides this to True. (`pipe_category` no longer encodes signature-ness.)
return False
+ def traced_step_count(self) -> int | None:
+ """The number of steps a sequence's author wrote, which the trace records on the pipe's start.
+
+ None for every pipe that runs no steps; `PipeSequence` overrides it.
+ """
+ return None
+
+ def traced_branch_count(self) -> int | None:
+ """The number of branches a parallel runs, which the trace records on the pipe's start.
+
+ None for every pipe that runs no branches; `PipeParallel` overrides it.
+ """
+ return None
+
def pipe_dependencies(self) -> set[str]:
"""Return the set of pipe codes that this pipe depends on.
@@ -196,21 +218,32 @@ class PipeAbstract(ABC, BaseModel):
return unique_concepts
+ def _trace_node(self, *, job_metadata: JobMetadata) -> PipeTraceNode | None:
+ """This running pipe's own node in the run's trace, or None when nothing traces the run.
+
+ The pipe's start hands it the trace context its children run under, whose `parent_node_id` is the pipe's own
+ node: whatever the pipe records about itself while it runs is recorded on that node.
+ """
+ trace_context = job_metadata.trace_context
+ if trace_context is None or trace_context.parent_node_id is None:
+ return None
+ tracer_manager = GraphTracerManager.get_instance()
+ if tracer_manager is None:
+ return None
+ return PipeTraceNode(tracer_manager=tracer_manager, lookup_key=trace_context.lookup_key, node_id=trace_context.parent_node_id)
+
def _register_execution_data(self, *, job_metadata: JobMetadata, execution_data: dict[str, Any]) -> None:
"""Register execution metadata with the graph tracer.
Called by pipe subclasses during execution to capture runtime-resolved data
(rendered prompts, resolved models, etc.) for the GraphSpec.
"""
- trace_context = job_metadata.trace_context
- if trace_context is None:
- return
- tracer_manager = GraphTracerManager.get_instance()
- if tracer_manager is None or trace_context.parent_node_id is None:
+ trace_node = self._trace_node(job_metadata=job_metadata)
+ if trace_node is None:
return
- tracer_manager.register_execution_data(
- lookup_key=trace_context.lookup_key,
- node_id=trace_context.parent_node_id,
+ trace_node.tracer_manager.register_execution_data(
+ lookup_key=trace_node.lookup_key,
+ node_id=trace_node.node_id,
execution_data=execution_data,
)
@@ -787,6 +820,8 @@ class PipeAbstract(ABC, BaseModel):
concept_data=concept_data,
description=self.description,
domain_code=self.domain_code,
+ step_count=self.traced_step_count(),
+ branch_count=self.traced_branch_count(),
)
# Update job metadata with child trace context for nested pipes
if child_trace_context is not None:
diff --git a/pipelex/pipe_run/dry_run_in_process.py b/pipelex/pipe_run/dry_run_in_process.py
index 70adbbb54..2d5b28033 100644
--- a/pipelex/pipe_run/dry_run_in_process.py
+++ b/pipelex/pipe_run/dry_run_in_process.py
@@ -16,7 +16,7 @@ from pipelex.base_exceptions import PipelexError
from pipelex.cogt.content_generation.content_generator import ContentGenerator
from pipelex.config import get_config
from pipelex.graph.graph_tracer_manager import GraphTracerManager
-from pipelex.graph.graphspec import GraphSpec, GraphSpecMode
+from pipelex.graph.graphspec import GraphSpec
from pipelex.interpreter_hub import get_library_manager, scoped_pipe_router
from pipelex.observer.observer_protocol import ObserverNoOp
from pipelex.pipe_machinery.pipe_abstract import PipeAbstract
@@ -160,7 +160,7 @@ async def dry_run_pipe_in_process(pipe: PipeAbstract, *, library_id: str, caller
emit_usage_events=False,
# Validate builds the I/O artifacts once, for its report; this dry run keeps only the graph.
describe_pipe_io=False,
- mode=GraphSpecMode.DRY,
+ run_mode=PipeRunMode.DRY,
)
try:
with (
diff --git a/pipelex/pipe_run/pipe_run.py b/pipelex/pipe_run/pipe_run.py
index 67977ac0a..56007752a 100644
--- a/pipelex/pipe_run/pipe_run.py
+++ b/pipelex/pipe_run/pipe_run.py
@@ -136,7 +136,6 @@ class PipeRun(PipeRunProtocol):
assemble_usage=trace_context.emit_usage_events,
domain_code=pipe_job.pipe.domain_code,
main_pipe_code=pipe_job.pipe.code,
- run_mode=pipe_job.pipe_run_params.run_mode,
trace_events=trace_events,
)
if trace_context.describe_pipe_io and pipe_output.graph_spec is not None:
diff --git a/pipelex/pipe_run/pipe_run_params_factory.py b/pipelex/pipe_run/pipe_run_params_factory.py
index 5d13a282a..1c978ad8c 100644
--- a/pipelex/pipe_run/pipe_run_params_factory.py
+++ b/pipelex/pipe_run/pipe_run_params_factory.py
@@ -1,10 +1,9 @@
from typing import Any, Literal
-from pipelex.cogt.content_generation.cogt_run_params import check_mock_usage_requires_dry
from pipelex.config import get_config
from pipelex.core.pipes.variable_multiplicity import VariableMultiplicity
from pipelex.pipe_run.pipe_run_params import BatchParams, PipeRunParams
-from pipelex.runtime_hub import resolve_run_mode_for_boot
+from pipelex.runtime_hub import resolve_effective_run_mode
from pipelex.system.pipe_run_mode import PipeRunMode
@@ -45,13 +44,13 @@ class PipeRunParamsFactory:
The REQUESTED mode is validated before the forced-DRY coercion, so a contract violation
(``is_mock_usage`` on a LIVE request) fails loud on every boot — the keyless coercion must
- not silently turn an illegal request into a legal one.
+ not silently turn an illegal request into a legal one. Both steps are ``resolve_effective_run_mode``,
+ which ``pipeline_run_setup`` also calls before it opens the run's tracer.
``batch_max_concurrency`` is resolved here for the same single-writer reason and because the
read must happen ONCE, at submit time: see the field's docstring on ``PipeRunParams``.
"""
- check_mock_usage_requires_dry(run_mode=pipe_run_mode, is_mock_usage=is_mock_usage)
- pipe_run_mode = resolve_run_mode_for_boot(requested=pipe_run_mode)
+ pipe_run_mode = resolve_effective_run_mode(requested=pipe_run_mode, is_mock_usage=is_mock_usage)
config = get_config().interpreter
pipe_stack_limit = pipe_stack_limit or config.pipe_run.pipe_stack_limit
return PipeRunParams(
diff --git a/pipelex/pipe_run/tracing_assembly.py b/pipelex/pipe_run/tracing_assembly.py
index db332d300..d29855830 100644
--- a/pipelex/pipe_run/tracing_assembly.py
+++ b/pipelex/pipe_run/tracing_assembly.py
@@ -24,10 +24,9 @@ from pipelex.reporting.reporting_types import AnyTokensUsage
from pipelex.runtime_hub import get_event_log_override
from pipelex.system.exceptions import MissingDependencyError
from pipelex.system.job_metadata import RunMetadata
-from pipelex.system.pipe_run_mode import PipeRunMode
from pipelex.tools.log.error_fields import error_fields
from pipelex.tracing.event_log_factory import make_event_log
-from pipelex.tracing.exceptions import EventLogError
+from pipelex.tracing.exceptions import EventLogError, TraceEventModeError
from pipelex.tracing.graphspec_assembler import GraphSpecAssembler
from pipelex.tracing.trace_events import TraceEvent
from pipelex.tracing.usage_aggregator import UsageAggregator
@@ -137,7 +136,6 @@ def assemble_tracing(
assemble_usage: bool,
domain_code: str | None = None,
main_pipe_code: str | None = None,
- run_mode: PipeRunMode = PipeRunMode.LIVE,
run_metadata: RunMetadata | None = None,
trace_events: RunTraceEvents | None = None,
) -> TracingAssembly:
@@ -154,8 +152,10 @@ def assemble_tracing(
errors, missing optional dependencies) are caught and recorded in the ``*_error``
fields, not propagated. Either backend refuses a log holding an event the current
models do not accept with ``EventLogSchemaMismatchError``, also an ``EventLogError``:
- the read fails as a whole, so such a run gets neither its graph nor its usage total. Programming bugs (KeyError,
- AttributeError, etc.) propagate so they surface during development.
+ the read fails as a whole, so such a run gets neither its graph nor its usage total. The graph's provenance mode is
+ the run mode its events state: events that mix modes hold more than one run, and get no graph, the assembler's
+ ``TraceEventModeError`` recorded in ``graph_assembly_error``. Programming bugs (KeyError, AttributeError, etc.)
+ propagate so they surface during development.
Args:
pipeline_run_id: The pipeline run ID to query events for.
@@ -163,7 +163,6 @@ def assemble_tracing(
assemble_usage: Whether to aggregate token usage from the events.
domain_code: Domain code for the graph's pipeline ref.
main_pipe_code: Main pipe code for the graph's pipeline ref.
- run_mode: Final pipe run mode used to stamp GraphSpec provenance.
run_metadata: The run this assembly belongs to, stamped onto the result so a
transport that offloads it keys it inside the run's namespace. None for a
run with no host-supplied scope.
@@ -186,7 +185,6 @@ def assemble_tracing(
assemble_usage=assemble_usage,
domain_code=domain_code,
main_pipe_code=main_pipe_code,
- run_mode=run_mode,
trace_events=trace_events,
)
@@ -199,7 +197,6 @@ def _assemble_tracing_events(
assemble_usage: bool,
domain_code: str | None,
main_pipe_code: str | None,
- run_mode: PipeRunMode,
trace_events: RunTraceEvents | None,
) -> TracingAssembly:
"""The body of ``assemble_tracing``, run inside the log context it binds: read the events once, assemble onto ``result``."""
@@ -230,12 +227,11 @@ def _assemble_tracing_events(
domain=domain_code,
main_pipe=main_pipe_code,
),
- mode=run_mode.graphspec_mode,
)
log.debug("Assembled the graph from the trace events", fields={"event_count": len(events)})
- except ValidationError as validation_error:
- log.warning("The graph could not be assembled from the trace events", fields=error_fields(exc=validation_error))
- result.graph_assembly_error = f"Graph assembly failed for pipeline_run_id={pipeline_run_id}: {validation_error}"
+ except (ValidationError, TraceEventModeError) as assembly_error:
+ log.warning("The graph could not be assembled from the trace events", fields=error_fields(exc=assembly_error))
+ result.graph_assembly_error = f"Graph assembly failed for pipeline_run_id={pipeline_run_id}: {assembly_error}"
return result
@@ -248,7 +244,6 @@ def assemble_tracing_on_output(
assemble_usage: bool,
domain_code: str | None = None,
main_pipe_code: str | None = None,
- run_mode: PipeRunMode = PipeRunMode.LIVE,
trace_events: RunTraceEvents | None = None,
) -> None:
"""Assemble graph and/or usage from trace events and set them on pipe_output (DIRECT mode).
@@ -264,7 +259,6 @@ def assemble_tracing_on_output(
assemble_usage: Whether to aggregate and set token usage.
domain_code: Domain code for the graph's pipeline ref.
main_pipe_code: Main pipe code for the graph's pipeline ref.
- run_mode: Final pipe run mode used to stamp GraphSpec provenance.
trace_events: The run's events when the caller already read them; None to read them here.
"""
result = assemble_tracing(
@@ -273,7 +267,6 @@ def assemble_tracing_on_output(
assemble_usage=assemble_usage,
domain_code=domain_code,
main_pipe_code=main_pipe_code,
- run_mode=run_mode,
# Inherited from the output this assembly decorates, so both key together.
run_metadata=pipe_output.job_metadata.run_metadata if pipe_output.job_metadata else None,
trace_events=trace_events,
diff --git a/pipelex/pipeline/pipeline_run_setup.py b/pipelex/pipeline/pipeline_run_setup.py
index 56ac749bc..d7ef4e23f 100644
--- a/pipelex/pipeline/pipeline_run_setup.py
+++ b/pipelex/pipeline/pipeline_run_setup.py
@@ -25,7 +25,13 @@ from pipelex.pipe_run.pipe_run_params import (
)
from pipelex.pipeline.exceptions import PipeExecutionError
from pipelex.pipeline.execution_seams import acquire_library, prepare_pipe_job
-from pipelex.runtime_hub import get_event_log_override, get_otel_tracer, get_report_delegate, get_telemetry_manager
+from pipelex.runtime_hub import (
+ get_event_log_override,
+ get_otel_tracer,
+ get_report_delegate,
+ get_telemetry_manager,
+ resolve_effective_run_mode,
+)
from pipelex.system.configuration.configs import PipelineExecutionConfig
from pipelex.system.environment import get_optional_env
from pipelex.system.job_metadata import OtelContext
@@ -122,7 +128,9 @@ async def pipeline_run_setup(
pipe_run_mode:
Pipe run mode: ``PipeRunMode.LIVE`` or ``PipeRunMode.DRY``. If not specified,
inferred from the environment variable ``PIPELEX_FORCE_DRY_RUN_MODE``. Defaults
- to ``PipeRunMode.LIVE`` if the environment variable is not set.
+ to ``PipeRunMode.LIVE`` if the environment variable is not set. A keyless boot
+ (``needs_inference=False``) runs a LIVE request DRY, and the run's trace, its
+ graph and its pipe job all state DRY.
is_mock_usage:
Internal sub-flag of ``run_mode=DRY`` — only CLI access is the hidden ``--mock-usage``
test trigger (requires ``--dry-run``, not shown in --help): when True, the dry
@@ -266,6 +274,15 @@ async def pipeline_run_setup(
else:
pipe_run_mode = PipeRunMode.LIVE
+ # The mode the run takes, resolved ONCE, here, above everything this function opens. A keyless
+ # boot forces a requested LIVE run DRY (eng review D4), and the tracer, the trace context, the
+ # OTel context and the pipe job must all state the mode the pipes run in. The coercion used to
+ # happen only in the run-params factory, below the tracer, so a forced-dry run stamped every
+ # trace event live, and the graph built from those events said live too. The requested mode is
+ # validated against `is_mock_usage` first, so the coercion cannot legalise a LIVE request
+ # carrying the dry sub-flag; the factory below sees DRY and coerces nothing again.
+ pipe_run_mode = resolve_effective_run_mode(requested=pipe_run_mode, is_mock_usage=is_mock_usage)
+
pipeline = get_pipeline_manager().add_new_pipeline(pipe_code=pipe_code, pipeline_run_id=pipeline_run_id)
pipeline_run_id = pipeline.pipeline_run_id
@@ -387,7 +404,7 @@ async def pipeline_run_setup(
emit_usage_events=is_generate_usage,
# The I/O artifacts are the graphspec's companions: a run that writes no graphspec builds none.
describe_pipe_io=is_generate_graph and execution_config.graph.graphs_inclusion.graphspec_json,
- mode=pipe_run_mode.graphspec_mode,
+ run_mode=pipe_run_mode,
)
# Register the event log on the report delegate for usage event emission — only when cost reporting
diff --git a/pipelex/reporting/reporting_manager.py b/pipelex/reporting/reporting_manager.py
index 3a3596945..8930abf6d 100644
--- a/pipelex/reporting/reporting_manager.py
+++ b/pipelex/reporting/reporting_manager.py
@@ -239,6 +239,7 @@ class ReportingManager(ReportingProtocol):
pipeline_run_id=context.pipeline_run_id,
workflow_id=context.workflow_id,
writer_id=context.event_log.writer_id,
+ mode=trace_context.run_mode,
timestamp=datetime.now(UTC),
sequence=seq,
node_id=node_id,
@@ -315,6 +316,7 @@ class ReportingManager(ReportingProtocol):
pipeline_run_id=inference_job.job_metadata.run_metadata.pipeline_run_id,
workflow_id=workflow_id,
writer_id=process_event_log.writer_id,
+ mode=trace_context.run_mode,
timestamp=datetime.now(UTC),
sequence=seq,
node_id=node_id,
diff --git a/pipelex/runtime_hub.py b/pipelex/runtime_hub.py
index 65beba670..8269c5d9d 100644
--- a/pipelex/runtime_hub.py
+++ b/pipelex/runtime_hub.py
@@ -29,6 +29,7 @@ from typing import TYPE_CHECKING, Any, ClassVar, Optional, TextIO
from kajson.class_registry_abstract import ClassRegistryAbstract
from pipelex import log
+from pipelex.cogt.content_generation.cogt_run_params import check_mock_usage_requires_dry
from pipelex.cogt.content_generation.content_generator_protocol import (
ContentGeneratorProtocol,
)
@@ -671,6 +672,31 @@ def resolve_run_mode_for_boot(*, requested: PipeRunMode) -> PipeRunMode:
return requested
+def resolve_effective_run_mode(*, requested: PipeRunMode, is_mock_usage: bool) -> PipeRunMode:
+ """The mode a run this process initiates takes on this boot: the requested mode, validated, then forced DRY when keyless.
+
+ The REQUESTED mode is validated before the keyless-boot coercion (eng review D4), so a contract violation
+ (``is_mock_usage`` on a LIVE request) fails loud on every boot: the coercion must not silently turn an illegal
+ request into a legal one. Every entry point resolves its mode through it: the pipe tier's
+ ``PipeRunParamsFactory.make_run_params``, ``pipeline_run_setup``, which needs the effective mode before it opens
+ the run's tracer so that the trace states the mode the pipes run in, and the kernel tier's ``PipelexKernel.make``.
+ It lives here, beside the flag it applies, so the kernel layer reaches it without importing the interpreter. A
+ caller that opens a trace for a kernel run states this mode on its ``TraceContext``.
+
+ Args:
+ requested: The run mode the caller asked for.
+ is_mock_usage: Whether the caller asked for the dry sub-flag that reports synthetic usage.
+
+ Returns:
+ The requested mode, or DRY when a keyless boot forces a LIVE request dry.
+
+ Raises:
+ ValueError: When ``is_mock_usage`` is set on a LIVE request.
+ """
+ check_mock_usage_requires_dry(run_mode=requested, is_mock_usage=is_mock_usage)
+ return resolve_run_mode_for_boot(requested=requested)
+
+
def get_content_generator() -> ContentGeneratorProtocol:
override = _content_generator_override.get()
if override is not None:
diff --git a/pipelex/system/trace_context.py b/pipelex/system/trace_context.py
index 6cf2b235c..f16265e47 100644
--- a/pipelex/system/trace_context.py
+++ b/pipelex/system/trace_context.py
@@ -17,6 +17,7 @@ The node tree is always built when either stream is on — including costs-only
from pydantic import BaseModel, ConfigDict, Field
from pipelex.system.data_inclusion_config import DataInclusionConfig
+from pipelex.system.pipe_run_mode import PipeRunMode
class TraceContext(BaseModel):
@@ -41,6 +42,9 @@ class TraceContext(BaseModel):
describe_pipe_io: Whether the run builds the three I/O artifacts that describe its graphspec's
data (driven by ``graphs_inclusion.graphspec_json``). Off for a run that writes no
graphspec, and for validate's own graph dry run, which already built them for its report.
+ run_mode: Whether the run is live or a dry run. Every trace event the run emits states it, the
+ usage events an activity emits included, which have this context and nothing else of the run.
+ Required, with no default: a context built without it would stamp a dry run's events as live.
"""
model_config = ConfigDict(strict=True, extra="forbid")
@@ -53,6 +57,7 @@ class TraceContext(BaseModel):
emit_graph_events: bool = Field(default=True, description="Whether to emit graph (node/edge) trace events")
emit_usage_events: bool = Field(default=True, description="Whether to emit usage (cost) trace events")
describe_pipe_io: bool = Field(default=True, description="Whether to build the I/O artifacts describing the graphspec's data")
+ run_mode: PipeRunMode = Field(strict=False, description="Whether the run is live or a dry run, stated by whoever opens the trace")
@property
def lookup_key(self) -> str:
@@ -86,4 +91,5 @@ class TraceContext(BaseModel):
emit_graph_events=self.emit_graph_events,
emit_usage_events=self.emit_usage_events,
describe_pipe_io=self.describe_pipe_io,
+ run_mode=self.run_mode,
)
diff --git a/pipelex/tracing/exceptions.py b/pipelex/tracing/exceptions.py
index e6203cf1a..7da25a7d1 100644
--- a/pipelex/tracing/exceptions.py
+++ b/pipelex/tracing/exceptions.py
@@ -41,3 +41,13 @@ class EventLogSetupError(EventLogError):
cannot even be built degrades to an ``*_assembly_error`` note rather than
aborting the pipeline run.
"""
+
+
+class TraceEventModeError(PipelexError):
+ """Raised when a set of trace events states no single run mode to build a graph in.
+
+ Every trace event states the mode of the run that wrote it, `live` or `dry`, and the graph the event-replay
+ assembler builds from a run's events takes that mode as its provenance. A set that mixes modes holds the events of
+ more than one run, such as a dry run and a live run written under one pipeline run id, and an empty set states no
+ mode at all: neither describes one run, so the assembler builds no graph from it rather than guess its mode.
+ """
diff --git a/pipelex/tracing/graphspec_assembler.py b/pipelex/tracing/graphspec_assembler.py
index 2f0c1cd43..12e14add9 100644
--- a/pipelex/tracing/graphspec_assembler.py
+++ b/pipelex/tracing/graphspec_assembler.py
@@ -3,7 +3,8 @@
This is the core algorithm for distributed tracing — equivalent to
GraphTracer.teardown() but working from serialized events instead of
in-memory state. Used after Temporal execution to assemble cross-worker
-graphs from NDJSON event files.
+graphs from NDJSON event files. The graph's provenance mode is the run mode
+its events state, never one the caller passes.
"""
from collections.abc import Sequence
@@ -30,11 +31,15 @@ from pipelex.graph.graphspec import (
make_graphspec_meta,
output_digest_is_optional,
)
+from pipelex.graph.selected_outcome import derive_selected_outcome_edges
+from pipelex.tracing.exceptions import TraceEventModeError
from pipelex.tracing.trace_events import (
UNATTRIBUTED_NODE_ID,
BatchAggregateEvent,
+ BatchFannedOutEvent,
BatchItemEvent,
ConditionOutputMergeEvent,
+ ConditionResolvedEvent,
ControllerOutputEvent,
EdgeEvent,
ExecutionDataEvent,
@@ -124,7 +129,9 @@ class GraphSpecAssembler:
- Pass 1: Build nodes, producer map, per-node usage totals, and collect metadata
from all events.
- Pass 2: Generate DATA, BATCH_ITEM, BATCH_AGGREGATE, PARALLEL_COMBINE edges
- using the complete producer map, then attribute usage — resolve each usage
+ using the complete producer map, and SELECTED_OUTCOME edges from each live
+ condition's resolution to the pipe it then started, whichever workflow wrote
+ that start, then attribute usage — resolve each usage
total against the now-complete node set, roll subtrees up the parentage chain,
and compute the run total.
@@ -140,25 +147,46 @@ class GraphSpecAssembler:
*,
graph_id: str,
pipeline_ref: PipelineRef | None = None,
- mode: GraphSpecMode = GraphSpecMode.LIVE,
) -> GraphSpec:
"""Assemble a GraphSpec from a flat list of trace events.
+ The GraphSpec's provenance mode is the run mode every event states.
+
Args:
events: Flat list of trace events, pre-sorted by (workflow_id, sequence).
graph_id: The graph identifier for the assembled GraphSpec.
pipeline_ref: Optional pipeline reference metadata.
- mode: Provenance mode to stamp onto the assembled GraphSpec.
Returns:
A complete GraphSpec with nodes and edges.
+
+ Raises:
+ TraceEventModeError: When the events mix modes, or there are none, and so state no single run mode.
"""
+ mode = _graphspec_mode_of_events(events=events, graph_id=graph_id)
assembler = _AssemblerState(graph_id=graph_id, pipeline_ref=pipeline_ref or PipelineRef(), mode=mode)
assembler.pass_one(events)
assembler.pass_two()
return assembler.build_graph_spec()
+def _graphspec_mode_of_events(*, events: Sequence[TraceEvent], graph_id: str) -> GraphSpecMode:
+ """The provenance mode of the graph built from `events`: the one run mode they all state.
+
+ Raises:
+ TraceEventModeError: When the events mix modes, or there are none.
+ """
+ run_modes = {event.mode for event in events}
+ if not run_modes:
+ msg = f"No trace events to assemble graph '{graph_id}' from, so no run mode to state as its provenance."
+ raise TraceEventModeError(msg)
+ if len(run_modes) > 1:
+ modes_str = ", ".join(sorted(run_modes))
+ msg = f"The trace events of graph '{graph_id}' mix run modes ({modes_str}), so they hold more than one run; no graph is built from them."
+ raise TraceEventModeError(msg)
+ return run_modes.pop().graphspec_mode
+
+
class _AssemblerState:
"""Mutable state for a single assembler invocation."""
@@ -176,6 +204,9 @@ class _AssemblerState:
self._batch_aggregate_map: dict[str, tuple[str | None, list[tuple[str, int]]]] = {}
self._parallel_combine_map: dict[str, tuple[str, list[tuple[str, str]]]] = {}
self._condition_output_merges: list[ConditionOutputMerge] = []
+ # The live conditions that have chosen their outcome, by condition node id, each with the outcome-map key it
+ # matched (None for the default outcome): pass 2 derives the SELECTED_OUTCOME edges from them
+ self._condition_resolutions: dict[str, str | None] = {}
# Registries (accumulated from events, deduplicated)
self._pipe_registry: dict[str, dict[str, Any]] = {}
@@ -221,6 +252,12 @@ class _AssemblerState:
self._handle_execution_data(event)
elif isinstance(event, UsageReportEvent):
self._handle_usage_report(event)
+ elif isinstance(event, ConditionResolvedEvent):
+ self._handle_condition_resolved(event)
+ elif isinstance(event, BatchFannedOutEvent):
+ # A batch's width is for the readers following a run while it goes on: the graph already holds its
+ # items, as the batch's children.
+ pass
else:
log.warning("A trace event of a type the assembler does not handle was dropped", fields={"trace_event_type": type(event).__name__})
@@ -233,6 +270,7 @@ class _AssemblerState:
self._generate_batch_item_edges()
self._generate_batch_aggregate_edges()
self._generate_parallel_combine_edges()
+ self._generate_selected_outcome_edges()
self._attribute_usage()
def build_graph_spec(self) -> GraphSpec:
@@ -357,10 +395,10 @@ class _AssemblerState:
self._stuff_producer_map[event.output_spec.digest] = event.node_id
def _handle_edge_event(self, event: EdgeEvent) -> None:
- # DATA, BATCH_ITEM, BATCH_AGGREGATE, PARALLEL_COMBINE are regenerated
+ # DATA, BATCH_ITEM, BATCH_AGGREGATE, PARALLEL_COMBINE and SELECTED_OUTCOME are regenerated
# in pass 2 with full cross-worker visibility — skip to avoid duplicates.
match event.edge_kind:
- case EdgeKind.CONTAINS | EdgeKind.SELECTED_OUTCOME | EdgeKind.CONTROL:
+ case EdgeKind.CONTAINS | EdgeKind.CONTROL:
edge = EdgeSpec(
edge_id=event.edge_id,
source=event.source_node_id,
@@ -372,7 +410,7 @@ class _AssemblerState:
target_stuff_digest=event.target_stuff_digest,
)
self._explicit_edges.append(edge)
- case EdgeKind.DATA | EdgeKind.BATCH_ITEM | EdgeKind.BATCH_AGGREGATE | EdgeKind.PARALLEL_COMBINE:
+ case EdgeKind.DATA | EdgeKind.BATCH_ITEM | EdgeKind.BATCH_AGGREGATE | EdgeKind.PARALLEL_COMBINE | EdgeKind.SELECTED_OUTCOME:
pass # Regenerated in pass 2
def _handle_controller_output(self, event: ControllerOutputEvent) -> None:
@@ -429,6 +467,10 @@ class _AssemblerState:
def _handle_condition_output_merge(self, event: ConditionOutputMergeEvent) -> None:
self._condition_output_merges.append(event.merge)
+ def _handle_condition_resolved(self, event: ConditionResolvedEvent) -> None:
+ # No node lookup: the condition's start can come from another workflow, read later in the stream.
+ self._condition_resolutions[event.node_id] = event.selected_outcome
+
def _mark_canceled_nodes(self) -> None:
"""Mark any still-RUNNING nodes as CANCELED (mirrors graph_tracer.py:160-164)."""
for node_data in self._nodes.values():
@@ -579,6 +621,22 @@ class _AssemblerState:
target_stuff_digest=combined_stuff_code,
)
+ def _generate_selected_outcome_edges(self) -> None:
+ """Generate the SELECTED_OUTCOME edges from each resolved condition to the pipe it then started.
+
+ Mirrors GraphTracer._generate_selected_outcome_edges(), through the same rule. The chosen pipe's start can come
+ from a workflow other than the condition's, under a tracer that never saw the resolution, so the edge is derived
+ here from both events rather than read from an edge event one tracer would have had to draw.
+ """
+ parent_by_node = {node_id: node_data.parent_node_id for node_id, node_data in self._nodes.items()}
+ for edge in derive_selected_outcome_edges(resolutions=self._condition_resolutions, parent_by_node=parent_by_node):
+ self._add_generated_edge(
+ source_node_id=edge.condition_node_id,
+ target_node_id=edge.outcome_node_id,
+ edge_kind=EdgeKind.SELECTED_OUTCOME,
+ label=edge.selected_outcome,
+ )
+
# ------------------------------------------------------------------
# Pass 2 usage attribution
# ------------------------------------------------------------------
diff --git a/pipelex/tracing/trace_events.py b/pipelex/tracing/trace_events.py
index 997f666f5..1135d80de 100644
--- a/pipelex/tracing/trace_events.py
+++ b/pipelex/tracing/trace_events.py
@@ -14,6 +14,7 @@ from pydantic import BaseModel, Field, ValidationError
from pipelex.graph.condition_output_merge import ConditionOutputMerge
from pipelex.graph.graphspec import EdgeKind, ErrorSpec, IOSpec, NodeKind
from pipelex.reporting.reporting_types import AnyTokensUsage
+from pipelex.system.pipe_run_mode import PipeRunMode
from pipelex.tools.typing.pydantic_utils import empty_list_factory_of
@@ -30,6 +31,8 @@ class TraceEventKind(StrEnum):
BATCH_AGGREGATE = "batch_aggregate"
PARALLEL_COMBINE = "parallel_combine"
CONDITION_OUTPUT_MERGE = "condition_output_merge"
+ BATCH_FANNED_OUT = "batch_fanned_out"
+ CONDITION_RESOLVED = "condition_resolved"
EXECUTION_DATA = "execution_data"
USAGE_REPORT = "usage_report"
@@ -47,6 +50,8 @@ class TraceEvent(BaseModel):
sort-key shape; non-default values come from the per-process activity
event log used by separate-process workers.
- event_kind: discriminator for the event subclass (defined on each subclass as Literal)
+ - mode: whether the run that emitted the event is live or a dry run. Every event of one run
+ carries the same mode, so a reader tells dry events from live ones by this field alone.
- timestamp: UTC, for display/debugging only
- sequence: per-writer monotonic counter, for ordering and deduplication
"""
@@ -54,6 +59,7 @@ class TraceEvent(BaseModel):
pipeline_run_id: str
workflow_id: str
writer_id: str = "primary"
+ mode: PipeRunMode
timestamp: datetime
sequence: int
@@ -64,7 +70,7 @@ class TraceEvent(BaseModel):
class PipeStartEvent(TraceEvent):
- """Emitted when a pipe begins execution."""
+ """Emitted when a pipe begins execution, before it runs anything."""
event_kind: Literal[TraceEventKind.PIPE_START] = TraceEventKind.PIPE_START
node_id: str
@@ -77,6 +83,11 @@ class PipeStartEvent(TraceEvent):
input_specs: list[IOSpec] = Field(default_factory=empty_list_factory_of(IOSpec))
pipe_data: dict[str, Any] = Field(default_factory=dict)
concept_data: list[dict[str, Any]] = Field(default_factory=empty_list_factory_of(dict))
+ step_count: int | None = None
+ """On a PipeSequence, the number of steps its author wrote. The binding a sequence writes for a dotted
+ `batch_over` is no step of the author's and is not counted. None on every other pipe."""
+ branch_count: int | None = None
+ """On a PipeParallel, the number of its branches. None on every other pipe."""
class PipeEndSuccessEvent(TraceEvent):
@@ -183,6 +194,42 @@ class ConditionOutputMergeEvent(TraceEvent):
merge: ConditionOutputMerge
+class BatchFannedOutEvent(TraceEvent):
+ """Emitted when a PipeBatch knows how many items it runs, before it starts any of them.
+
+ A live reader learns the batch's width the moment it is fixed, so it never has to guess it from the
+ items that have started.
+ """
+
+ event_kind: Literal[TraceEventKind.BATCH_FANNED_OUT] = TraceEventKind.BATCH_FANNED_OUT
+ node_id: str
+ """The PipeBatch's own node."""
+ item_count: int = Field(ge=0)
+ branch_pipe_code: str
+ """The bare code of the pipe each item runs."""
+ branch_pipe_ref: str
+ """The `domain.code` reference of the pipe each item runs."""
+ branch_pipe_type: str
+ """The type of the pipe each item runs, such as `PipeSequence` or `PipeLLM`."""
+
+
+class ConditionResolvedEvent(TraceEvent):
+ """Emitted when a live PipeCondition has chosen its outcome, before the chosen pipe starts.
+
+ A dry run emits none, since its condition runs the pipe of every outcome.
+ """
+
+ event_kind: Literal[TraceEventKind.CONDITION_RESOLVED] = TraceEventKind.CONDITION_RESOLVED
+ node_id: str
+ """The PipeCondition's own node."""
+ rendered_value: str
+ """The value the condition's expression rendered. It may quote text of the method, or of a library the method
+ depends on, so it is kept for whoever reads the trace and never reaches a caller-facing summary."""
+ selected_outcome: str | None
+ """The key of the outcome map the rendered value matched, or None when it matched none and the condition took its
+ default outcome: `"default"` can itself be a key of an outcome map, so it cannot stand for the default."""
+
+
class ExecutionDataEvent(TraceEvent):
"""Emitted when a pipe registers execution metadata (rendered prompts, resolved models, etc.)."""
@@ -220,6 +267,8 @@ AnyTraceEvent = Annotated[
| BatchAggregateEvent
| ParallelCombineEvent
| ConditionOutputMergeEvent
+ | BatchFannedOutEvent
+ | ConditionResolvedEvent
| ExecutionDataEvent
| UsageReportEvent,
Field(discriminator="event_kind"),
diff --git a/subject_grants.toml b/subject_grants.toml
index a9df7958d..d1d6a0413 100644
--- a/subject_grants.toml
+++ b/subject_grants.toml
@@ -6932,6 +6932,10 @@ rationale = "Dispatch handler: handles the event; single operand."
param = "event"
rationale = "An event handler of the assembler's pass one, called with the one event it folds in, like its sibling handlers."
+["pipelex/tracing/graphspec_assembler.py::_AssemblerState._handle_condition_resolved"]
+param = "event"
+rationale = "An event handler of the assembler's pass one, called with the one event it folds in, like its sibling handlers."
+
["pipelex/tracing/graphspec_assembler.py::_AssemblerState._handle_controller_output"]
param = "event"
rationale = "Dispatch handler: handles the event; single operand."
diff --git a/tests/data/errors/error_identity.txt b/tests/data/errors/error_identity.txt
index 043dcd040..700782a19 100644
--- a/tests/data/errors/error_identity.txt
+++ b/tests/data/errors/error_identity.txt
@@ -338,6 +338,7 @@ TelemetryConfigValidationError | Telemetry config validation | https://docs.pipe
TemplateSigilSyntaxError | Template sigil syntax | https://docs.pipelex.com/latest/errors/template-sigil-syntax-error/
TomlError | TOML parse error | https://docs.pipelex.com/latest/errors/toml-error/
ToolError | Tool error | https://docs.pipelex.com/latest/errors/tool-error/
+TraceEventModeError | Trace event mode | https://docs.pipelex.com/latest/errors/trace-event-mode-error/
TracebackMessageError | Traceback message | https://docs.pipelex.com/latest/errors/traceback-message-error/
TypesafeError | Typesafe | https://docs.pipelex.com/latest/errors/typesafe-error/
TypesafeJudgmentResponseError | Typesafe judgment response | https://docs.pipelex.com/latest/errors/typesafe-judgment-response-error/
diff --git a/tests/helpers/usage_stats_run_events.py b/tests/helpers/usage_stats_run_events.py
index 3cb9abd2b..a545d53d3 100644
--- a/tests/helpers/usage_stats_run_events.py
+++ b/tests/helpers/usage_stats_run_events.py
@@ -8,6 +8,7 @@ from pipelex.cogt.usage.cost_category import CostCategory
from pipelex.cogt.usage.token_category import TokenCategory
from pipelex.graph.graphspec import NodeKind
from pipelex.system.job_metadata import JobCategory, JobMetadata, RunMetadata, UnitJobId
+from pipelex.system.pipe_run_mode import PipeRunMode
from pipelex.system.usage_stats.usage_stats_contract import UsageStatsArch, UsageStatsIntegration, UsageStatsOs, UsageStatsSurface
from pipelex.system.usage_stats.usage_stats_events import CommonProperties, UsageStatsEvent, make_envelope
from pipelex.tracing.trace_events import PipeStartEvent, UsageReportEvent
@@ -50,6 +51,7 @@ def llm_usage_event(
workflow_id="direct",
timestamp=datetime.now(UTC),
sequence=sequence,
+ mode=PipeRunMode.LIVE,
node_id=f"{USAGE_STATS_RUN_ID}:node_{sequence}",
tokens_usage=tokens_usage,
)
@@ -61,6 +63,7 @@ def pipe_start_event(*, pipe_type: str, sequence: int) -> PipeStartEvent:
workflow_id="direct",
timestamp=datetime.now(UTC),
sequence=sequence,
+ mode=PipeRunMode.LIVE,
node_id=f"{USAGE_STATS_RUN_ID}:node_{sequence}",
pipe_code=f"secret_pipe_{sequence}",
pipe_type=pipe_type,
diff --git a/tests/integration/pipelex/kernel/test_usage_parity.py b/tests/integration/pipelex/kernel/test_usage_parity.py
index 31cfbf7ec..3b728d33c 100644
--- a/tests/integration/pipelex/kernel/test_usage_parity.py
+++ b/tests/integration/pipelex/kernel/test_usage_parity.py
@@ -118,6 +118,7 @@ class TestKernelUsageParity:
data_inclusion=get_config().interpreter.pipeline_execution.graph.data_inclusion,
emit_graph_events=False,
emit_usage_events=True,
+ run_mode=PipeRunMode.DRY,
)
get_report_delegate().set_event_log(
context_key=trace_context.lookup_key,
diff --git a/tests/integration/pipelex/pipeline/test_decisions_at_decision_time.py b/tests/integration/pipelex/pipeline/test_decisions_at_decision_time.py
new file mode 100644
index 000000000..fcf5a3774
--- /dev/null
+++ b/tests/integration/pipelex/pipeline/test_decisions_at_decision_time.py
@@ -0,0 +1,285 @@
+"""A run's NDJSON trace states each controller decision before the work under it, and every event states its mode.
+
+The trace is read as a reader tailing the file would read it: line by line, in the order the run wrote it. A condition's
+resolution comes before the first event of the pipe it chose, a batch's fan-out before the first event of its items, a
+sequence and a parallel state their width on their own start, and every line says whether the run is live or dry.
+"""
+
+import json
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+import pytest
+from pytest_mock import MockerFixture
+
+import pipelex.pipeline.pipeline_run_setup as pipeline_run_setup_module
+from pipelex.config import get_config
+from pipelex.pipeline.runner import PipelexMTHDSProtocol
+from pipelex.runtime_hub import scoped_event_log
+from pipelex.system.pipe_run_mode import PipeRunMode
+from pipelex.tracing.ndjson_event_log import NdjsonEventLog
+
+if TYPE_CHECKING:
+ from pipelex.pipe_run.pipe_job import PipeJob
+
+_DOMAIN = "decisions_at_decision_time"
+_BUNDLE = f"""
+domain = "{_DOMAIN}"
+description = "Route a request, answer every ticket of an inbox, then weigh the outcome two ways"
+main_pipe = "triage"
+
+[concept.Ticket]
+description = "A support ticket"
+
+[concept.Ticket.structure]
+subject = {{ type = "text", description = "What the ticket is about", required = true }}
+
+[concept.Inbox]
+description = "An inbox of support tickets"
+
+[concept.Inbox.structure]
+tickets = {{ type = "list", item_type = "concept", item_concept_ref = "Ticket", description = "The tickets", required = true }}
+
+[pipe.triage]
+type = "PipeSequence"
+description = "Route, answer each ticket, weigh, conclude"
+inputs = {{ request = "Text", inbox = "Inbox" }}
+output = "Text"
+steps = [
+ {{ pipe = "route", result = "routed" }},
+ {{ pipe = "answer_ticket", batch_over = "inbox.tickets", batch_as = "ticket", result = "answers" }},
+ {{ pipe = "weigh", result = "weighed" }},
+ {{ pipe = "conclude", result = "conclusion" }},
+]
+
+[pipe.route]
+type = "PipeCondition"
+description = "Escalate an urgent request, file any other"
+inputs = {{ request = "Text" }}
+output = "Text"
+expression = "request"
+outcomes = {{ urgent = "escalate" }}
+default_outcome = "file_away"
+
+[pipe.escalate]
+type = "PipeCompose"
+description = "Escalate the request"
+inputs = {{ request = "Text" }}
+output = "Text"
+template = "Escalated: {{{{ request }}}}"
+
+[pipe.file_away]
+type = "PipeCompose"
+description = "File the request"
+inputs = {{ request = "Text" }}
+output = "Text"
+template = "Filed: {{{{ request }}}}"
+
+[pipe.answer_ticket]
+type = "PipeCompose"
+description = "Answer one ticket"
+inputs = {{ ticket = "Ticket" }}
+output = "Text"
+template = "Answer to {{{{ ticket.subject }}}}"
+
+[pipe.weigh]
+type = "PipeParallel"
+description = "Weigh the routed request two ways at once"
+inputs = {{ routed = "Text" }}
+output = "Composite"
+add_each_output = true
+branches = [
+ {{ pipe = "weigh_short", result = "short_view" }},
+ {{ pipe = "weigh_long", result = "long_view" }},
+]
+
+[pipe.weigh_short]
+type = "PipeCompose"
+description = "The short view"
+inputs = {{ routed = "Text" }}
+output = "Text"
+template = "Short: {{{{ routed }}}}"
+
+[pipe.weigh_long]
+type = "PipeCompose"
+description = "The long view"
+inputs = {{ routed = "Text" }}
+output = "Text"
+template = "Long: {{{{ routed }}}}"
+
+[pipe.conclude]
+type = "PipeCompose"
+description = "Conclude on both views and the answers"
+inputs = {{ short_view = "Text", long_view = "Text", answers = "Text[]" }}
+output = "Text"
+template = "{{{{ short_view }}}} / {{{{ long_view }}}} / {{{{ answers }}}}"
+"""
+
+_INBOX = {"tickets": [{"subject": "Login"}, {"subject": "Billing"}]}
+
+
+def _read_trace_lines(traces_dir: Path) -> list[dict[str, Any]]:
+ """Every line of the run's one NDJSON file, in the order the run wrote them."""
+ ndjson_files = list(traces_dir.rglob("*.ndjson"))
+ assert len(ndjson_files) == 1, ndjson_files
+ return [json.loads(line) for line in ndjson_files[0].read_text(encoding="utf-8").splitlines() if line.strip()]
+
+
+def _descendants(*, lines: list[dict[str, Any]], node_id: str) -> set[str]:
+ """The node IDs of every pipe started under `node_id`, at any depth."""
+ children_by_parent: dict[str, list[str]] = {}
+ for line in lines:
+ if line["event_kind"] == "pipe_start" and line["parent_node_id"] is not None:
+ children_by_parent.setdefault(line["parent_node_id"], []).append(line["node_id"])
+ found: set[str] = set()
+ pending = [node_id]
+ while pending:
+ for child_id in children_by_parent.get(pending.pop(), []):
+ found.add(child_id)
+ pending.append(child_id)
+ return found
+
+
+def _first_line_under(*, lines: list[dict[str, Any]], node_id: str) -> int:
+ """The position of the first line naming a pipe started under `node_id`: its start, an edge into it, or its data."""
+ under = _descendants(lines=lines, node_id=node_id)
+ assert under, f"nothing ran under {node_id}"
+ return min(
+ position
+ for position, line in enumerate(lines)
+ if line.get("node_id") in under or line.get("target_node_id") in under or line.get("batch_controller_node_id") in under
+ )
+
+
+def _one(*, lines: list[dict[str, Any]], event_kind: str) -> tuple[int, dict[str, Any]]:
+ found = [(position, line) for position, line in enumerate(lines) if line["event_kind"] == event_kind]
+ assert len(found) == 1, f"{event_kind}: {found}"
+ return found[0]
+
+
+def _starts_by_code(lines: list[dict[str, Any]]) -> dict[str, list[dict[str, Any]]]:
+ starts: dict[str, list[dict[str, Any]]] = {}
+ for line in lines:
+ if line["event_kind"] == "pipe_start":
+ starts.setdefault(line["pipe_code"], []).append(line)
+ return starts
+
+
[email protected](loop_scope="class")
+class TestDecisionsAtDecisionTime:
+ @pytest.mark.parametrize(
+ ("request_text", "selected_outcome", "chosen_pipe_code"),
+ [("urgent", "urgent", "escalate"), ("routine", None, "file_away")],
+ ids=["matched_key", "default_outcome"],
+ )
+ async def test_live_trace_states_each_decision_before_the_work_under_it(
+ self, tmp_path: Path, request_text: str, selected_outcome: str | None, chosen_pipe_code: str
+ ) -> None:
+ execution_config = get_config().interpreter.pipeline_execution.with_execution_overrides(generate_graph=True)
+ runner = PipelexMTHDSProtocol(execution_config=execution_config)
+
+ with scoped_event_log(NdjsonEventLog(traces_dir=str(tmp_path))):
+ response = await runner.execute(
+ mthds_contents=[_BUNDLE],
+ inputs={"request": request_text, "inbox": {"concept": f"{_DOMAIN}.Inbox", "content": _INBOX}},
+ )
+
+ prefix = "Escalated" if chosen_pipe_code == "escalate" else "Filed"
+ assert response.pipe_output.main_stuff.as_text.text.startswith(f"Short: {prefix}: {request_text} / Long: {prefix}: {request_text} / ")
+ lines = _read_trace_lines(tmp_path)
+
+ # Every line states the run's mode, and so does the graph built from them.
+ assert {line["mode"] for line in lines} == {"live"}
+ assert response.pipe_output.graph_spec is not None
+ assert response.pipe_output.graph_spec.meta["mode"] == "live"
+
+ # The condition's resolution precedes the first event of the pipe it chose.
+ resolution_position, resolution = _one(lines=lines, event_kind="condition_resolved")
+ starts = _starts_by_code(lines)
+ route_start = starts["route"][0]
+ assert resolution["node_id"] == route_start["node_id"]
+ assert resolution["rendered_value"] == request_text
+ assert resolution["selected_outcome"] == selected_outcome
+ assert resolution_position < _first_line_under(lines=lines, node_id=route_start["node_id"])
+ chosen_start = starts[chosen_pipe_code][0]
+ assert chosen_start["parent_node_id"] == route_start["node_id"]
+ # The graph built from those events draws the choice as its SELECTED_OUTCOME edge, labelled with the key.
+ graph_spec = response.pipe_output.graph_spec
+ assert graph_spec is not None
+ assert [(edge.source, edge.target, edge.label) for edge in graph_spec.edges if edge.kind.is_selected_outcome] == [
+ (route_start["node_id"], chosen_start["node_id"], selected_outcome)
+ ]
+
+ # The batch's fan-out precedes the first event of its items.
+ fan_out_position, fan_out = _one(lines=lines, event_kind="batch_fanned_out")
+ batch_start = next(line for line in lines if line["event_kind"] == "pipe_start" and line["node_id"] == fan_out["node_id"])
+ assert batch_start["pipe_type"] == "PipeBatch"
+ assert (fan_out["item_count"], fan_out["branch_pipe_code"], fan_out["branch_pipe_ref"], fan_out["branch_pipe_type"]) == (
+ 2,
+ "answer_ticket",
+ f"{_DOMAIN}.answer_ticket",
+ "PipeCompose",
+ )
+ assert fan_out_position < _first_line_under(lines=lines, node_id=fan_out["node_id"])
+ assert len(starts["answer_ticket"]) == 2
+
+ # A sequence states the steps its author wrote, the binding of the dotted `batch_over` not among them, and a
+ # parallel its branches, each on its own start; no other pipe states either.
+ assert (starts["triage"][0]["step_count"], starts["triage"][0]["branch_count"]) == (4, None)
+ assert (starts["weigh"][0]["step_count"], starts["weigh"][0]["branch_count"]) == (None, 2)
+ for pipe_code, code_starts in starts.items():
+ if pipe_code not in {"triage", "weigh"}:
+ assert all(start["step_count"] is None and start["branch_count"] is None for start in code_starts), pipe_code
+
+ async def test_dry_trace_states_its_mode_and_resolves_no_condition(self, tmp_path: Path) -> None:
+ """A dry run runs every outcome of a condition and chooses none, so it records no resolution."""
+ execution_config = get_config().interpreter.pipeline_execution.with_execution_overrides(generate_graph=True, mock_inputs=True)
+ runner = PipelexMTHDSProtocol(pipe_run_mode=PipeRunMode.DRY, execution_config=execution_config)
+
+ with scoped_event_log(NdjsonEventLog(traces_dir=str(tmp_path))):
+ response = await runner.execute(mthds_contents=[_BUNDLE])
+
+ lines = _read_trace_lines(tmp_path)
+ assert {line["mode"] for line in lines} == {"dry"}
+ assert [line for line in lines if line["event_kind"] == "condition_resolved"] == []
+ graph_spec = response.pipe_output.graph_spec
+ assert graph_spec is not None
+ assert graph_spec.meta["mode"] == "dry"
+ assert [edge for edge in graph_spec.edges if edge.kind.is_selected_outcome] == []
+ starts = _starts_by_code(lines)
+ assert starts["triage"][0]["step_count"] == 4
+ assert starts["weigh"][0]["branch_count"] == 2
+
+ async def test_a_live_request_on_a_keyless_boot_is_dry_throughout(self, tmp_path: Path, mocker: MockerFixture) -> None:
+ """A keyless boot runs a requested live run dry: its job, its trace context, every event and its graph all say dry."""
+ mocker.patch("pipelex.runtime_hub.is_dry_run_forced", return_value=True)
+ prepare_pipe_job_spy = mocker.spy(pipeline_run_setup_module, "prepare_pipe_job")
+ execution_config = get_config().interpreter.pipeline_execution.with_execution_overrides(generate_graph=True)
+ runner = PipelexMTHDSProtocol(pipe_run_mode=PipeRunMode.LIVE, execution_config=execution_config)
+
+ with scoped_event_log(NdjsonEventLog(traces_dir=str(tmp_path))):
+ response = await runner.execute(
+ mthds_contents=[_BUNDLE],
+ inputs={"request": "urgent", "inbox": {"concept": f"{_DOMAIN}.Inbox", "content": _INBOX}},
+ )
+
+ pipe_job: PipeJob = prepare_pipe_job_spy.spy_return
+ assert pipe_job.pipe_run_params.run_mode.is_dry
+ trace_context = pipe_job.job_metadata.trace_context
+ assert trace_context is not None
+ assert trace_context.run_mode.is_dry
+ lines = _read_trace_lines(tmp_path)
+ assert {line["mode"] for line in lines} == {"dry"}
+ graph_spec = response.pipe_output.graph_spec
+ assert graph_spec is not None
+ assert graph_spec.meta["mode"] == "dry"
+ # The pipes ran dry too: the condition ran every outcome and resolved none.
+ assert [line for line in lines if line["event_kind"] == "condition_resolved"] == []
+
+ async def test_a_keyless_boot_still_refuses_mock_usage_on_a_live_request(self, mocker: MockerFixture) -> None:
+ """The requested mode is validated before the keyless coercion, so it cannot legalise mock usage on a live request."""
+ mocker.patch("pipelex.runtime_hub.is_dry_run_forced", return_value=True)
+ runner = PipelexMTHDSProtocol(pipe_run_mode=PipeRunMode.LIVE, is_mock_usage=True)
+
+ with pytest.raises(ValueError, match="is_mock_usage"):
+ await runner.execute(mthds_contents=[_BUNDLE], inputs={"request": "urgent"})
diff --git a/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_method_faults.py b/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_method_faults.py
index 7f0c5f5a6..66a3f8349 100644
--- a/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_method_faults.py
+++ b/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_method_faults.py
@@ -7,6 +7,7 @@ its message under STRICT disclosure with a next step, and quotes no text of the
host library's.
"""
+from datetime import UTC, datetime
from pathlib import Path
from typing import Any, Callable
@@ -17,6 +18,8 @@ from pipelex.core.concepts.native.concept_native import NativeConceptCode
from pipelex.core.domains.domain import SpecialDomain
from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
from pipelex.core.pipes.exceptions import PipeRunError
+from pipelex.graph.graph_tracer_manager import GraphTracerManager
+from pipelex.graph.graphspec import NodeKind
from pipelex.pipe_controllers.condition.pipe_condition import PipeCondition
from pipelex.pipe_controllers.condition.pipe_condition_blueprint import PipeConditionBlueprint
from pipelex.pipe_machinery.pipe_factory import PipeFactory
@@ -24,6 +27,9 @@ from pipelex.pipe_run.located_failure import find_root_fault
from pipelex.pipe_run.pipe_run_params_factory import PipeRunParamsFactory
from pipelex.system.job_metadata import JobMetadata
from pipelex.system.pipe_run_mode import PipeRunMode
+from pipelex.tracing.in_memory_event_log import InMemoryEventLog
+from pipelex.tracing.trace_events import ConditionResolvedEvent, ExecutionDataEvent
+from tests.unit.pipelex.graph.conftest import make_defaulted_data_inclusion_config
_TEXT = f"{SpecialDomain.NATIVE}.{NativeConceptCode.TEXT}"
_CONDITION_LIBRARY = Path("tests/integration/pipelex/pipes/controller/pipe_condition")
@@ -79,6 +85,77 @@ class TestPipeConditionMethodFaults:
),
}
+ @pytest.mark.parametrize(
+ ("outcome", "resolution"),
+ [
+ ("test_integration2.capitalize_long_text", None),
+ ("fail", ("long", "long")),
+ ],
+ ids=["chosen_pipe_missing_its_inputs", "fail_outcome"],
+ )
+ async def test_the_trace_records_a_choice_only_once_it_is_final(
+ self,
+ job_metadata: JobMetadata,
+ load_test_library: Callable[[list[Path]], None],
+ outcome: str,
+ resolution: tuple[str, str] | None,
+ ) -> None:
+ """A chosen pipe whose inputs are missing fails the condition before it runs: the trace records no outcome taken.
+
+ A `fail` outcome is the condition's own final choice, so its resolution is recorded before the condition raises.
+ Neither starts a pipe under the condition, so the graph draws no SELECTED_OUTCOME edge.
+ """
+ load_test_library([_CONDITION_LIBRARY])
+ pipe_condition = _make_condition(
+ pipe_code="length_gate",
+ expression_template="long",
+ outcomes={"long": outcome},
+ default_outcome="test_integration2.add_prefix_short_text",
+ )
+ pipeline_run_id = job_metadata.run_metadata.pipeline_run_id
+ event_log = InMemoryEventLog()
+ tracer_manager = GraphTracerManager.get_or_create_instance()
+ root_context = tracer_manager.open_tracer(
+ graph_id=pipeline_run_id,
+ data_inclusion=make_defaulted_data_inclusion_config(),
+ event_log=event_log,
+ pipeline_run_id=pipeline_run_id,
+ run_mode=PipeRunMode.LIVE,
+ )
+ try:
+ condition_node_id, condition_context = tracer_manager.on_pipe_start(
+ trace_context=root_context,
+ pipe_code=pipe_condition.code,
+ pipe_type=pipe_condition.type,
+ node_kind=NodeKind.CONTROLLER,
+ started_at=datetime.now(UTC),
+ )
+ assert condition_node_id is not None
+ with pytest.raises(PipeRunError):
+ await pipe_condition._live_run_controller_pipe( # ruff: ignore[private-member-access] # pyright: ignore[reportPrivateUsage]
+ job_metadata=job_metadata.model_copy(update={"trace_context": condition_context}),
+ working_memory=WorkingMemoryFactory.make_empty(),
+ pipe_run_params=PipeRunParamsFactory.make_run_params(pipe_run_mode=PipeRunMode.LIVE),
+ )
+ finally:
+ graph_spec = tracer_manager.close_tracer(pipeline_run_id)
+
+ events = event_log.read_events(pipeline_run_id)
+ resolutions = [(event.rendered_value, event.selected_outcome) for event in events if isinstance(event, ConditionResolvedEvent)]
+ selected_outcomes = [
+ event.execution_data["selected_outcome"]
+ for event in events
+ if isinstance(event, ExecutionDataEvent) and "selected_outcome" in event.execution_data
+ ]
+ if resolution is None:
+ assert resolutions == []
+ assert selected_outcomes == []
+ else:
+ assert resolutions == [resolution]
+ assert selected_outcomes == [outcome]
+ assert graph_spec is not None
+ assert [edge for edge in graph_spec.edges if edge.kind.is_selected_outcome] == []
+
async def test_an_expression_that_does_not_parse_names_its_line_only(
self, job_metadata: JobMetadata, load_test_library: Callable[[list[Path]], None]
) -> None:
diff --git a/tests/unit/pipelex/graph/conftest.py b/tests/unit/pipelex/graph/conftest.py
index ac394f073..3f3551bc6 100644
--- a/tests/unit/pipelex/graph/conftest.py
+++ b/tests/unit/pipelex/graph/conftest.py
@@ -5,6 +5,7 @@ import pytest
from pipelex.config import get_config
from pipelex.graph.graph_config import GraphConfig
from pipelex.system.data_inclusion_config import DataInclusionConfig
+from pipelex.system.pipe_run_mode import PipeRunMode
from pipelex.system.trace_context import TraceContext
@@ -57,6 +58,7 @@ def make_trace_context(
stuff_json_content=stuff_json_content,
error_stack_traces=error_stack_traces,
),
+ run_mode=PipeRunMode.LIVE,
)
diff --git a/tests/unit/pipelex/graph/test_graph_tracer.py b/tests/unit/pipelex/graph/test_graph_tracer.py
index 0e0af3954..b7f35d480 100644
--- a/tests/unit/pipelex/graph/test_graph_tracer.py
+++ b/tests/unit/pipelex/graph/test_graph_tracer.py
@@ -4,6 +4,7 @@ from datetime import UTC, datetime, timedelta
from pipelex.graph.graph_tracer import GraphTracer
from pipelex.graph.graphspec import EdgeKind, GraphSpecMode, IOSpec, NodeKind, NodeStatus
+from pipelex.system.pipe_run_mode import PipeRunMode
from tests.unit.pipelex.graph.conftest import make_defaulted_data_inclusion_config
@@ -13,7 +14,7 @@ class TestGraphTracer:
def test_setup_returns_initial_context(self) -> None:
"""Test that setup returns a properly initialized TraceContext."""
tracer = GraphTracer()
- context = tracer.setup(graph_id="test-graph-001", data_inclusion=make_defaulted_data_inclusion_config())
+ context = tracer.setup(graph_id="test-graph-001", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)
assert context.graph_id == "test-graph-001"
assert context.parent_node_id is None
@@ -27,6 +28,7 @@ class TestGraphTracer:
data_inclusion=make_defaulted_data_inclusion_config(),
pipeline_ref_domain="test.domain",
pipeline_ref_main_pipe="main_pipe",
+ run_mode=PipeRunMode.LIVE,
)
assert context.graph_id == "test-graph-002"
@@ -39,11 +41,12 @@ class TestGraphTracer:
def test_setup_with_dry_mode_stamps_graphspec_meta(self) -> None:
tracer = GraphTracer()
- tracer.setup(
+ context = tracer.setup(
graph_id="test-graph-dry",
data_inclusion=make_defaulted_data_inclusion_config(),
- mode=GraphSpecMode.DRY,
+ run_mode=PipeRunMode.DRY,
)
+ assert context.run_mode == PipeRunMode.DRY
graph_spec = tracer.teardown()
@@ -61,7 +64,7 @@ class TestGraphTracer:
def test_full_lifecycle_single_node(self) -> None:
"""Test tracking a single pipe execution."""
tracer = GraphTracer()
- context = tracer.setup(graph_id="lifecycle-test", data_inclusion=make_defaulted_data_inclusion_config())
+ context = tracer.setup(graph_id="lifecycle-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)
started_at = datetime.now(UTC)
node_id, child_context = tracer.on_pipe_start(
@@ -103,7 +106,7 @@ class TestGraphTracer:
def test_nested_pipe_execution(self) -> None:
"""Test tracking nested pipe execution with containment edges."""
tracer = GraphTracer()
- context = tracer.setup(graph_id="nested-test", data_inclusion=make_defaulted_data_inclusion_config())
+ context = tracer.setup(graph_id="nested-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)
# Start parent (sequence controller)
started_at = datetime.now(UTC)
@@ -164,7 +167,7 @@ class TestGraphTracer:
def test_error_tracking(self) -> None:
"""Test tracking failed pipe execution."""
tracer = GraphTracer()
- context = tracer.setup(graph_id="error-test", data_inclusion=make_defaulted_data_inclusion_config())
+ context = tracer.setup(graph_id="error-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)
started_at = datetime.now(UTC)
node_id, _ = tracer.on_pipe_start(
@@ -196,7 +199,7 @@ class TestGraphTracer:
def test_skipped_tracking(self) -> None:
"""A lifted (skipped) pipe ends its node in the SKIPPED state with the skip reason."""
tracer = GraphTracer()
- context = tracer.setup(graph_id="skip-test", data_inclusion=make_defaulted_data_inclusion_config())
+ context = tracer.setup(graph_id="skip-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)
started_at = datetime.now(UTC)
node_id, _ = tracer.on_pipe_start(
@@ -227,7 +230,7 @@ class TestGraphTracer:
rides the skip and registers in the producer map, so the downstream DATA edge resolves.
"""
tracer = GraphTracer()
- context = tracer.setup(graph_id="skip-plural-edge-test", data_inclusion=make_defaulted_data_inclusion_config())
+ context = tracer.setup(graph_id="skip-plural-edge-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)
started_at = datetime.now(UTC)
producer_id, _ = tracer.on_pipe_start(
@@ -269,7 +272,7 @@ class TestGraphTracer:
def test_data_edge_from_optional_output_carries_marker(self) -> None:
"""A DATA edge whose producer output is declared optional (`?`) reports optional=True."""
tracer = GraphTracer()
- context = tracer.setup(graph_id="optional-edge-test", data_inclusion=make_defaulted_data_inclusion_config())
+ context = tracer.setup(graph_id="optional-edge-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)
started_at = datetime.now(UTC)
producer_id, _ = tracer.on_pipe_start(
@@ -307,7 +310,7 @@ class TestGraphTracer:
def test_running_nodes_marked_canceled_on_teardown(self) -> None:
"""Test that running nodes are marked as canceled on teardown."""
tracer = GraphTracer()
- context = tracer.setup(graph_id="cancel-test", data_inclusion=make_defaulted_data_inclusion_config())
+ context = tracer.setup(graph_id="cancel-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)
started_at = datetime.now(UTC)
_node_id, _ = tracer.on_pipe_start(
@@ -330,7 +333,7 @@ class TestGraphTracer:
def test_custom_edges(self) -> None:
"""Test adding custom edges between nodes."""
tracer = GraphTracer()
- context = tracer.setup(graph_id="edge-test", data_inclusion=make_defaulted_data_inclusion_config())
+ context = tracer.setup(graph_id="edge-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)
started_at = datetime.now(UTC)
node1_id, _ = tracer.on_pipe_start(
@@ -372,7 +375,7 @@ class TestGraphTracer:
def test_selected_outcome_edge(self) -> None:
"""Test adding selected outcome edge for conditions."""
tracer = GraphTracer()
- context = tracer.setup(graph_id="condition-test", data_inclusion=make_defaulted_data_inclusion_config())
+ context = tracer.setup(graph_id="condition-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)
started_at = datetime.now(UTC)
condition_id, cond_ctx = tracer.on_pipe_start(
@@ -396,7 +399,7 @@ class TestGraphTracer:
tracer.add_selected_outcome_edge(
condition_node_id=condition_id,
outcome_node_id=outcome_id,
- outcome_value="true",
+ selected_outcome="true",
)
tracer.on_pipe_end_success(node_id=condition_id, ended_at=started_at + timedelta(milliseconds=25))
@@ -411,7 +414,7 @@ class TestGraphTracer:
def test_input_specs_captured(self) -> None:
"""Test that input IOSpecs are captured and stored in node_io."""
tracer = GraphTracer()
- context = tracer.setup(graph_id="io-test", data_inclusion=make_defaulted_data_inclusion_config())
+ context = tracer.setup(graph_id="io-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)
input_specs = [
IOSpec(name="document", concept="Text", digest="abc12"),
@@ -442,7 +445,7 @@ class TestGraphTracer:
def test_output_spec_captured(self) -> None:
"""Test that output IOSpec is captured and stored in node_io."""
tracer = GraphTracer()
- context = tracer.setup(graph_id="io-test", data_inclusion=make_defaulted_data_inclusion_config())
+ context = tracer.setup(graph_id="io-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)
output_spec = IOSpec(
name="summary",
@@ -477,7 +480,7 @@ class TestGraphTracer:
def test_data_edge_generation_from_stuff_codes(self) -> None:
"""Test that DATA edges are created when stuff_codes match between output and input."""
tracer = GraphTracer()
- context = tracer.setup(graph_id="data-flow-test", data_inclusion=make_defaulted_data_inclusion_config())
+ context = tracer.setup(graph_id="data-flow-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)
started_at = datetime.now(UTC)
@@ -521,7 +524,7 @@ class TestGraphTracer:
def test_data_edge_not_created_for_unknown_producer(self) -> None:
"""Test that no DATA edge is created if the producer is unknown (initial pipeline input)."""
tracer = GraphTracer()
- context = tracer.setup(graph_id="no-producer-test", data_inclusion=make_defaulted_data_inclusion_config())
+ context = tracer.setup(graph_id="no-producer-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)
started_at = datetime.now(UTC)
@@ -548,7 +551,7 @@ class TestGraphTracer:
def test_no_self_loop_data_edges(self) -> None:
"""Test that DATA edges are not created as self-loops."""
tracer = GraphTracer()
- context = tracer.setup(graph_id="no-self-loop-test", data_inclusion=make_defaulted_data_inclusion_config())
+ context = tracer.setup(graph_id="no-self-loop-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)
started_at = datetime.now(UTC)
@@ -577,7 +580,7 @@ class TestGraphTracer:
def test_multiple_consumers_same_stuff(self) -> None:
"""Test DATA edges when multiple pipes consume the same stuff."""
tracer = GraphTracer()
- context = tracer.setup(graph_id="multi-consumer-test", data_inclusion=make_defaulted_data_inclusion_config())
+ context = tracer.setup(graph_id="multi-consumer-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)
started_at = datetime.now(UTC)
@@ -635,7 +638,7 @@ class TestGraphTracer:
created from the list consumer to the item consumers.
"""
tracer = GraphTracer()
- context = tracer.setup(graph_id="batch-item-test", data_inclusion=make_defaulted_data_inclusion_config())
+ context = tracer.setup(graph_id="batch-item-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)
started_at = datetime.now(UTC)
@@ -719,7 +722,7 @@ class TestGraphTracer:
created from the item producers to the list producer.
"""
tracer = GraphTracer()
- context = tracer.setup(graph_id="batch-aggregate-test", data_inclusion=make_defaulted_data_inclusion_config())
+ context = tracer.setup(graph_id="batch-aggregate-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)
started_at = datetime.now(UTC)
@@ -797,7 +800,7 @@ class TestGraphTracer:
def test_batch_edges_combined(self) -> None:
"""Test both BATCH_ITEM and BATCH_AGGREGATE edges in a complete batch scenario."""
tracer = GraphTracer()
- context = tracer.setup(graph_id="batch-combined-test", data_inclusion=make_defaulted_data_inclusion_config())
+ context = tracer.setup(graph_id="batch-combined-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)
started_at = datetime.now(UTC)
@@ -891,7 +894,7 @@ class TestGraphTracer:
and target_stuff_digest should be the item stuff code.
"""
tracer = GraphTracer()
- context = tracer.setup(graph_id="batch-digest-test", data_inclusion=make_defaulted_data_inclusion_config())
+ context = tracer.setup(graph_id="batch-digest-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)
started_at = datetime.now(UTC)
@@ -948,7 +951,7 @@ class TestGraphTracer:
and target_stuff_digest should be the output list stuff code.
"""
tracer = GraphTracer()
- context = tracer.setup(graph_id="batch-agg-digest-test", data_inclusion=make_defaulted_data_inclusion_config())
+ context = tracer.setup(graph_id="batch-agg-digest-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)
started_at = datetime.now(UTC)
@@ -1006,7 +1009,7 @@ class TestGraphTracer:
the controller node to consumers of those outputs.
"""
tracer = GraphTracer()
- context = tracer.setup(graph_id="controller-output-test", data_inclusion=make_defaulted_data_inclusion_config())
+ context = tracer.setup(graph_id="controller-output-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)
started_at = datetime.now(UTC)
@@ -1103,7 +1106,7 @@ class TestGraphTracer:
the output should not be registered to avoid corrupting data edges.
"""
tracer = GraphTracer()
- context = tracer.setup(graph_id="passthrough-test", data_inclusion=make_defaulted_data_inclusion_config())
+ context = tracer.setup(graph_id="passthrough-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)
started_at = datetime.now(UTC)
@@ -1174,7 +1177,7 @@ class TestGraphTracer:
All registered outputs should produce correct DATA edges to their consumers.
"""
tracer = GraphTracer()
- context = tracer.setup(graph_id="multi-output-test", data_inclusion=make_defaulted_data_inclusion_config())
+ context = tracer.setup(graph_id="multi-output-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)
started_at = datetime.now(UTC)
diff --git a/tests/unit/pipelex/graph/test_graph_tracer_decision_events.py b/tests/unit/pipelex/graph/test_graph_tracer_decision_events.py
new file mode 100644
index 000000000..738669a6d
--- /dev/null
+++ b/tests/unit/pipelex/graph/test_graph_tracer_decision_events.py
@@ -0,0 +1,334 @@
+"""Unit tests for the decisions a controller records as it makes them, and for the run mode every trace event states."""
+
+from datetime import UTC, datetime, timedelta
+
+import pytest
+from pydantic import ValidationError
+
+from pipelex.graph.graph_tracer import GraphTracer
+from pipelex.graph.graph_tracer_manager import GraphTracerManager
+from pipelex.graph.graph_tracer_protocol import GraphTracerNoOp
+from pipelex.graph.graphspec import IOSpec, NodeKind
+from pipelex.system.pipe_run_mode import PipeRunMode
+from pipelex.system.trace_context import TraceContext
+from pipelex.tracing.in_memory_event_log import InMemoryEventLog
+from pipelex.tracing.trace_events import (
+ BatchFannedOutEvent,
+ ConditionResolvedEvent,
+ PipeStartEvent,
+ TraceEvent,
+)
+from tests.unit.pipelex.graph.conftest import make_defaulted_data_inclusion_config
+
+_PIPELINE_RUN_ID = "run_decisions_001"
+_GRAPH_ID = "graph-decisions"
+_T0 = datetime(2026, 10, 13, 9, 0, 0, tzinfo=UTC)
+
+
+def _make_tracer(*, run_mode: PipeRunMode) -> tuple[GraphTracer, TraceContext, InMemoryEventLog]:
+ event_log = InMemoryEventLog()
+ tracer = GraphTracer()
+ context = tracer.setup(
+ graph_id=_GRAPH_ID,
+ data_inclusion=make_defaulted_data_inclusion_config(),
+ event_log=event_log,
+ pipeline_run_id=_PIPELINE_RUN_ID,
+ run_mode=run_mode,
+ )
+ return tracer, context, event_log
+
+
+def _start_of(*, events: list[TraceEvent], node_id: str) -> PipeStartEvent:
+ return next(event for event in events if isinstance(event, PipeStartEvent) and event.node_id == node_id)
+
+
+class TestGraphTracerDecisionEvents:
+ @pytest.mark.parametrize(
+ ("rendered_value", "selected_outcome"),
+ [("urgent", "urgent"), ("unmatched", None)],
+ ids=["matched_key", "default_outcome"],
+ )
+ def test_condition_resolution_precedes_the_chosen_pipe(self, rendered_value: str, selected_outcome: str | None) -> None:
+ """The resolution is recorded before the chosen pipe starts, and the graph's SELECTED_OUTCOME edge to it is labelled with the key."""
+ tracer, context, event_log = _make_tracer(run_mode=PipeRunMode.LIVE)
+ condition_id, condition_ctx = tracer.on_pipe_start(
+ trace_context=context, pipe_code="route", pipe_type="PipeCondition", node_kind=NodeKind.CONTROLLER, started_at=_T0
+ )
+
+ tracer.register_condition_resolution(condition_node_id=condition_id, rendered_value=rendered_value, selected_outcome=selected_outcome)
+ chosen_id, _ = tracer.on_pipe_start(
+ trace_context=condition_ctx,
+ pipe_code="handle",
+ pipe_type="PipeCompose",
+ node_kind=NodeKind.OPERATOR,
+ started_at=_T0 + timedelta(seconds=1),
+ )
+ tracer.on_pipe_end_success(node_id=chosen_id, ended_at=_T0 + timedelta(seconds=2))
+ tracer.on_pipe_end_success(node_id=condition_id, ended_at=_T0 + timedelta(seconds=3))
+ graph_spec = tracer.teardown()
+
+ events = event_log.read_events(_PIPELINE_RUN_ID)
+ resolutions = [event for event in events if isinstance(event, ConditionResolvedEvent)]
+ assert len(resolutions) == 1
+ resolution = resolutions[0]
+ assert resolution.node_id == condition_id
+ assert resolution.rendered_value == rendered_value
+ assert resolution.selected_outcome == selected_outcome
+ assert resolution.sequence < _start_of(events=events, node_id=chosen_id).sequence
+
+ assert graph_spec is not None
+ graph_edges = [edge for edge in graph_spec.edges if edge.kind.is_selected_outcome]
+ assert [(edge.source, edge.target, edge.label) for edge in graph_edges] == [(condition_id, chosen_id, selected_outcome)]
+
+ def test_condition_resolution_without_a_pipe_draws_no_edge(self) -> None:
+ """An outcome mapped to `continue` or `fail` starts no pipe: the resolution is recorded, and the graph has no edge from it."""
+ tracer, context, event_log = _make_tracer(run_mode=PipeRunMode.LIVE)
+ sequence_id, sequence_ctx = tracer.on_pipe_start(
+ trace_context=context, pipe_code="flow", pipe_type="PipeSequence", node_kind=NodeKind.CONTROLLER, started_at=_T0, step_count=2
+ )
+ condition_id, _ = tracer.on_pipe_start(
+ trace_context=sequence_ctx, pipe_code="route", pipe_type="PipeCondition", node_kind=NodeKind.CONTROLLER, started_at=_T0
+ )
+
+ tracer.register_condition_resolution(condition_node_id=condition_id, rendered_value="skip", selected_outcome="skip")
+ tracer.on_pipe_end_success(node_id=condition_id, ended_at=_T0 + timedelta(seconds=1))
+ # The next step starts under the sequence, never under the condition
+ next_step_id, _ = tracer.on_pipe_start(
+ trace_context=sequence_ctx,
+ pipe_code="later",
+ pipe_type="PipeCompose",
+ node_kind=NodeKind.OPERATOR,
+ started_at=_T0 + timedelta(seconds=2),
+ )
+ tracer.on_pipe_end_success(node_id=next_step_id, ended_at=_T0 + timedelta(seconds=3))
+ tracer.on_pipe_end_success(node_id=sequence_id, ended_at=_T0 + timedelta(seconds=4))
+ graph_spec = tracer.teardown()
+
+ events = event_log.read_events(_PIPELINE_RUN_ID)
+ assert [event.selected_outcome for event in events if isinstance(event, ConditionResolvedEvent)] == ["skip"]
+ assert graph_spec is not None
+ assert [edge for edge in graph_spec.edges if edge.kind.is_selected_outcome] == []
+
+ def test_batch_fan_out_precedes_the_first_item(self) -> None:
+ tracer, context, event_log = _make_tracer(run_mode=PipeRunMode.LIVE)
+ batch_id, batch_ctx = tracer.on_pipe_start(
+ trace_context=context,
+ pipe_code="per_item",
+ pipe_type="PipeBatch",
+ node_kind=NodeKind.CONTROLLER,
+ started_at=_T0,
+ input_specs=[IOSpec(name="items", digest="digest_items", multiplicity=True)],
+ )
+
+ tracer.register_batch_fan_out(
+ batch_node_id=batch_id, item_count=2, branch_pipe_code="process", branch_pipe_ref="flows.process", branch_pipe_type="PipeCompose"
+ )
+ item_ids: list[str] = []
+ for index_item in range(2):
+ item_id, _ = tracer.on_pipe_start(
+ trace_context=batch_ctx,
+ pipe_code="process",
+ pipe_type="PipeCompose",
+ node_kind=NodeKind.OPERATOR,
+ started_at=_T0 + timedelta(seconds=1 + index_item),
+ )
+ item_ids.append(item_id)
+
+ events = event_log.read_events(_PIPELINE_RUN_ID)
+ fan_outs = [event for event in events if isinstance(event, BatchFannedOutEvent)]
+ assert len(fan_outs) == 1
+ fan_out = fan_outs[0]
+ assert fan_out.node_id == batch_id
+ assert fan_out.item_count == 2
+ assert fan_out.branch_pipe_code == "process"
+ assert fan_out.branch_pipe_ref == "flows.process"
+ assert fan_out.branch_pipe_type == "PipeCompose"
+ first_item_start = _start_of(events=events, node_id=item_ids[0])
+ assert fan_out.sequence < first_item_start.sequence
+
+ def test_controllers_state_their_width_as_they_start(self) -> None:
+ tracer, context, event_log = _make_tracer(run_mode=PipeRunMode.LIVE)
+ sequence_id, sequence_ctx = tracer.on_pipe_start(
+ trace_context=context, pipe_code="flow", pipe_type="PipeSequence", node_kind=NodeKind.CONTROLLER, started_at=_T0, step_count=3
+ )
+ parallel_id, _ = tracer.on_pipe_start(
+ trace_context=sequence_ctx,
+ pipe_code="fan",
+ pipe_type="PipeParallel",
+ node_kind=NodeKind.CONTROLLER,
+ started_at=_T0 + timedelta(seconds=1),
+ branch_count=2,
+ )
+
+ events = event_log.read_events(_PIPELINE_RUN_ID)
+ sequence_start = _start_of(events=events, node_id=sequence_id)
+ parallel_start = _start_of(events=events, node_id=parallel_id)
+ assert (sequence_start.step_count, sequence_start.branch_count) == (3, None)
+ assert (parallel_start.step_count, parallel_start.branch_count) == (None, 2)
+
+ @pytest.mark.parametrize("run_mode", [PipeRunMode.LIVE, PipeRunMode.DRY], ids=["live", "dry"])
+ def test_every_event_states_the_run_mode(self, run_mode: PipeRunMode) -> None:
+ """The tracer stamps its run's mode on every event it writes, and its contexts carry the mode down."""
+ tracer, context, event_log = _make_tracer(run_mode=run_mode)
+ assert context.run_mode == run_mode
+
+ batch_id, batch_ctx = tracer.on_pipe_start(
+ trace_context=context, pipe_code="per_item", pipe_type="PipeBatch", node_kind=NodeKind.CONTROLLER, started_at=_T0
+ )
+ assert batch_ctx.run_mode == run_mode
+ tracer.register_batch_fan_out(
+ batch_node_id=batch_id, item_count=1, branch_pipe_code="route", branch_pipe_ref="flows.route", branch_pipe_type="PipeCondition"
+ )
+ tracer.register_batch_item_extraction(list_stuff_code="list", item_stuff_code="item", item_index=0, batch_controller_node_id=batch_id)
+ condition_id, condition_ctx = tracer.on_pipe_start(
+ trace_context=batch_ctx,
+ pipe_code="route",
+ pipe_type="PipeCondition",
+ node_kind=NodeKind.CONTROLLER,
+ started_at=_T0 + timedelta(seconds=1),
+ )
+ tracer.register_condition_resolution(condition_node_id=condition_id, rendered_value="yes", selected_outcome="yes")
+ chosen_id, _ = tracer.on_pipe_start(
+ trace_context=condition_ctx,
+ pipe_code="handle",
+ pipe_type="PipeCompose",
+ node_kind=NodeKind.OPERATOR,
+ started_at=_T0 + timedelta(seconds=2),
+ )
+ tracer.register_execution_data(node_id=chosen_id, execution_data={"rendered": "text"})
+ tracer.on_pipe_end_success(node_id=chosen_id, ended_at=_T0 + timedelta(seconds=3), output_spec=IOSpec(name="result", digest="result"))
+ tracer.on_pipe_end_success(node_id=condition_id, ended_at=_T0 + timedelta(seconds=4))
+ tracer.register_batch_aggregation(output_list_stuff_code="out", item_stuff_code="result", item_index=0, batch_controller_node_id=batch_id)
+ tracer.on_pipe_end_success(node_id=batch_id, ended_at=_T0 + timedelta(seconds=5))
+ tracer.teardown()
+
+ events = event_log.read_events(_PIPELINE_RUN_ID)
+ kinds = {event.model_dump()["event_kind"] for event in events}
+ assert {"pipe_start", "pipe_end_success", "edge", "batch_fanned_out", "condition_resolved", "batch_item", "batch_aggregate"} <= kinds
+ assert {event.mode for event in events} == {run_mode}
+
+ def test_an_inactive_tracer_records_no_decision(self) -> None:
+ tracer, _, event_log = _make_tracer(run_mode=PipeRunMode.LIVE)
+ tracer.teardown()
+
+ tracer.register_batch_fan_out(
+ batch_node_id="gone", item_count=1, branch_pipe_code="process", branch_pipe_ref="flows.process", branch_pipe_type="PipeCompose"
+ )
+ tracer.register_condition_resolution(condition_node_id="gone", rendered_value="yes", selected_outcome="yes")
+
+ assert event_log.read_events(_PIPELINE_RUN_ID) == []
+
+ def test_a_costs_only_run_records_no_decision(self) -> None:
+ """A run that emits no graph events records no decision, even on a tracer it hands an event log."""
+ event_log = InMemoryEventLog()
+ tracer = GraphTracer()
+ context = tracer.setup(
+ graph_id=_GRAPH_ID,
+ data_inclusion=make_defaulted_data_inclusion_config(),
+ event_log=event_log,
+ pipeline_run_id=_PIPELINE_RUN_ID,
+ emit_graph_events=False,
+ run_mode=PipeRunMode.LIVE,
+ )
+ batch_id, batch_ctx = tracer.on_pipe_start(
+ trace_context=context, pipe_code="per_item", pipe_type="PipeBatch", node_kind=NodeKind.CONTROLLER, started_at=_T0
+ )
+ tracer.register_batch_fan_out(
+ batch_node_id=batch_id, item_count=1, branch_pipe_code="route", branch_pipe_ref="flows.route", branch_pipe_type="PipeCondition"
+ )
+ condition_id, _ = tracer.on_pipe_start(
+ trace_context=batch_ctx,
+ pipe_code="route",
+ pipe_type="PipeCondition",
+ node_kind=NodeKind.CONTROLLER,
+ started_at=_T0 + timedelta(seconds=1),
+ )
+ tracer.register_condition_resolution(condition_node_id=condition_id, rendered_value="yes", selected_outcome="yes")
+ assert tracer.teardown() is None
+
+ events = event_log.read_events(_PIPELINE_RUN_ID)
+ assert [event for event in events if isinstance(event, (BatchFannedOutEvent, ConditionResolvedEvent))] == []
+
+ def test_a_tracer_records_no_decision_on_a_node_it_never_started(self) -> None:
+ """A decision looked up on a tracer that does not hold the controller's node is dropped, as its execution data is.
+
+ The pipe started under that node, as a distributed run's child workflow starts its first pipe under its parent's
+ controller, gets no SELECTED_OUTCOME edge from a node absent from this tracer's graph.
+ """
+ tracer, context, event_log = _make_tracer(run_mode=PipeRunMode.LIVE)
+ foreign_node_id = "graph-decisions:other_workflow:node_0"
+ foreign_ctx = context.copy_for_child(child_node_id=foreign_node_id, next_sequence=0)
+
+ tracer.register_batch_fan_out(
+ batch_node_id=foreign_node_id,
+ item_count=1,
+ branch_pipe_code="process",
+ branch_pipe_ref="flows.process",
+ branch_pipe_type="PipeCompose",
+ )
+ tracer.register_condition_resolution(condition_node_id=foreign_node_id, rendered_value="yes", selected_outcome="yes")
+ chosen_id, _ = tracer.on_pipe_start(
+ trace_context=foreign_ctx, pipe_code="handle", pipe_type="PipeCompose", node_kind=NodeKind.OPERATOR, started_at=_T0
+ )
+ tracer.on_pipe_end_success(node_id=chosen_id, ended_at=_T0 + timedelta(seconds=1))
+ graph_spec = tracer.teardown()
+
+ events = event_log.read_events(_PIPELINE_RUN_ID)
+ assert [event for event in events if isinstance(event, (BatchFannedOutEvent, ConditionResolvedEvent))] == []
+ assert graph_spec is not None
+ assert [edge for edge in graph_spec.edges if edge.kind.is_selected_outcome] == []
+
+ def test_a_trace_context_states_its_run_mode(self) -> None:
+ """A context has no default mode: one built without it would stamp a dry run's events as live."""
+ with pytest.raises(ValidationError, match="run_mode"):
+ TraceContext(graph_id=_GRAPH_ID, data_inclusion=make_defaulted_data_inclusion_config()) # type: ignore[call-arg] # pyright: ignore[reportCallIssue] # the static error is the runtime contract under test
+
+ def test_the_noop_tracer_accepts_decisions_and_carries_the_mode(self) -> None:
+ tracer = GraphTracerNoOp()
+ context = tracer.setup(graph_id="noop", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.DRY)
+
+ tracer.register_batch_fan_out(
+ batch_node_id="node", item_count=1, branch_pipe_code="process", branch_pipe_ref="flows.process", branch_pipe_type="PipeCompose"
+ )
+ tracer.register_condition_resolution(condition_node_id="node", rendered_value="yes", selected_outcome=None)
+
+ assert context.run_mode == PipeRunMode.DRY
+ assert tracer.teardown() is None
+
+ def test_the_manager_routes_decisions_to_the_run_tracer(self) -> None:
+ GraphTracerManager.clear_instance()
+ manager = GraphTracerManager.get_or_create_instance()
+ event_log = InMemoryEventLog()
+ try:
+ context = manager.open_tracer(
+ graph_id=_GRAPH_ID,
+ data_inclusion=make_defaulted_data_inclusion_config(),
+ event_log=event_log,
+ pipeline_run_id=_PIPELINE_RUN_ID,
+ run_mode=PipeRunMode.DRY,
+ )
+ batch_id, _ = manager.on_pipe_start(
+ trace_context=context, pipe_code="per_item", pipe_type="PipeBatch", node_kind=NodeKind.CONTROLLER, started_at=_T0
+ )
+ assert batch_id is not None
+ manager.register_batch_fan_out(
+ lookup_key=context.lookup_key,
+ batch_node_id=batch_id,
+ item_count=4,
+ branch_pipe_code="process",
+ branch_pipe_ref="flows.process",
+ branch_pipe_type="PipeCompose",
+ )
+ manager.register_condition_resolution(
+ lookup_key=context.lookup_key, condition_node_id=batch_id, rendered_value="yes", selected_outcome="yes"
+ )
+ # A key no tracer answers to is a no-op
+ manager.register_condition_resolution(lookup_key="unknown", condition_node_id=batch_id, rendered_value="yes", selected_outcome="yes")
+ finally:
+ manager.close_tracer(_GRAPH_ID)
+ GraphTracerManager.clear_instance()
+
+ events = event_log.read_events(_PIPELINE_RUN_ID)
+ assert [event.item_count for event in events if isinstance(event, BatchFannedOutEvent)] == [4]
+ assert [event.rendered_value for event in events if isinstance(event, ConditionResolvedEvent)] == ["yes"]
+ assert {event.mode for event in events} == {PipeRunMode.DRY}
diff --git a/tests/unit/pipelex/graph/test_graph_tracer_emit_gating.py b/tests/unit/pipelex/graph/test_graph_tracer_emit_gating.py
index 3ff068c2c..0c39dfb6d 100644
--- a/tests/unit/pipelex/graph/test_graph_tracer_emit_gating.py
+++ b/tests/unit/pipelex/graph/test_graph_tracer_emit_gating.py
@@ -2,6 +2,7 @@ from datetime import UTC, datetime
from pipelex.graph.graph_tracer import GraphTracer
from pipelex.graph.graphspec import NodeKind
+from pipelex.system.pipe_run_mode import PipeRunMode
from tests.unit.pipelex.graph.conftest import make_defaulted_data_inclusion_config
@@ -24,13 +25,14 @@ class TestGraphTracerEmitGating:
data_inclusion=make_defaulted_data_inclusion_config(),
emit_graph_events=False,
emit_usage_events=True,
+ run_mode=PipeRunMode.LIVE,
)
assert context.emit_graph_events is False
assert context.emit_usage_events is True
def test_setup_defaults_emit_flags_true(self) -> None:
tracer = GraphTracer()
- context = tracer.setup(graph_id="emit-defaults", data_inclusion=make_defaulted_data_inclusion_config())
+ context = tracer.setup(graph_id="emit-defaults", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)
assert context.emit_graph_events is True
assert context.emit_usage_events is True
@@ -38,9 +40,7 @@ class TestGraphTracerEmitGating:
"""Costs-only mode: the tracer still mints node ids, but teardown skips the discarded spec build."""
tracer = GraphTracer()
context = tracer.setup(
- graph_id="costs-only",
- data_inclusion=make_defaulted_data_inclusion_config(),
- emit_graph_events=False,
+ graph_id="costs-only", data_inclusion=make_defaulted_data_inclusion_config(), emit_graph_events=False, run_mode=PipeRunMode.LIVE
)
self._start_a_node(tracer, context)
assert tracer.teardown() is None
@@ -49,9 +49,7 @@ class TestGraphTracerEmitGating:
def test_teardown_builds_spec_when_graph_events_on(self) -> None:
tracer = GraphTracer()
context = tracer.setup(
- graph_id="graph-on",
- data_inclusion=make_defaulted_data_inclusion_config(),
- emit_graph_events=True,
+ graph_id="graph-on", data_inclusion=make_defaulted_data_inclusion_config(), emit_graph_events=True, run_mode=PipeRunMode.LIVE
)
self._start_a_node(tracer, context)
graph_spec = tracer.teardown()
diff --git a/tests/unit/pipelex/graph/test_graph_tracer_event_emission.py b/tests/unit/pipelex/graph/test_graph_tracer_event_emission.py
index b47ba1e40..c10780b5b 100644
--- a/tests/unit/pipelex/graph/test_graph_tracer_event_emission.py
+++ b/tests/unit/pipelex/graph/test_graph_tracer_event_emission.py
@@ -8,6 +8,7 @@ from datetime import UTC, datetime, timedelta
from pipelex.graph.graph_tracer import GraphTracer
from pipelex.graph.graphspec import EdgeKind, IOSpec, NodeKind, NodeStatus
+from pipelex.system.pipe_run_mode import PipeRunMode
from pipelex.tracing.in_memory_event_log import InMemoryEventLog
from pipelex.tracing.trace_events import (
BatchAggregateEvent,
@@ -42,6 +43,7 @@ class TestGraphTracerEventEmission:
event_log=event_log,
workflow_id=workflow_id or self.WORKFLOW_ID,
pipeline_run_id=self.PIPELINE_RUN_ID,
+ run_mode=PipeRunMode.LIVE,
)
return tracer, event_log
@@ -55,6 +57,7 @@ class TestGraphTracerEventEmission:
event_log=event_log,
workflow_id=self.WORKFLOW_ID,
pipeline_run_id=self.PIPELINE_RUN_ID,
+ run_mode=PipeRunMode.LIVE,
)
started_at = datetime(2025, 6, 15, 10, 0, 0, tzinfo=UTC)
@@ -89,6 +92,7 @@ class TestGraphTracerEventEmission:
event_log=event_log,
workflow_id=self.WORKFLOW_ID,
pipeline_run_id=self.PIPELINE_RUN_ID,
+ run_mode=PipeRunMode.LIVE,
)
started_at = datetime(2025, 6, 15, 10, 0, 0, tzinfo=UTC)
@@ -125,6 +129,7 @@ class TestGraphTracerEventEmission:
event_log=event_log,
workflow_id=self.WORKFLOW_ID,
pipeline_run_id=self.PIPELINE_RUN_ID,
+ run_mode=PipeRunMode.LIVE,
)
started_at = datetime(2025, 6, 15, 10, 0, 0, tzinfo=UTC)
@@ -166,6 +171,7 @@ class TestGraphTracerEventEmission:
event_log=event_log,
workflow_id=self.WORKFLOW_ID,
pipeline_run_id=self.PIPELINE_RUN_ID,
+ run_mode=PipeRunMode.LIVE,
)
started_at = datetime(2025, 6, 15, 10, 0, 0, tzinfo=UTC)
@@ -206,6 +212,7 @@ class TestGraphTracerEventEmission:
event_log=event_log,
workflow_id=self.WORKFLOW_ID,
pipeline_run_id=self.PIPELINE_RUN_ID,
+ run_mode=PipeRunMode.LIVE,
)
started_at = datetime(2025, 6, 15, 10, 0, 0, tzinfo=UTC)
@@ -254,6 +261,7 @@ class TestGraphTracerEventEmission:
event_log=event_log,
workflow_id=self.WORKFLOW_ID,
pipeline_run_id=self.PIPELINE_RUN_ID,
+ run_mode=PipeRunMode.LIVE,
)
started_at = datetime(2025, 6, 15, 10, 0, 0, tzinfo=UTC)
@@ -286,6 +294,7 @@ class TestGraphTracerEventEmission:
event_log=event_log,
workflow_id=self.WORKFLOW_ID,
pipeline_run_id=self.PIPELINE_RUN_ID,
+ run_mode=PipeRunMode.LIVE,
)
tracer.register_batch_item_extraction(
@@ -315,6 +324,7 @@ class TestGraphTracerEventEmission:
event_log=event_log,
workflow_id=self.WORKFLOW_ID,
pipeline_run_id=self.PIPELINE_RUN_ID,
+ run_mode=PipeRunMode.LIVE,
)
tracer.register_batch_aggregation(
@@ -344,6 +354,7 @@ class TestGraphTracerEventEmission:
event_log=event_log,
workflow_id=self.WORKFLOW_ID,
pipeline_run_id=self.PIPELINE_RUN_ID,
+ run_mode=PipeRunMode.LIVE,
)
started_at = datetime(2025, 6, 15, 10, 0, 0, tzinfo=UTC)
@@ -406,10 +417,7 @@ class TestGraphTracerEventEmission:
def test_no_event_log_works_as_before(self) -> None:
"""GraphTracer without event_log (direct mode) works exactly as before."""
tracer = GraphTracer()
- context = tracer.setup(
- graph_id="direct-mode-test",
- data_inclusion=make_defaulted_data_inclusion_config(),
- )
+ context = tracer.setup(graph_id="direct-mode-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)
started_at = datetime(2025, 6, 15, 10, 0, 0, tzinfo=UTC)
node_id, _child_ctx = tracer.on_pipe_start(
@@ -442,6 +450,7 @@ class TestGraphTracerEventEmission:
event_log=event_log,
workflow_id="wf_run_123",
pipeline_run_id=self.PIPELINE_RUN_ID,
+ run_mode=PipeRunMode.LIVE,
)
started_at = datetime(2025, 6, 15, 10, 0, 0, tzinfo=UTC)
@@ -466,6 +475,7 @@ class TestGraphTracerEventEmission:
event_log=event_log,
workflow_id="wf_run_456",
pipeline_run_id=self.PIPELINE_RUN_ID,
+ run_mode=PipeRunMode.LIVE,
)
started_at = datetime(2025, 6, 15, 10, 0, 0, tzinfo=UTC)
@@ -505,6 +515,7 @@ class TestGraphTracerEventEmission:
event_log=event_log,
workflow_id=self.WORKFLOW_ID,
pipeline_run_id=self.PIPELINE_RUN_ID,
+ run_mode=PipeRunMode.LIVE,
)
started_at = datetime(2025, 6, 15, 10, 0, 0, tzinfo=UTC)
@@ -529,10 +540,7 @@ class TestGraphTracerEventEmission:
def test_direct_mode_node_id_unchanged(self) -> None:
"""In direct mode (no workflow_id), node ID format is unchanged."""
tracer = GraphTracer()
- context = tracer.setup(
- graph_id="direct-graph",
- data_inclusion=make_defaulted_data_inclusion_config(),
- )
+ context = tracer.setup(graph_id="direct-graph", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)
started_at = datetime(2025, 6, 15, 10, 0, 0, tzinfo=UTC)
node_id, _ = tracer.on_pipe_start(
diff --git a/tests/unit/pipelex/graph/test_graph_tracer_manager.py b/tests/unit/pipelex/graph/test_graph_tracer_manager.py
index 3ce01ed08..f71600a38 100644
--- a/tests/unit/pipelex/graph/test_graph_tracer_manager.py
+++ b/tests/unit/pipelex/graph/test_graph_tracer_manager.py
@@ -6,6 +6,7 @@ from pytest_mock import MockerFixture
from pipelex.graph.graph_tracer import GraphTracer
from pipelex.graph.graph_tracer_manager import GraphTracerManager
from pipelex.system.data_inclusion_config import DataInclusionConfig
+from pipelex.system.pipe_run_mode import PipeRunMode
def _make_data_inclusion() -> DataInclusionConfig:
@@ -22,7 +23,7 @@ class TestGraphTracerManagerOpenTracer:
GraphTracerManager.clear_instance()
manager = GraphTracerManager.get_or_create_instance()
try:
- trace_context = manager.open_tracer(graph_id="lifecycle_graph", data_inclusion=_make_data_inclusion())
+ trace_context = manager.open_tracer(graph_id="lifecycle_graph", data_inclusion=_make_data_inclusion(), run_mode=PipeRunMode.LIVE)
assert trace_context.graph_id == "lifecycle_graph"
assert manager.get_tracer("lifecycle_graph") is not None
@@ -39,7 +40,7 @@ class TestGraphTracerManagerOpenTracer:
mocker.patch.object(GraphTracer, "setup", side_effect=ValueError("setup blew up"))
with pytest.raises(ValueError, match="setup blew up"):
- manager.open_tracer(graph_id="leak_check_graph", data_inclusion=_make_data_inclusion())
+ manager.open_tracer(graph_id="leak_check_graph", data_inclusion=_make_data_inclusion(), run_mode=PipeRunMode.LIVE)
assert manager.get_tracer("leak_check_graph") is None
finally:
diff --git a/tests/unit/pipelex/graph/test_graph_tracer_manager_stale_key_healing.py b/tests/unit/pipelex/graph/test_graph_tracer_manager_stale_key_healing.py
index 65f83ebc4..ea3c130c2 100644
--- a/tests/unit/pipelex/graph/test_graph_tracer_manager_stale_key_healing.py
+++ b/tests/unit/pipelex/graph/test_graph_tracer_manager_stale_key_healing.py
@@ -15,6 +15,7 @@ from pytest_mock import MockerFixture
from pipelex.graph.graph_tracer_manager import GraphTracerManager
from pipelex.system.data_inclusion_config import DataInclusionConfig
+from pipelex.system.pipe_run_mode import PipeRunMode
_DATA_INCLUSION_OFF = DataInclusionConfig(
pipe_and_concept_registry=False,
@@ -48,6 +49,7 @@ class TestGraphTracerManagerStaleKeyHealing:
tracer_key=_TRACER_KEY,
emit_graph_events=False,
emit_usage_events=True,
+ run_mode=PipeRunMode.LIVE,
)
def test_open_tracer_replaces_stale_tracer_under_same_key(self, caplog: pytest.LogCaptureFixture) -> None:
diff --git a/tests/unit/pipelex/graph/test_graph_tracer_node_metadata.py b/tests/unit/pipelex/graph/test_graph_tracer_node_metadata.py
index 4343cf242..91ab0948c 100644
--- a/tests/unit/pipelex/graph/test_graph_tracer_node_metadata.py
+++ b/tests/unit/pipelex/graph/test_graph_tracer_node_metadata.py
@@ -4,13 +4,14 @@ from datetime import UTC, datetime, timedelta
from pipelex.graph.graph_tracer import GraphTracer
from pipelex.graph.graphspec import NodeKind
+from pipelex.system.pipe_run_mode import PipeRunMode
from tests.unit.pipelex.graph.conftest import make_defaulted_data_inclusion_config
class TestGraphTracerNodeMetadata:
def test_on_pipe_start_persists_description_and_domain_code(self) -> None:
tracer = GraphTracer()
- context = tracer.setup(graph_id="meta-test", data_inclusion=make_defaulted_data_inclusion_config())
+ context = tracer.setup(graph_id="meta-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)
started_at = datetime.now(UTC)
node_id, _child = tracer.on_pipe_start(
@@ -38,7 +39,7 @@ class TestGraphTracerNodeMetadata:
def test_on_pipe_start_defaults_metadata_to_none(self) -> None:
tracer = GraphTracer()
- context = tracer.setup(graph_id="meta-default-test", data_inclusion=make_defaulted_data_inclusion_config())
+ context = tracer.setup(graph_id="meta-default-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)
started_at = datetime.now(UTC)
node_id, _child = tracer.on_pipe_start(
diff --git a/tests/unit/pipelex/graph/test_graph_tracer_noop.py b/tests/unit/pipelex/graph/test_graph_tracer_noop.py
index 803f62e66..5df846054 100644
--- a/tests/unit/pipelex/graph/test_graph_tracer_noop.py
+++ b/tests/unit/pipelex/graph/test_graph_tracer_noop.py
@@ -4,6 +4,7 @@ from datetime import UTC, datetime, timedelta
from pipelex.graph.graph_tracer_protocol import GraphTracerNoOp
from pipelex.graph.graphspec import EdgeKind, NodeKind
+from pipelex.system.pipe_run_mode import PipeRunMode
from tests.unit.pipelex.graph.conftest import make_defaulted_data_inclusion_config
@@ -13,14 +14,14 @@ class TestGraphTracerNoOp:
def test_noop_returns_context(self) -> None:
"""Test that no-op tracer still returns valid context."""
tracer = GraphTracerNoOp()
- context = tracer.setup(graph_id="noop-test", data_inclusion=make_defaulted_data_inclusion_config())
+ context = tracer.setup(graph_id="noop-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)
assert context.graph_id == "noop-test"
def test_noop_teardown_returns_none(self) -> None:
"""Test that no-op tracer returns None on teardown."""
tracer = GraphTracerNoOp()
- tracer.setup(graph_id="noop-test", data_inclusion=make_defaulted_data_inclusion_config())
+ tracer.setup(graph_id="noop-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)
result = tracer.teardown()
assert result is None
@@ -28,7 +29,7 @@ class TestGraphTracerNoOp:
def test_noop_pipe_lifecycle(self) -> None:
"""Test that no-op tracer handles pipe lifecycle without errors."""
tracer = GraphTracerNoOp()
- context = tracer.setup(graph_id="noop-test", data_inclusion=make_defaulted_data_inclusion_config())
+ context = tracer.setup(graph_id="noop-test", data_inclusion=make_defaulted_data_inclusion_config(), run_mode=PipeRunMode.LIVE)
started_at = datetime.now(UTC)
node_id, child_ctx = tracer.on_pipe_start(
diff --git a/tests/unit/pipelex/graph/test_trace_context_emit_flags.py b/tests/unit/pipelex/graph/test_trace_context_emit_flags.py
index df472f6db..c64110895 100644
--- a/tests/unit/pipelex/graph/test_trace_context_emit_flags.py
+++ b/tests/unit/pipelex/graph/test_trace_context_emit_flags.py
@@ -6,6 +6,7 @@ carried — so a costs-only or graph-only run keeps its gating as nested pipes s
"""
from pipelex.system.data_inclusion_config import DataInclusionConfig
+from pipelex.system.pipe_run_mode import PipeRunMode
from pipelex.system.trace_context import TraceContext
DATA_INCLUSION_OFF = DataInclusionConfig(
@@ -18,7 +19,7 @@ DATA_INCLUSION_OFF = DataInclusionConfig(
class TestTraceContextEmitFlags:
def test_defaults_emit_both_streams(self) -> None:
"""A TraceContext built without explicit flags emits both streams (legacy behavior)."""
- context = TraceContext(graph_id="run_1", data_inclusion=DATA_INCLUSION_OFF)
+ context = TraceContext(graph_id="run_1", data_inclusion=DATA_INCLUSION_OFF, run_mode=PipeRunMode.LIVE)
assert context.emit_graph_events is True
assert context.emit_usage_events is True
assert context.describe_pipe_io is True
@@ -26,10 +27,7 @@ class TestTraceContextEmitFlags:
def test_copy_for_child_preserves_costs_only_flags(self) -> None:
"""costs-only flags survive into a child context (graph off, costs on)."""
parent = TraceContext(
- graph_id="run_1",
- data_inclusion=DATA_INCLUSION_OFF,
- emit_graph_events=False,
- emit_usage_events=True,
+ graph_id="run_1", data_inclusion=DATA_INCLUSION_OFF, emit_graph_events=False, emit_usage_events=True, run_mode=PipeRunMode.LIVE
)
child = parent.copy_for_child(child_node_id="run_1:node_0", next_sequence=1)
assert child.parent_node_id == "run_1:node_0"
@@ -40,10 +38,7 @@ class TestTraceContextEmitFlags:
def test_copy_for_child_preserves_graph_only_flags(self) -> None:
"""graph-only flags survive into a child context (graph on, costs off)."""
parent = TraceContext(
- graph_id="run_1",
- data_inclusion=DATA_INCLUSION_OFF,
- emit_graph_events=True,
- emit_usage_events=False,
+ graph_id="run_1", data_inclusion=DATA_INCLUSION_OFF, emit_graph_events=True, emit_usage_events=False, run_mode=PipeRunMode.LIVE
)
child = parent.copy_for_child(child_node_id="run_1:node_0", next_sequence=2)
assert child.emit_graph_events is True
@@ -51,6 +46,6 @@ class TestTraceContextEmitFlags:
def test_copy_for_child_preserves_describe_pipe_io_off(self) -> None:
"""A run that will write no graphspec keeps its I/O description off in every child context."""
- parent = TraceContext(graph_id="run_1", data_inclusion=DATA_INCLUSION_OFF, describe_pipe_io=False)
+ parent = TraceContext(graph_id="run_1", data_inclusion=DATA_INCLUSION_OFF, describe_pipe_io=False, run_mode=PipeRunMode.LIVE)
child = parent.copy_for_child(child_node_id="run_1:node_0", next_sequence=1)
assert child.describe_pipe_io is False
diff --git a/tests/unit/pipelex/kernel/test_pipelex_kernel_run_state.py b/tests/unit/pipelex/kernel/test_pipelex_kernel_run_state.py
index 27cd2e33b..b17645248 100644
--- a/tests/unit/pipelex/kernel/test_pipelex_kernel_run_state.py
+++ b/tests/unit/pipelex/kernel/test_pipelex_kernel_run_state.py
@@ -20,6 +20,7 @@ from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
from pipelex.core.stuffs.exceptions import StuffContentTypeError
from pipelex.core.stuffs.list_content import ListContent
from pipelex.core.stuffs.text_content import TextContent
+from pipelex.kernel.exceptions import TraceContextRunModeError
from pipelex.kernel.memory_ops import (
extract_main_content,
extract_main_content_as_list,
@@ -39,7 +40,7 @@ if TYPE_CHECKING:
_GRAPH_ID = "kernel-run-state"
-def _trace_context() -> TraceContext:
+def _trace_context(*, run_mode: PipeRunMode = PipeRunMode.DRY) -> TraceContext:
return TraceContext(
graph_id=_GRAPH_ID,
data_inclusion=DataInclusionConfig(
@@ -49,6 +50,7 @@ def _trace_context() -> TraceContext:
),
emit_graph_events=False,
emit_usage_events=True,
+ run_mode=run_mode,
)
@@ -124,6 +126,51 @@ class TestPipelexKernelRunState:
pipeline_run_id="plr-elsewhere",
)
+ @pytest.mark.parametrize(
+ ("is_dry_run_forced", "requested_run_mode", "context_run_mode"),
+ [
+ (True, PipeRunMode.LIVE, PipeRunMode.LIVE),
+ (False, PipeRunMode.DRY, PipeRunMode.LIVE),
+ (False, PipeRunMode.LIVE, PipeRunMode.DRY),
+ ],
+ ids=["keyless_boot_forces_the_live_request_dry", "dry_request_live_context", "live_request_dry_context"],
+ )
+ def test_a_trace_context_stating_another_mode_than_the_run_takes_is_refused(
+ self, mocker: MockerFixture, is_dry_run_forced: bool, requested_run_mode: PipeRunMode, context_run_mode: PipeRunMode
+ ) -> None:
+ """Every usage event states the context's mode, so a context saying live for a run whose steps run dry is refused.
+
+ The keyless arm is the one a caller cannot see in its own arguments: it requests a live run and opens a live
+ trace, and the boot runs the steps dry.
+ """
+ mocker.patch("pipelex.runtime_hub.is_dry_run_forced", return_value=is_dry_run_forced)
+
+ with pytest.raises(TraceContextRunModeError, match="trace context stating a"):
+ PipelexKernel.make(
+ storage_scope="test/scope",
+ read_scope=None,
+ run_mode=requested_run_mode,
+ user_id="test-user",
+ trace_context=_trace_context(run_mode=context_run_mode),
+ )
+
+ def test_on_a_keyless_boot_a_dry_trace_context_carries_a_live_request(self, mocker: MockerFixture) -> None:
+ """A caller that opens its trace with the mode the boot gives the run gets a dry run whose every step states dry."""
+ mocker.patch("pipelex.runtime_hub.is_dry_run_forced", return_value=True)
+
+ kernel = PipelexKernel.make(
+ storage_scope="test/scope",
+ read_scope=None,
+ run_mode=PipeRunMode.LIVE,
+ user_id="test-user",
+ trace_context=_trace_context(run_mode=PipeRunMode.DRY),
+ )
+
+ assert kernel.cogt_run_params.run_mode.is_dry
+ step_trace_context = kernel.make_step_metadata().trace_context
+ assert step_trace_context is not None
+ assert step_trace_context.run_mode.is_dry
+
def test_an_empty_pipeline_run_id_is_refused(self) -> None:
"""An empty id is a caller's bug, and minting one in its place would hide it from a replaying host."""
with pytest.raises(ValueError, match="empty pipeline_run_id"):
diff --git a/tests/unit/pipelex/pipe_run/test_pipe_run_usage_stats.py b/tests/unit/pipelex/pipe_run/test_pipe_run_usage_stats.py
index b1320d2cf..3ef809b7e 100644
--- a/tests/unit/pipelex/pipe_run/test_pipe_run_usage_stats.py
+++ b/tests/unit/pipelex/pipe_run/test_pipe_run_usage_stats.py
@@ -35,6 +35,7 @@ def _job(*, mocker: MockerFixture, is_traced: bool) -> Any:
data_inclusion=DataInclusionConfig(pipe_and_concept_registry=False, stuff_json_content=False, error_stack_traces=False),
emit_graph_events=False,
emit_usage_events=True,
+ run_mode=PipeRunMode.LIVE,
)
job.job_metadata = JobMetadata(
run_metadata=RunMetadata(user_id="pytest", pipeline_run_id=USAGE_STATS_RUN_ID, storage_scope="test/scope", read_scope=None),
diff --git a/tests/unit/pipelex/pipe_run/test_tracing_assembly.py b/tests/unit/pipelex/pipe_run/test_tracing_assembly.py
index 347c00a86..948622b46 100644
--- a/tests/unit/pipelex/pipe_run/test_tracing_assembly.py
+++ b/tests/unit/pipelex/pipe_run/test_tracing_assembly.py
@@ -11,7 +11,6 @@ from pipelex.cogt.llm.llm_report import LLMTokensUsage
from pipelex.cogt.usage.cost_category import CostCategory
from pipelex.cogt.usage.token_category import TokenCategory
from pipelex.core.pipes.pipe_output import PipeOutput
-from pipelex.graph.graphspec import GraphSpecMode
from pipelex.pipe_run.tracing_assembly import TracingAssembly, assemble_tracing, assemble_tracing_on_output
from pipelex.system.exceptions import MissingDependencyError
from pipelex.system.job_metadata import JobMetadata, RunMetadata
@@ -23,7 +22,7 @@ from pipelex.tracing.trace_events import UsageReportEvent
_MODULE = "pipelex.pipe_run.tracing_assembly"
-def _make_usage_event(pipeline_run_id: str, node_id: str) -> UsageReportEvent:
+def _make_usage_event(pipeline_run_id: str, node_id: str, run_mode: PipeRunMode = PipeRunMode.LIVE) -> UsageReportEvent:
"""A UsageReportEvent carrying a real LLMTokensUsage (so the aggregator returns it)."""
metadata = JobMetadata(
run_metadata=RunMetadata(storage_scope="test/scope", read_scope=None, user_id="tracing-assembly-test", pipeline_run_id=pipeline_run_id)
@@ -40,6 +39,7 @@ def _make_usage_event(pipeline_run_id: str, node_id: str) -> UsageReportEvent:
workflow_id="direct",
timestamp=datetime.now(UTC),
sequence=0,
+ mode=run_mode,
node_id=node_id,
tokens_usage=tokens_usage,
)
@@ -116,13 +116,41 @@ class TestTracingAssembly:
assemble_usage=False,
domain_code="dom",
main_pipe_code="main",
- run_mode=PipeRunMode.DRY,
)
assert result.graph_spec is sentinel_graph
assert result.tokens_usages is None
assemble.assert_called_once()
- assert assemble.call_args.kwargs["mode"] == GraphSpecMode.DRY
+
+ @pytest.mark.parametrize("run_mode", [PipeRunMode.LIVE, PipeRunMode.DRY], ids=["live", "dry"])
+ def test_the_graph_states_the_mode_its_events_state(self, mocker: MockerFixture, run_mode: PipeRunMode) -> None:
+ """No caller passes the graph's mode: the assembled graph takes the one its events state."""
+ self._enable_tracing(mocker)
+ event_log = mocker.MagicMock()
+ event_log.read_events = mocker.MagicMock(return_value=[_make_usage_event("plr-mode", "plr-mode:node_0", run_mode=run_mode)])
+ mocker.patch(f"{_MODULE}.make_event_log", return_value=event_log)
+
+ result = assemble_tracing(pipeline_run_id="plr-mode", assemble_graph=True, assemble_usage=False)
+
+ assert result.graph_assembly_error is None
+ assert result.graph_spec is not None
+ assert result.graph_spec.meta["mode"] == run_mode.graphspec_mode
+
+ def test_events_that_mix_modes_get_no_graph_and_keep_their_usage(self, mocker: MockerFixture) -> None:
+ """Events of a dry run and a live run under one pipeline run id describe no one run: no graph, and the reason why."""
+ self._enable_tracing(mocker)
+ live_event = _make_usage_event("plr-mixed-modes", "plr-mixed-modes:node_0", run_mode=PipeRunMode.LIVE)
+ dry_event = _make_usage_event("plr-mixed-modes", "plr-mixed-modes:node_1", run_mode=PipeRunMode.DRY)
+ event_log = mocker.MagicMock()
+ event_log.read_events = mocker.MagicMock(return_value=[live_event, dry_event])
+ mocker.patch(f"{_MODULE}.make_event_log", return_value=event_log)
+
+ result = assemble_tracing(pipeline_run_id="plr-mixed-modes", assemble_graph=True, assemble_usage=True)
+
+ assert result.graph_spec is None
+ assert result.graph_assembly_error is not None
+ assert "mix run modes (dry, live)" in result.graph_assembly_error
+ assert result.tokens_usages == [live_event.tokens_usage, dry_event.tokens_usage]
def test_graph_and_usage_both_assembled(self, mocker: MockerFixture) -> None:
self._enable_tracing(mocker)
diff --git a/tests/unit/pipelex/reporting/test_emit_runner_fallback.py b/tests/unit/pipelex/reporting/test_emit_runner_fallback.py
index 155814924..d8a8a77df 100644
--- a/tests/unit/pipelex/reporting/test_emit_runner_fallback.py
+++ b/tests/unit/pipelex/reporting/test_emit_runner_fallback.py
@@ -30,6 +30,7 @@ from pipelex.system.configuration.configs import NdjsonTracingConfig, TracingBac
from pipelex.system.data_inclusion_config import DataInclusionConfig
from pipelex.system.exceptions import MissingDependencyError
from pipelex.system.job_metadata import JobMetadata, RunMetadata, UnitJobId
+from pipelex.system.pipe_run_mode import PipeRunMode
from pipelex.system.trace_context import TraceContext
from pipelex.tracing import activity_event_log
from pipelex.tracing.activity_event_log import ActivityEventLogCache
@@ -81,6 +82,7 @@ def _make_trace_context(
graph_id: str,
parent_node_id: str | None = "g:node_0",
tracer_key: str | None = None,
+ run_mode: PipeRunMode = PipeRunMode.LIVE,
) -> TraceContext:
return TraceContext(
graph_id=graph_id,
@@ -88,6 +90,7 @@ def _make_trace_context(
parent_node_id=parent_node_id,
node_sequence=0,
data_inclusion=DATA_INCLUSION_OFF,
+ run_mode=run_mode,
)
@@ -139,6 +142,22 @@ class TestEmitRunnerFallback:
assert usage_events[0].node_id == "g:node_2"
assert usage_events[0].writer_id.startswith("act_")
+ def test_fallback_states_the_run_mode_of_the_trace_context(
+ self,
+ tmp_path: Path,
+ mocker: MockerFixture,
+ ) -> None:
+ """An activity's usage event states the mode its trace context carries, since it holds nothing else of the run."""
+ _enable_ndjson_tracing(mocker, tmp_path)
+
+ manager = ReportingManager()
+ manager.setup()
+ trace_context = _make_trace_context(graph_id="run_dry", tracer_key="wf_dry", run_mode=PipeRunMode.DRY)
+ manager.report_inference_job(_make_llm_job("run_dry", trace_context=trace_context))
+
+ usage_events = [evt for evt in NdjsonEventLog(traces_dir=str(tmp_path)).read_events("run_dry") if isinstance(evt, UsageReportEvent)]
+ assert [evt.mode for evt in usage_events] == [PipeRunMode.DRY]
+
def test_runner_fallback_uses_tracer_key_when_set(
self,
tmp_path: Path,
diff --git a/tests/unit/pipelex/reporting/test_emit_runner_fallback_non_llm.py b/tests/unit/pipelex/reporting/test_emit_runner_fallback_non_llm.py
index 0cf16d92c..15098d35a 100644
--- a/tests/unit/pipelex/reporting/test_emit_runner_fallback_non_llm.py
+++ b/tests/unit/pipelex/reporting/test_emit_runner_fallback_non_llm.py
@@ -37,6 +37,7 @@ from pipelex.reporting.reporting_manager import ReportingManager
from pipelex.system.configuration.configs import NdjsonTracingConfig, TracingBackend
from pipelex.system.data_inclusion_config import DataInclusionConfig
from pipelex.system.job_metadata import JobMetadata, RunMetadata
+from pipelex.system.pipe_run_mode import PipeRunMode
from pipelex.system.trace_context import TraceContext
from pipelex.tools.misc.image_utils import ImageFormat
from pipelex.tracing.activity_event_log import ActivityEventLogCache
@@ -72,6 +73,7 @@ def _trace_context(run_id: str, tracer_key: str) -> TraceContext:
parent_node_id="g:node_0",
node_sequence=0,
data_inclusion=_DATA_INCLUSION_OFF,
+ run_mode=PipeRunMode.LIVE,
)
diff --git a/tests/unit/pipelex/reporting/test_emit_usage_event_gating.py b/tests/unit/pipelex/reporting/test_emit_usage_event_gating.py
index 294339a6d..482ebed37 100644
--- a/tests/unit/pipelex/reporting/test_emit_usage_event_gating.py
+++ b/tests/unit/pipelex/reporting/test_emit_usage_event_gating.py
@@ -23,6 +23,7 @@ from pipelex.reporting.reporting_manager import ReportingManager
from pipelex.system.configuration.configs import NdjsonTracingConfig, TracingBackend
from pipelex.system.data_inclusion_config import DataInclusionConfig
from pipelex.system.job_metadata import JobMetadata, RunMetadata, UnitJobId
+from pipelex.system.pipe_run_mode import PipeRunMode
from pipelex.system.trace_context import TraceContext
from pipelex.tracing.activity_event_log import ActivityEventLogCache
from pipelex.tracing.ndjson_event_log import NdjsonEventLog
@@ -69,6 +70,7 @@ def _make_trace_context(graph_id: str, *, emit_usage_events: bool, emit_graph_ev
data_inclusion=DATA_INCLUSION_OFF,
emit_graph_events=emit_graph_events,
emit_usage_events=emit_usage_events,
+ run_mode=PipeRunMode.LIVE,
)
diff --git a/tests/unit/pipelex/reporting/test_reporting_event_emission.py b/tests/unit/pipelex/reporting/test_reporting_event_emission.py
index 56fb4b76a..504411b4e 100644
--- a/tests/unit/pipelex/reporting/test_reporting_event_emission.py
+++ b/tests/unit/pipelex/reporting/test_reporting_event_emission.py
@@ -21,6 +21,7 @@ from pipelex.config import get_config
from pipelex.reporting.reporting_manager import ReportingManager
from pipelex.system.data_inclusion_config import DataInclusionConfig
from pipelex.system.job_metadata import JobMetadata, RunMetadata, UnitJobId
+from pipelex.system.pipe_run_mode import PipeRunMode
from pipelex.system.trace_context import TraceContext
from pipelex.tracing.in_memory_event_log import InMemoryEventLog
from pipelex.tracing.trace_events import UsageReportEvent
@@ -66,6 +67,7 @@ def _make_trace_context(
parent_node_id: str | None = None,
tracer_key: str | None = None,
node_sequence: int = 0,
+ run_mode: PipeRunMode = PipeRunMode.LIVE,
) -> TraceContext:
"""Create a TraceContext for testing with lookup_key resolution."""
return TraceContext(
@@ -74,6 +76,7 @@ def _make_trace_context(
parent_node_id=parent_node_id,
node_sequence=node_sequence,
data_inclusion=DATA_INCLUSION_OFF,
+ run_mode=run_mode,
)
@@ -118,6 +121,17 @@ class TestReportingEventEmission:
assert usage_events[0].tokens_usage.nb_tokens_by_category[TokenCategory.INPUT] == 100
assert usage_events[0].node_id == "unknown" # No trace_context parent set
+ @pytest.mark.parametrize("run_mode", [PipeRunMode.LIVE, PipeRunMode.DRY], ids=["live", "dry"])
+ def test_usage_event_states_the_run_mode(self, run_mode: PipeRunMode) -> None:
+ """A usage event states the mode its trace context carries, the one fact of the run the reporting side holds."""
+ manager, event_log = self._make_reporting_manager_with_event_log()
+
+ trace_context = _make_trace_context(graph_id=self.PIPELINE_RUN_ID, run_mode=run_mode)
+ manager.report_inference_job(_make_test_llm_job(self.PIPELINE_RUN_ID, trace_context=trace_context))
+
+ usage_events = [evt for evt in event_log.read_events(self.PIPELINE_RUN_ID) if isinstance(evt, UsageReportEvent)]
+ assert [evt.mode for evt in usage_events] == [run_mode]
+
def test_report_inference_job_captures_node_id_from_trace_context(self) -> None:
"""UsageReportEvent captures the parent node_id from graph context."""
manager, event_log = self._make_reporting_manager_with_event_log()
diff --git a/tests/unit/pipelex/tracing/conftest.py b/tests/unit/pipelex/tracing/conftest.py
index 48aef1fe8..1169e8392 100644
--- a/tests/unit/pipelex/tracing/conftest.py
+++ b/tests/unit/pipelex/tracing/conftest.py
@@ -3,6 +3,7 @@
from datetime import UTC, datetime
from pipelex.graph.graphspec import EdgeKind, NodeKind
+from pipelex.system.pipe_run_mode import PipeRunMode
from pipelex.tracing.trace_events import EdgeEvent, PipeStartEvent
@@ -17,6 +18,7 @@ def make_trace_event(
workflow_id=workflow_id,
timestamp=datetime(2025, 6, 15, 10, 30, 0, tzinfo=UTC),
sequence=sequence,
+ mode=PipeRunMode.LIVE,
node_id=f"graph_1:{workflow_id}:node_{sequence}",
pipe_code="test_pipe",
pipe_type="PipeLLMGenText",
@@ -35,6 +37,7 @@ def make_edge_event(
workflow_id=workflow_id,
timestamp=datetime(2025, 6, 15, 10, 30, 0, tzinfo=UTC),
sequence=sequence,
+ mode=PipeRunMode.LIVE,
edge_id=f"graph_1:{workflow_id}:edge_{sequence}",
source_node_id=f"graph_1:{workflow_id}:node_0",
target_node_id=f"graph_1:{workflow_id}:node_1",
diff --git a/tests/unit/pipelex/tracing/test_assembler_equivalence.py b/tests/unit/pipelex/tracing/test_assembler_equivalence.py
index c243e25d5..9ccdd19c9 100644
--- a/tests/unit/pipelex/tracing/test_assembler_equivalence.py
+++ b/tests/unit/pipelex/tracing/test_assembler_equivalence.py
@@ -13,6 +13,7 @@ from datetime import UTC, datetime, timedelta
from typing import Any, Callable
import pytest
+from pytest_mock import MockerFixture
from pipelex.cogt.llm.llm_report import LLMTokensUsage
from pipelex.cogt.usage.cost_category import CostCategory
@@ -21,7 +22,9 @@ from pipelex.graph.condition_output_merge import ConditionOutputMerge, Condition
from pipelex.graph.graph_tracer import GraphTracer
from pipelex.graph.graphspec import EdgeKind, EdgeSpec, GraphSpec, IOSpec, NodeKind, NodeSpec
from pipelex.system.job_metadata import JobCategory, JobMetadata, RunMetadata, UnitJobId
+from pipelex.system.pipe_run_mode import PipeRunMode
from pipelex.system.trace_context import TraceContext
+from pipelex.tracing import graphspec_assembler as graphspec_assembler_module
from pipelex.tracing.graphspec_assembler import GraphSpecAssembler
from pipelex.tracing.in_memory_event_log import InMemoryEventLog
from pipelex.tracing.trace_events import PipeStartEvent, UsageReportEvent
@@ -129,7 +132,7 @@ def _run_both_paths(scenario: ScenarioFn) -> tuple[GraphSpec, GraphSpec]:
# Path A: direct mode (no event log)
tracer_direct = GraphTracer()
- ctx_direct = tracer_direct.setup(graph_id=_GRAPH_ID, data_inclusion=data_inclusion)
+ ctx_direct = tracer_direct.setup(graph_id=_GRAPH_ID, data_inclusion=data_inclusion, run_mode=PipeRunMode.LIVE)
scenario(tracer_direct, ctx_direct)
direct_spec = tracer_direct.teardown()
assert direct_spec is not None
@@ -143,6 +146,7 @@ def _run_both_paths(scenario: ScenarioFn) -> tuple[GraphSpec, GraphSpec]:
event_log=event_log,
workflow_id=_WORKFLOW_ID,
pipeline_run_id=_PIPELINE_RUN_ID,
+ run_mode=PipeRunMode.LIVE,
)
scenario(tracer_event, ctx_event)
# Don't call teardown on the event-log tracer — we want to assemble from events
@@ -474,6 +478,7 @@ def _usage_events_for(*, assembled_child_node_id: str, event_log: InMemoryEventL
writer_id=event_log.writer_id,
timestamp=_T0 + timedelta(seconds=2),
sequence=event_log.next_sequence(),
+ mode=PipeRunMode.LIVE,
node_id=assembled_child_node_id,
tokens_usage=LLMTokensUsage(
job_metadata=JobMetadata(
@@ -492,7 +497,7 @@ def _usage_events_for(*, assembled_child_node_id: str, event_log: InMemoryEventL
def _scenario_condition_selected_outcome(tracer: GraphTracer, context: TraceContext) -> None:
- """Condition pipe with SELECTED_OUTCOME edge."""
+ """Condition pipe that resolves to a key, then runs that outcome's pipe: both builders derive the SELECTED_OUTCOME edge."""
started_at = _T0
cond_id, cond_ctx = tracer.on_pipe_start(
@@ -502,6 +507,7 @@ def _scenario_condition_selected_outcome(tracer: GraphTracer, context: TraceCont
node_kind=NodeKind.CONTROLLER,
started_at=started_at,
)
+ tracer.register_condition_resolution(condition_node_id=cond_id, rendered_value="true", selected_outcome="true")
outcome_id, _ = tracer.on_pipe_start(
trace_context=cond_ctx,
@@ -511,13 +517,6 @@ def _scenario_condition_selected_outcome(tracer: GraphTracer, context: TraceCont
started_at=started_at + timedelta(seconds=1),
)
- tracer.add_edge(
- source_node_id=cond_id,
- target_node_id=outcome_id,
- edge_kind=EdgeKind.SELECTED_OUTCOME,
- label="true",
- )
-
tracer.on_pipe_end_success(
node_id=outcome_id,
ended_at=started_at + timedelta(seconds=2),
@@ -528,6 +527,72 @@ def _scenario_condition_selected_outcome(tracer: GraphTracer, context: TraceCont
)
+def _scenario_decisions_at_decision_time(tracer: GraphTracer, context: TraceContext) -> None:
+ """A sequence whose steps record each decision as it is made: a condition's resolution, then a batch's fan-out.
+
+ The condition takes its default outcome, so its SELECTED_OUTCOME edge is unlabelled; the batch states its width
+ before its first item starts.
+ """
+ started_at = _T0
+
+ seq_id, seq_ctx = tracer.on_pipe_start(
+ trace_context=context,
+ pipe_code="flow",
+ pipe_type="PipeSequence",
+ node_kind=NodeKind.CONTROLLER,
+ started_at=started_at,
+ step_count=2,
+ )
+
+ cond_id, cond_ctx = tracer.on_pipe_start(
+ trace_context=seq_ctx,
+ pipe_code="route",
+ pipe_type="PipeCondition",
+ node_kind=NodeKind.CONTROLLER,
+ started_at=started_at + timedelta(seconds=1),
+ )
+ tracer.register_condition_resolution(condition_node_id=cond_id, rendered_value="unmatched", selected_outcome=None)
+ outcome_id, _ = tracer.on_pipe_start(
+ trace_context=cond_ctx,
+ pipe_code="fallback",
+ pipe_type="PipeCompose",
+ node_kind=NodeKind.OPERATOR,
+ started_at=started_at + timedelta(seconds=2),
+ )
+ tracer.on_pipe_end_success(
+ node_id=outcome_id, ended_at=started_at + timedelta(seconds=3), output_spec=IOSpec(name="routed", digest="digest_routed")
+ )
+ tracer.on_pipe_end_success(node_id=cond_id, ended_at=started_at + timedelta(seconds=4))
+
+ batch_id, batch_ctx = tracer.on_pipe_start(
+ trace_context=seq_ctx,
+ pipe_code="per_item",
+ pipe_type="PipeBatch",
+ node_kind=NodeKind.CONTROLLER,
+ started_at=started_at + timedelta(seconds=5),
+ input_specs=[IOSpec(name="items", digest="digest_items", multiplicity=True)],
+ )
+ tracer.register_batch_fan_out(
+ batch_node_id=batch_id, item_count=1, branch_pipe_code="process", branch_pipe_ref="flows.process", branch_pipe_type="PipeCompose"
+ )
+ tracer.register_batch_item_extraction(
+ list_stuff_code="digest_items", item_stuff_code="digest_item_0", item_index=0, batch_controller_node_id=batch_id
+ )
+ item_id, _ = tracer.on_pipe_start(
+ trace_context=batch_ctx,
+ pipe_code="process",
+ pipe_type="PipeCompose",
+ node_kind=NodeKind.OPERATOR,
+ started_at=started_at + timedelta(seconds=6),
+ input_specs=[IOSpec(name="item", digest="digest_item_0")],
+ )
+ tracer.on_pipe_end_success(
+ node_id=item_id, ended_at=started_at + timedelta(seconds=7), output_spec=IOSpec(name="result", digest="digest_result_0")
+ )
+ tracer.on_pipe_end_success(node_id=batch_id, ended_at=started_at + timedelta(seconds=8))
+ tracer.on_pipe_end_success(node_id=seq_id, ended_at=started_at + timedelta(seconds=9))
+
+
def _scenario_dry_condition_merge(tracer: GraphTracer, context: TraceContext) -> None:
"""Dry-run condition whose two outcomes write different concepts into the slot a consumer reads.
@@ -660,6 +725,7 @@ class TestAssemblerEquivalence:
pytest.param(_scenario_pass_through, id="pass_through"),
pytest.param(_scenario_condition_selected_outcome, id="condition_selected_outcome"),
pytest.param(_scenario_dry_condition_merge, id="dry_condition_merge"),
+ pytest.param(_scenario_decisions_at_decision_time, id="decisions_at_decision_time"),
],
)
def test_equivalence(self, scenario_fn: ScenarioFn) -> None:
@@ -667,6 +733,98 @@ class TestAssemblerEquivalence:
direct_spec, assembled_spec = _run_both_paths(scenario_fn)
_assert_graphs_equivalent(direct_spec, assembled_spec)
+ def test_decision_events_give_both_graphs_the_choice_edge(self, mocker: MockerFixture) -> None:
+ """The decision events are read without a warning, and both graphs derive the condition's choice as its edge."""
+ warning_spy = mocker.spy(graphspec_assembler_module.log, "warning")
+ direct_spec, assembled_spec = _run_both_paths(_scenario_decisions_at_decision_time)
+
+ assert warning_spy.call_count == 0
+ for spec in (direct_spec, assembled_spec):
+ pipe_code_by_node_id = {node.node_id: node.pipe_code for node in spec.nodes}
+ selected_edges = [edge for edge in spec.edges if edge.kind.is_selected_outcome]
+ assert [(pipe_code_by_node_id[edge.source], pipe_code_by_node_id[edge.target], edge.label) for edge in selected_edges] == [
+ ("route", "fallback", None)
+ ]
+
+ def test_selected_outcome_edge_when_the_chosen_pipe_starts_under_another_tracer(self) -> None:
+ """A distributed run: the condition resolves under its workflow's tracer, and the chosen pipe starts under another.
+
+ A sub-pipe dispatched to a workflow of its own runs under a tracer that workflow opens, which keeps the
+ condition as the parent of its first node and never saw the resolution. The graph assembled from the run's
+ events still draws the SELECTED_OUTCOME edge, as the in-process graph of the same run does.
+ """
+ data_inclusion = make_defaulted_data_inclusion_config()
+ event_log = InMemoryEventLog()
+
+ parent_tracer = GraphTracer()
+ parent_ctx = parent_tracer.setup(
+ graph_id=_GRAPH_ID,
+ data_inclusion=data_inclusion,
+ event_log=event_log,
+ workflow_id="wf_parent",
+ pipeline_run_id=_PIPELINE_RUN_ID,
+ run_mode=PipeRunMode.LIVE,
+ )
+ cond_id, cond_ctx = parent_tracer.on_pipe_start(
+ trace_context=parent_ctx, pipe_code="route", pipe_type="PipeCondition", node_kind=NodeKind.CONTROLLER, started_at=_T0
+ )
+ parent_tracer.register_condition_resolution(condition_node_id=cond_id, rendered_value="urgent", selected_outcome="urgent")
+
+ # The dispatched workflow opens its own tracer, and keeps the parent node of the context it received.
+ child_tracer = GraphTracer()
+ child_ctx = child_tracer.setup(
+ graph_id=_GRAPH_ID,
+ data_inclusion=data_inclusion,
+ event_log=event_log,
+ workflow_id="wf_child",
+ pipeline_run_id=_PIPELINE_RUN_ID,
+ run_mode=cond_ctx.run_mode,
+ ).model_copy(update={"parent_node_id": cond_ctx.parent_node_id})
+ chosen_id, _ = child_tracer.on_pipe_start(
+ trace_context=child_ctx,
+ pipe_code="escalate",
+ pipe_type="PipeCompose",
+ node_kind=NodeKind.OPERATOR,
+ started_at=_T0 + timedelta(seconds=1),
+ )
+ child_tracer.on_pipe_end_success(node_id=chosen_id, ended_at=_T0 + timedelta(seconds=2))
+ child_tracer.teardown()
+ parent_tracer.on_pipe_end_success(node_id=cond_id, ended_at=_T0 + timedelta(seconds=3))
+ parent_tracer.teardown()
+ assembled_spec = GraphSpecAssembler.assemble(events=event_log.read_events(_PIPELINE_RUN_ID), graph_id=_GRAPH_ID)
+
+ # The same run, in process: one tracer sees both the resolution and the chosen pipe's start.
+ direct_tracer = GraphTracer()
+ direct_ctx = direct_tracer.setup(graph_id=_GRAPH_ID, data_inclusion=data_inclusion, run_mode=PipeRunMode.LIVE)
+ direct_cond_id, direct_cond_ctx = direct_tracer.on_pipe_start(
+ trace_context=direct_ctx, pipe_code="route", pipe_type="PipeCondition", node_kind=NodeKind.CONTROLLER, started_at=_T0
+ )
+ direct_tracer.register_condition_resolution(condition_node_id=direct_cond_id, rendered_value="urgent", selected_outcome="urgent")
+ direct_chosen_id, _ = direct_tracer.on_pipe_start(
+ trace_context=direct_cond_ctx,
+ pipe_code="escalate",
+ pipe_type="PipeCompose",
+ node_kind=NodeKind.OPERATOR,
+ started_at=_T0 + timedelta(seconds=1),
+ )
+ direct_tracer.on_pipe_end_success(node_id=direct_chosen_id, ended_at=_T0 + timedelta(seconds=2))
+ direct_tracer.on_pipe_end_success(node_id=direct_cond_id, ended_at=_T0 + timedelta(seconds=3))
+ direct_spec = direct_tracer.teardown()
+ assert direct_spec is not None
+
+ assert [(edge.source, edge.target, edge.label) for edge in assembled_spec.edges if edge.kind.is_selected_outcome] == [
+ (cond_id, chosen_id, "urgent")
+ ]
+ for spec in (direct_spec, assembled_spec):
+ pipe_code_by_node_id = {node.node_id: node.pipe_code for node in spec.nodes}
+ edges_by_codes = sorted(
+ (pipe_code_by_node_id[edge.source], pipe_code_by_node_id[edge.target], edge.kind, edge.label) for edge in spec.edges
+ )
+ assert edges_by_codes == [
+ ("route", "escalate", EdgeKind.CONTAINS, None),
+ ("route", "escalate", EdgeKind.SELECTED_OUTCOME, "urgent"),
+ ]
+
def test_multiplicity_survives_both_builders(self) -> None:
"""Agreement alone would pass if both builders dropped the marker, so pin its value too:
the batch's list io items stay plural and its items stay single, on both routes.
@@ -709,7 +867,7 @@ class TestAssemblerEquivalence:
data_inclusion = make_defaulted_data_inclusion_config()
tracer_direct = GraphTracer()
- ctx_direct = tracer_direct.setup(graph_id=_GRAPH_ID, data_inclusion=data_inclusion)
+ ctx_direct = tracer_direct.setup(graph_id=_GRAPH_ID, data_inclusion=data_inclusion, run_mode=PipeRunMode.LIVE)
_scenario_with_usage(tracer_direct, ctx_direct)
direct_spec = tracer_direct.teardown()
assert direct_spec is not None
@@ -722,6 +880,7 @@ class TestAssemblerEquivalence:
event_log=event_log,
workflow_id=_WORKFLOW_ID,
pipeline_run_id=_PIPELINE_RUN_ID,
+ run_mode=PipeRunMode.LIVE,
)
_scenario_with_usage(tracer_event, ctx_event)
tracer_event.teardown()
diff --git a/tests/unit/pipelex/tracing/test_assembler_usage_attribution.py b/tests/unit/pipelex/tracing/test_assembler_usage_attribution.py
index c5493ac54..3fa374ece 100644
--- a/tests/unit/pipelex/tracing/test_assembler_usage_attribution.py
+++ b/tests/unit/pipelex/tracing/test_assembler_usage_attribution.py
@@ -10,6 +10,7 @@ from pipelex.cogt.usage.token_category import TokenCategory
from pipelex.graph.graphspec import GraphSpec, NodeKind, NodeSpec
from pipelex.reporting.reporting_types import AnyTokensUsage
from pipelex.system.job_metadata import JobCategory, JobMetadata, RunMetadata, UnitJobId
+from pipelex.system.pipe_run_mode import PipeRunMode
from pipelex.tracing.graphspec_assembler import GraphSpecAssembler
from pipelex.tracing.trace_events import (
UNATTRIBUTED_NODE_ID,
@@ -39,6 +40,7 @@ def _base(sequence: int) -> dict[str, Any]:
"workflow_id": _WORKFLOW_ID,
"timestamp": _T0 + timedelta(seconds=sequence),
"sequence": sequence,
+ "mode": PipeRunMode.LIVE,
}
diff --git a/tests/unit/pipelex/tracing/test_dynamodb_event_log.py b/tests/unit/pipelex/tracing/test_dynamodb_event_log.py
index c266ce8d5..05122105f 100644
--- a/tests/unit/pipelex/tracing/test_dynamodb_event_log.py
+++ b/tests/unit/pipelex/tracing/test_dynamodb_event_log.py
@@ -1,4 +1,6 @@
+import json
import time
+from datetime import UTC, datetime
from typing import Any
import pytest
@@ -6,14 +8,45 @@ from botocore.exceptions import BotoCoreError, ClientError, EndpointConnectionEr
from pydantic import ValidationError
from pytest_mock import MockerFixture
+from pipelex.graph.graphspec import NodeKind
from pipelex.system.configuration.configs import DynamoDBTracingConfig, TracingConfig
+from pipelex.system.pipe_run_mode import PipeRunMode
+from pipelex.tracing.buffering_event_log import BufferingEventLog
from pipelex.tracing.dynamodb_event_log import DynamoDBEventLog
from pipelex.tracing.event_log_factory import make_event_log
from pipelex.tracing.event_log_protocol import EventLogProtocol
-from pipelex.tracing.exceptions import EventLogReadError, EventLogSetupError
+from pipelex.tracing.exceptions import EventLogReadError, EventLogSchemaMismatchError, EventLogSetupError
+from pipelex.tracing.trace_events import BatchFannedOutEvent, ConditionResolvedEvent, PipeStartEvent, TraceEvent
from tests.unit.pipelex.tracing.conftest import make_trace_event
_SECONDS_PER_DAY = 86_400
+_RUN_ID = "plr-decisions"
+_T0 = datetime(2026, 10, 13, 9, 0, 0, tzinfo=UTC)
+
+
+def _decision_events(*, mode: PipeRunMode) -> list[TraceEvent]:
+ base: dict[str, Any] = {"pipeline_run_id": _RUN_ID, "workflow_id": "wf_flow", "timestamp": _T0, "mode": mode}
+ return [
+ PipeStartEvent(
+ **base,
+ sequence=0,
+ node_id="g:wf_flow:node_0",
+ pipe_code="flow",
+ pipe_type="PipeSequence",
+ node_kind=NodeKind.CONTROLLER,
+ step_count=3,
+ ),
+ ConditionResolvedEvent(**base, sequence=1, node_id="g:wf_flow:node_1", rendered_value="unmatched", selected_outcome=None),
+ BatchFannedOutEvent(
+ **base,
+ sequence=2,
+ node_id="g:wf_flow:node_2",
+ item_count=5,
+ branch_pipe_code="process",
+ branch_pipe_ref="flows.process",
+ branch_pipe_type="PipeCompose",
+ ),
+ ]
class TestDynamoDBEventLog:
@@ -109,6 +142,33 @@ class TestDynamoDBEventLog:
with pytest.raises(EventLogReadError):
event_log.read_events("plr-paginated")
+ def test_decision_events_survive_the_buffer_and_the_table(self, mocker: MockerFixture) -> None:
+ """The Temporal path: a workflow buffers the events, an activity writes them to DynamoDB, a reader reads them back whole."""
+ event_log, mock_table = self._make_event_log_with_table(mocker)
+ buffer = BufferingEventLog()
+ events = _decision_events(mode=PipeRunMode.DRY)
+ for event in events:
+ buffer.emit(event)
+ for event in buffer.drain():
+ event_log.emit(event)
+ stored_items = [call.kwargs["Item"] for call in mock_table.put_item.call_args_list]
+ assert all(isinstance(item["ttl"], int) for item in stored_items)
+ mock_table.query.return_value = {"Items": stored_items}
+
+ assert event_log.read_events(_RUN_ID) == events
+
+ def test_an_event_stored_without_a_mode_is_refused(self, mocker: MockerFixture) -> None:
+ """A payload written before every event stated its mode is in a shape this version refuses, never read as live."""
+ event_log, mock_table = self._make_event_log_with_table(mocker)
+ payload = json.loads(_decision_events(mode=PipeRunMode.LIVE)[0].model_dump_json())
+ del payload["mode"]
+ mock_table.query.return_value = {
+ "Items": [{"PK": f"PIPELINE_RUN#{_RUN_ID}", "SK": "EVENT#wf_flow#primary#0000000000", "payload": json.dumps(payload)}]
+ }
+
+ with pytest.raises(EventLogSchemaMismatchError):
+ event_log.read_events(_RUN_ID)
+
def test_init_translates_construction_error(self, mocker: MockerFixture) -> None:
# A botocore failure while building the boto3 client (e.g. a misconfigured region) must surface as
# the domain EventLogSetupError so best-effort callers (tracing assembly) degrade rather than abort.
diff --git a/tests/unit/pipelex/tracing/test_graphspec_assembler.py b/tests/unit/pipelex/tracing/test_graphspec_assembler.py
index 8d2e2c916..c243fd638 100644
--- a/tests/unit/pipelex/tracing/test_graphspec_assembler.py
+++ b/tests/unit/pipelex/tracing/test_graphspec_assembler.py
@@ -3,11 +3,16 @@
from datetime import UTC, datetime, timedelta
from typing import Any
+import pytest
+
from pipelex.graph.graphspec import EdgeKind, ErrorSpec, GraphSpecMode, IOSpec, NodeKind, NodeStatus
+from pipelex.system.pipe_run_mode import PipeRunMode
+from pipelex.tracing.exceptions import TraceEventModeError
from pipelex.tracing.graphspec_assembler import GraphSpecAssembler
from pipelex.tracing.trace_events import (
BatchAggregateEvent,
BatchItemEvent,
+ ConditionResolvedEvent,
ControllerOutputEvent,
EdgeEvent,
ParallelCombineEvent,
@@ -41,13 +46,14 @@ def _edge_id(workflow_id: str, index: int) -> str:
return f"{_GRAPH_ID}:{workflow_id}:edge_{index}"
-def _base(workflow_id: str, sequence: int, timestamp_offset: int = 0) -> dict[str, Any]:
+def _base(workflow_id: str, sequence: int, timestamp_offset: int = 0, run_mode: PipeRunMode = PipeRunMode.LIVE) -> dict[str, Any]:
"""Base fields for event construction."""
return {
"pipeline_run_id": _PIPELINE_RUN_ID,
"workflow_id": workflow_id,
"timestamp": _time_at(timestamp_offset or sequence),
"sequence": sequence,
+ "mode": run_mode,
}
@@ -63,19 +69,48 @@ def _io_spec(name: str, digest: str) -> IOSpec:
class TestGraphSpecAssembler:
"""Tests for GraphSpecAssembler.assemble()."""
- def test_empty_events(self) -> None:
- """Empty event list produces empty GraphSpec."""
- result = GraphSpecAssembler.assemble(events=[], graph_id=_GRAPH_ID)
- assert result.graph_id == _GRAPH_ID
- assert result.nodes == []
- assert result.edges == []
- assert result.meta["mode"] == GraphSpecMode.LIVE
-
- def test_explicit_mode(self) -> None:
- """Assembler stamps the requested GraphSpec provenance mode."""
- result = GraphSpecAssembler.assemble(events=[], graph_id=_GRAPH_ID, mode=GraphSpecMode.DRY)
+ @pytest.mark.parametrize(
+ ("run_mode", "graphspec_mode"),
+ [(PipeRunMode.LIVE, GraphSpecMode.LIVE), (PipeRunMode.DRY, GraphSpecMode.DRY)],
+ ids=["live", "dry"],
+ )
+ def test_the_graph_takes_the_mode_its_events_state(self, run_mode: PipeRunMode, graphspec_mode: GraphSpecMode) -> None:
+ node_id = _node_id(_WF_A, 0)
+ result = GraphSpecAssembler.assemble(
+ events=[
+ PipeStartEvent(
+ **_base(_WF_A, 0, run_mode=run_mode), node_id=node_id, pipe_code="only", pipe_type="PipeLLM", node_kind=NodeKind.OPERATOR
+ ),
+ PipeEndSuccessEvent(**_base(_WF_A, 1, run_mode=run_mode), node_id=node_id, ended_at=_time_at(1)),
+ ],
+ graph_id=_GRAPH_ID,
+ )
assert result.meta["format"] == "mthds"
- assert result.meta["mode"] == GraphSpecMode.DRY
+ assert result.meta["mode"] == graphspec_mode
+
+ @pytest.mark.parametrize(
+ ("run_modes", "expected_message"),
+ [
+ ([], "No trace events to assemble graph 'test_graph' from"),
+ ([PipeRunMode.LIVE, PipeRunMode.DRY], "The trace events of graph 'test_graph' mix run modes (dry, live)"),
+ ],
+ ids=["no_events", "mixed_modes"],
+ )
+ def test_events_stating_no_single_mode_are_refused(self, run_modes: list[PipeRunMode], expected_message: str) -> None:
+ """A set of events that mixes modes holds more than one run, and an empty one states no mode: neither gets a graph."""
+ events = [
+ PipeStartEvent(
+ **_base(_WF_A, index_event, run_mode=run_mode),
+ node_id=_node_id(_WF_A, index_event),
+ pipe_code=f"pipe_{index_event}",
+ pipe_type="PipeLLM",
+ node_kind=NodeKind.OPERATOR,
+ )
+ for index_event, run_mode in enumerate(run_modes)
+ ]
+ with pytest.raises(TraceEventModeError) as exc_info:
+ GraphSpecAssembler.assemble(events=events, graph_id=_GRAPH_ID)
+ assert str(exc_info.value).startswith(expected_message)
def test_registry_source_payloads_survive_assembly(self) -> None:
"""PipeStartEvent and PipeEndSuccessEvent registry payloads are copied unchanged."""
@@ -505,20 +540,17 @@ class TestGraphSpecAssembler:
assert (branch_1, controller) in batch_agg_pairs
def test_condition_selected_outcome(self) -> None:
- """Condition pipe with SELECTED_OUTCOME edge."""
- condition = _node_id(_WF_A, 0)
- outcome = _node_id(_WF_A, 1)
+ """The SELECTED_OUTCOME edge is derived from the condition's resolution and the start of the pipe it chose.
+
+ The chosen pipe starts in another workflow than the condition, whose events sort first, as a dispatched sub-pipe
+ does in a distributed run; an edge event of the same kind a tracer drew is regenerated, never duplicated.
+ """
+ condition = _node_id(_WF_B, 0)
+ outcome = _node_id(_WF_A, 0)
events = [
PipeStartEvent(
**_base(_WF_A, 0),
- node_id=condition,
- pipe_code="check",
- pipe_type="PipeCondition",
- node_kind=NodeKind.CONTROLLER,
- ),
- PipeStartEvent(
- **_base(_WF_A, 1),
node_id=outcome,
parent_node_id=condition,
pipe_code="branch_true",
@@ -526,27 +558,40 @@ class TestGraphSpecAssembler:
node_kind=NodeKind.OPERATOR,
),
EdgeEvent(
- **_base(_WF_A, 2),
+ **_base(_WF_A, 1),
edge_id=_edge_id(_WF_A, 0),
source_node_id=condition,
target_node_id=outcome,
edge_kind=EdgeKind.CONTAINS,
),
+ PipeEndSuccessEvent(
+ **_base(_WF_A, 2),
+ node_id=outcome,
+ ended_at=_time_at(4),
+ ),
+ PipeStartEvent(
+ **_base(_WF_B, 0),
+ node_id=condition,
+ pipe_code="check",
+ pipe_type="PipeCondition",
+ node_kind=NodeKind.CONTROLLER,
+ ),
+ ConditionResolvedEvent(
+ **_base(_WF_B, 1),
+ node_id=condition,
+ rendered_value="true",
+ selected_outcome="true",
+ ),
EdgeEvent(
- **_base(_WF_A, 3),
- edge_id=_edge_id(_WF_A, 1),
+ **_base(_WF_B, 2),
+ edge_id=_edge_id(_WF_B, 0),
source_node_id=condition,
target_node_id=outcome,
edge_kind=EdgeKind.SELECTED_OUTCOME,
label="true",
),
PipeEndSuccessEvent(
- **_base(_WF_A, 4),
- node_id=outcome,
- ended_at=_time_at(4),
- ),
- PipeEndSuccessEvent(
- **_base(_WF_A, 5),
+ **_base(_WF_B, 3),
node_id=condition,
ended_at=_time_at(5),
),
@@ -555,10 +600,39 @@ class TestGraphSpecAssembler:
result = GraphSpecAssembler.assemble(events=events, graph_id=_GRAPH_ID)
selected_edges = [edge for edge in result.edges if edge.kind == EdgeKind.SELECTED_OUTCOME]
- assert len(selected_edges) == 1
- assert selected_edges[0].source == condition
- assert selected_edges[0].target == outcome
- assert selected_edges[0].label == "true"
+ assert [(edge.source, edge.target, edge.label) for edge in selected_edges] == [(condition, outcome, "true")]
+
+ def test_a_resolution_of_a_condition_absent_from_the_graph_draws_no_edge(self) -> None:
+ """A resolution whose condition node no event started draws no SELECTED_OUTCOME edge from a node the graph lacks."""
+ absent_condition = _node_id(_WF_B, 0)
+ outcome = _node_id(_WF_A, 0)
+
+ events = [
+ PipeStartEvent(
+ **_base(_WF_A, 0),
+ node_id=outcome,
+ parent_node_id=absent_condition,
+ pipe_code="branch_true",
+ pipe_type="PipeLLM",
+ node_kind=NodeKind.OPERATOR,
+ ),
+ PipeEndSuccessEvent(
+ **_base(_WF_A, 1),
+ node_id=outcome,
+ ended_at=_time_at(4),
+ ),
+ ConditionResolvedEvent(
+ **_base(_WF_B, 0),
+ node_id=absent_condition,
+ rendered_value="true",
+ selected_outcome="true",
+ ),
+ ]
+
+ result = GraphSpecAssembler.assemble(events=events, graph_id=_GRAPH_ID)
+
+ assert [node.node_id for node in result.nodes] == [outcome]
+ assert [edge for edge in result.edges if edge.kind == EdgeKind.SELECTED_OUTCOME] == []
def test_partial_failure(self) -> None:
"""Child fails with error, parent has no end event → FAILED + CANCELED."""
diff --git a/tests/unit/pipelex/tracing/test_ndjson_dedup_assembly.py b/tests/unit/pipelex/tracing/test_ndjson_dedup_assembly.py
index 901635531..cf8b2bdc3 100644
--- a/tests/unit/pipelex/tracing/test_ndjson_dedup_assembly.py
+++ b/tests/unit/pipelex/tracing/test_ndjson_dedup_assembly.py
@@ -10,6 +10,7 @@ from typing import Any
import pytest
from pipelex.graph.graphspec import EdgeKind, IOSpec, NodeKind, NodeStatus
+from pipelex.system.pipe_run_mode import PipeRunMode
from pipelex.tracing.graphspec_assembler import GraphSpecAssembler
from pipelex.tracing.ndjson_event_log import NdjsonEventLog
from pipelex.tracing.trace_events import (
@@ -35,6 +36,7 @@ def _base(sequence: int) -> dict[str, Any]:
"workflow_id": _WF_ID,
"timestamp": _time_at(sequence),
"sequence": sequence,
+ "mode": PipeRunMode.LIVE,
}
diff --git a/tests/unit/pipelex/tracing/test_non_llm_usage_aggregation.py b/tests/unit/pipelex/tracing/test_non_llm_usage_aggregation.py
index 3cb51081d..943643eea 100644
--- a/tests/unit/pipelex/tracing/test_non_llm_usage_aggregation.py
+++ b/tests/unit/pipelex/tracing/test_non_llm_usage_aggregation.py
@@ -21,6 +21,7 @@ from pipelex.cogt.usage.cost_registry import CostRegistry, ModelUsageKey
from pipelex.cogt.usage.token_category import TokenCategory
from pipelex.reporting.reporting_types import AnyTokensUsage
from pipelex.system.job_metadata import JobMetadata, RunMetadata
+from pipelex.system.pipe_run_mode import PipeRunMode
from pipelex.tracing.trace_events import UsageReportEvent
from pipelex.tracing.usage_aggregator import UsageAggregator
@@ -36,6 +37,7 @@ def _usage_event(tokens_usage: AnyTokensUsage, sequence: int) -> UsageReportEven
writer_id="act_test",
timestamp=datetime.now(UTC),
sequence=sequence,
+ mode=PipeRunMode.LIVE,
node_id=f"g:node_{sequence}",
tokens_usage=tokens_usage,
)
diff --git a/tests/unit/pipelex/tracing/test_trace_events.py b/tests/unit/pipelex/tracing/test_trace_events.py
index 9219da969..f3565deb1 100644
--- a/tests/unit/pipelex/tracing/test_trace_events.py
+++ b/tests/unit/pipelex/tracing/test_trace_events.py
@@ -4,7 +4,7 @@ from datetime import UTC, datetime
from typing import Any, ClassVar
import pytest
-from pydantic import TypeAdapter
+from pydantic import TypeAdapter, ValidationError
from pipelex.cogt.llm.llm_report import LLMTokensUsage
from pipelex.cogt.usage.cost_category import CostCategory
@@ -12,11 +12,14 @@ from pipelex.cogt.usage.token_category import TokenCategory
from pipelex.graph.condition_output_merge import ConditionOutputMerge, ConditionOutputTyping
from pipelex.graph.graphspec import EdgeKind, ErrorSpec, IOSpec, NodeKind
from pipelex.system.job_metadata import JobCategory, JobMetadata, RunMetadata, UnitJobId
+from pipelex.system.pipe_run_mode import PipeRunMode
from pipelex.tracing.trace_events import (
AnyTraceEvent,
BatchAggregateEvent,
+ BatchFannedOutEvent,
BatchItemEvent,
ConditionOutputMergeEvent,
+ ConditionResolvedEvent,
ControllerOutputEvent,
EdgeEvent,
ExecutionDataEvent,
@@ -47,6 +50,7 @@ class _Shared:
"workflow_id": _Shared.WORKFLOW_ID,
"timestamp": _Shared.TIMESTAMP,
"sequence": sequence,
+ "mode": PipeRunMode.LIVE,
}
@staticmethod
@@ -199,6 +203,26 @@ class TestTraceEvents:
),
},
),
+ (
+ "batch_fanned_out",
+ TraceEventKind.BATCH_FANNED_OUT,
+ {
+ "node_id": _Shared.NODE_ID,
+ "item_count": 3,
+ "branch_pipe_code": "summarize_record",
+ "branch_pipe_ref": "records.summarize_record",
+ "branch_pipe_type": "PipeLLM",
+ },
+ ),
+ (
+ "condition_resolved",
+ TraceEventKind.CONDITION_RESOLVED,
+ {
+ "node_id": _Shared.NODE_ID,
+ "rendered_value": "urgent",
+ "selected_outcome": "urgent",
+ },
+ ),
(
"execution_data",
TraceEventKind.EXECUTION_DATA,
@@ -228,6 +252,8 @@ class TestTraceEvents:
TraceEventKind.BATCH_AGGREGATE: BatchAggregateEvent,
TraceEventKind.PARALLEL_COMBINE: ParallelCombineEvent,
TraceEventKind.CONDITION_OUTPUT_MERGE: ConditionOutputMergeEvent,
+ TraceEventKind.BATCH_FANNED_OUT: BatchFannedOutEvent,
+ TraceEventKind.CONDITION_RESOLVED: ConditionResolvedEvent,
TraceEventKind.EXECUTION_DATA: ExecutionDataEvent,
TraceEventKind.USAGE_REPORT: UsageReportEvent,
}
@@ -407,6 +433,86 @@ class TestTraceEvents:
assert restored.description == "Summarize the input document."
assert restored.domain_code == "summarization"
+ @pytest.mark.parametrize(
+ ("step_count", "branch_count"),
+ [(None, None), (3, None), (None, 2)],
+ ids=["operator", "sequence", "parallel"],
+ )
+ def test_pipe_start_carries_step_and_branch_counts(self, step_count: int | None, branch_count: int | None) -> None:
+ """A sequence states its authored step count and a parallel its branch count as they start; any other pipe states neither."""
+ event = PipeStartEvent(
+ **_Shared.make_base_fields(),
+ node_id=_Shared.NODE_ID,
+ pipe_code="controller",
+ pipe_type="PipeSequence",
+ node_kind=NodeKind.CONTROLLER,
+ step_count=step_count,
+ branch_count=branch_count,
+ )
+
+ payload = event.model_dump(mode="json")
+ restored = _any_trace_event_adapter.validate_json(event.model_dump_json())
+
+ assert payload["step_count"] == step_count
+ assert payload["branch_count"] == branch_count
+ assert isinstance(restored, PipeStartEvent)
+ assert restored.step_count == step_count
+ assert restored.branch_count == branch_count
+
+ def test_condition_resolved_default_outcome_reads_null(self) -> None:
+ """A condition that took its default outcome states `selected_outcome` as null, with the value it rendered."""
+ event = ConditionResolvedEvent(
+ **_Shared.make_base_fields(),
+ node_id=_Shared.NODE_ID,
+ rendered_value="unmatched",
+ selected_outcome=None,
+ )
+
+ payload = event.model_dump(mode="json")
+ restored = _any_trace_event_adapter.validate_json(event.model_dump_json())
+
+ assert payload["event_kind"] == "condition_resolved"
+ assert payload["selected_outcome"] is None
+ assert payload["rendered_value"] == "unmatched"
+ assert restored == event
+
+ def test_batch_fanned_out_refuses_a_negative_item_count(self) -> None:
+ with pytest.raises(ValidationError):
+ BatchFannedOutEvent(
+ **_Shared.make_base_fields(),
+ node_id=_Shared.NODE_ID,
+ item_count=-1,
+ branch_pipe_code="summarize_record",
+ branch_pipe_ref="records.summarize_record",
+ branch_pipe_type="PipeLLM",
+ )
+
+ @pytest.mark.parametrize("mode", [PipeRunMode.LIVE, PipeRunMode.DRY], ids=["live", "dry"])
+ def test_every_event_states_its_mode(self, mode: PipeRunMode) -> None:
+ """Every event kind serializes the run mode it was written in, and reads it back."""
+ for _, event_kind, extra_fields in self.EVENTS_AND_KINDS:
+ event_cls = self.EVENT_CLASSES[event_kind]
+ event = event_cls(**{**_Shared.make_base_fields(), "mode": mode}, **extra_fields)
+
+ payload = event.model_dump(mode="json")
+ restored = _any_trace_event_adapter.validate_json(event.model_dump_json())
+
+ assert payload["mode"] == mode, event_kind
+ assert restored.mode == mode, event_kind
+
+ def test_an_event_without_a_mode_is_refused(self) -> None:
+ """The mode is required: a line written before it existed fails to load rather than reading as live."""
+ base_fields = _Shared.make_base_fields()
+ del base_fields["mode"]
+ with pytest.raises(ValidationError):
+ PipeStartEvent(
+ **base_fields,
+ node_id=_Shared.NODE_ID,
+ pipe_code="noop_pipe",
+ pipe_type="PipeNoop",
+ node_kind=NodeKind.OPERATOR,
+ )
+
def test_all_event_kinds_covered(self) -> None:
"""Every TraceEventKind has a corresponding test case and event class."""
tested_kinds = {entry[1] for entry in self.EVENTS_AND_KINDS}
diff --git a/tests/unit/pipelex/tracing/test_usage_aggregator.py b/tests/unit/pipelex/tracing/test_usage_aggregator.py
index 13f1a44ac..79076e8c2 100644
--- a/tests/unit/pipelex/tracing/test_usage_aggregator.py
+++ b/tests/unit/pipelex/tracing/test_usage_aggregator.py
@@ -8,6 +8,7 @@ from pipelex.cogt.usage.cost_category import CostCategory
from pipelex.cogt.usage.token_category import TokenCategory
from pipelex.graph.graphspec import NodeKind
from pipelex.system.job_metadata import JobCategory, JobMetadata, RunMetadata, UnitJobId
+from pipelex.system.pipe_run_mode import PipeRunMode
from pipelex.tracing.trace_events import PipeStartEvent, TraceEvent, UsageReportEvent
from pipelex.tracing.usage_aggregator import UsageAggregator
@@ -37,6 +38,7 @@ def _make_base_fields(sequence: int = 0) -> dict[str, Any]:
"workflow_id": _WORKFLOW_ID,
"timestamp": _TIMESTAMP,
"sequence": sequence,
+ "mode": PipeRunMode.LIVE,
}
diff --git a/tests/unit/pipelex/tracing/test_writer_id_schema.py b/tests/unit/pipelex/tracing/test_writer_id_schema.py
index 7ef62145b..c532b46a8 100644
--- a/tests/unit/pipelex/tracing/test_writer_id_schema.py
+++ b/tests/unit/pipelex/tracing/test_writer_id_schema.py
@@ -13,6 +13,7 @@ from typing import Any
from pytest_mock import MockerFixture
from pipelex.graph.graphspec import NodeKind
+from pipelex.system.pipe_run_mode import PipeRunMode
from pipelex.tracing.buffering_event_log import BufferingEventLog
from pipelex.tracing.dynamodb_event_log import DynamoDBEventLog
from pipelex.tracing.in_memory_event_log import InMemoryEventLog
@@ -31,6 +32,7 @@ def _make_pipe_start(sequence: int, writer_id: str = "primary", workflow_id: str
writer_id=writer_id,
timestamp=_TS,
sequence=sequence,
+ mode=PipeRunMode.LIVE,
node_id=f"g:{workflow_id}:node_{sequence}",
pipe_code="pipe_x",
pipe_type="PipeLLM",
@@ -48,6 +50,7 @@ class TestWriterIdSchema:
workflow_id=_WORKFLOW_ID,
timestamp=_TS,
sequence=0,
+ mode=PipeRunMode.LIVE,
node_id=f"g:{_WORKFLOW_ID}:node_0",
pipe_code="pipe_x",
pipe_type="PipeLLM",
@@ -75,6 +78,7 @@ class TestWriterIdSchema:
"workflow_id": _WORKFLOW_ID,
"timestamp": _TS.isoformat(),
"sequence": 0,
+ "mode": "live",
"event_kind": "pipe_start",
"node_id": f"g:{_WORKFLOW_ID}:node_0",
"pipe_code": "pipe_x",