Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
Feature/assignment pattern (#812)
Thomas Hebrardf2f7ed8e26f5
diff --git a/.pipelex/pipelex.toml b/.pipelex/pipelex.tomlindex 9a44d8f78..719a461f3 100644--- a/.pipelex/pipelex.toml+++ b/.pipelex/pipelex.toml@@ -42,6 +42,7 @@ stuff_json_content = truestuff_text_content = truestuff_html_content = trueerror_stack_traces = true+pipe_and_concept_registry = true[pipelex.pipeline_execution_config.graph_config.graphs_inclusion]# Control which graph outputs are generateddiff --git a/pipelex/core/pipes/pipe_abstract.py b/pipelex/core/pipes/pipe_abstract.pyindex d6143f488..fc40435b2 100644--- a/pipelex/core/pipes/pipe_abstract.py+++ b/pipelex/core/pipes/pipe_abstract.py@@ -84,6 +84,37 @@ class PipeAbstract(ABC, BaseModel):return unique_concepts+ def _register_execution_data(self, job_metadata: JobMetadata, execution_data: dict[str, Any]) -> None:+ """Register execution metadata with the graph tracer.++ Called by pipe subclasses during execution to capture runtime-resolved data+ (rendered prompts, resolved models, etc.) for the GraphSpec.+ """+ graph_context = job_metadata.graph_context+ if graph_context is None:+ return+ tracer_manager = GraphTracerManager.get_instance()+ if tracer_manager is None or graph_context.parent_node_id is None:+ return+ tracer_manager.register_execution_data(+ graph_id=graph_context.graph_id,+ node_id=graph_context.parent_node_id,+ execution_data=execution_data,+ )++ def _make_single_concept_data_for_registry(self, concept: Concept) -> dict[str, Any]:+ """Serialize a single concept for the graph registry, including its JSON Schema."""+ concept_dict = concept.model_dump(mode="json")+ try:+ concept_dict["json_schema"] = concept.get_structure_class().model_json_schema()+ except (TypeError, ValueError):+ concept_dict["json_schema"] = None+ return concept_dict++ def _make_concept_data_for_registry(self) -> list[dict[str, Any]]:+ """Serialize all unique concepts from this pipe for the graph registry."""+ return [self._make_single_concept_data_for_registry(concept) for concept in self.concept_dependencies]+@field_validator("code", mode="before")@classmethoddef validate_pipe_code_syntax(cls, code: str) -> str:@@ -411,6 +442,13 @@ class PipeAbstract(ABC, BaseModel):)input_specs.append(input_spec)+ # Serialize pipe and concept data for registries if enabled+ pipe_data: dict[str, Any] | None = None+ concept_data: list[dict[str, Any]] | None = None+ if parent_graph_context.data_inclusion.pipe_and_concept_registry:+ pipe_data = self.model_dump(mode="json")+ concept_data = self._make_concept_data_for_registry()+graph_node_id, child_graph_context = tracer_manager.on_pipe_start(graph_context=parent_graph_context,pipe_code=self.code,@@ -418,6 +456,8 @@ class PipeAbstract(ABC, BaseModel):node_kind=node_kind,started_at=started_at,input_specs=input_specs or None,+ pipe_data=pipe_data,+ concept_data=concept_data,)# Update job metadata with child graph context for nested pipesif child_graph_context is not None:@@ -480,11 +520,17 @@ class PipeAbstract(ABC, BaseModel):data_html=main_stuff.content.rendered_pretty_html() if parent_graph_context.data_inclusion.stuff_html_content else None,)+ # Serialize output concept for registry if enabled+ output_concept_data: dict[str, Any] | None = None+ if parent_graph_context.data_inclusion.pipe_and_concept_registry and main_stuff is not None:+ output_concept_data = self._make_single_concept_data_for_registry(main_stuff.concept)+tracer_manager.on_pipe_end_success(graph_id=parent_graph_context.graph_id,node_id=graph_node_id,ended_at=datetime.now(timezone.utc),output_spec=output_spec,+ output_concept_data=output_concept_data,)pipe_run_params.pop_pipe_from_stack(pipe_code=self.code)diff --git a/pipelex/core/pipes/pipe_assignment.py b/pipelex/core/pipes/pipe_assignment.pynew file mode 100644index 000000000..54ea86466--- /dev/null+++ b/pipelex/core/pipes/pipe_assignment.py@@ -0,0 +1,25 @@+"""Base Assignment class for the pipe execution pre_run/run/post_run pattern.++An Assignment carries resolved runtime data through the execution lifecycle:+- Created in prepare_assignment() (pre_run phase): models resolved, prompts rendered+- Consumed in execute() (run phase): pure execution using pre-resolved data+- Finalized in finalize_assignment() (post_run phase): execution metadata captured++Each pipe subclass defines its own Assignment type with operator-specific fields.+"""++from typing import Any++from pydantic import BaseModel, ConfigDict, Field+++class PipeAssignment(BaseModel):+ """Base Assignment for pipe execution.++ Subclasses add operator-specific fields (rendered prompts, resolved models, etc.).+ The execution_data dict collects metadata for the GraphSpec's execution_data field.+ """++ model_config = ConfigDict(arbitrary_types_allowed=True)++ execution_data: dict[str, Any] = Field(default_factory=dict)diff --git a/pipelex/graph/graph_config.py b/pipelex/graph/graph_config.pyindex 021d9bc4d..e555ca3bf 100644--- a/pipelex/graph/graph_config.py+++ b/pipelex/graph/graph_config.py@@ -10,6 +10,7 @@ class DataInclusionConfig(ConfigModel):stuff_text_content: boolstuff_html_content: boolerror_stack_traces: bool+ pipe_and_concept_registry: boolclass GraphsInclusionConfig(ConfigModel):diff --git a/pipelex/graph/graph_tracer.py b/pipelex/graph/graph_tracer.pyindex 125273799..df146b21e 100644--- a/pipelex/graph/graph_tracer.py+++ b/pipelex/graph/graph_tracer.py@@ -1,6 +1,7 @@"""GraphTracer implementation that builds GraphSpec during pipeline execution."""from datetime import datetime, timezone+from typing import Anyfrom typing_extensions import override@@ -48,6 +49,7 @@ class _MutableNodeData:self.error: ErrorSpec | None = Noneself.input_specs: list[IOSpec] = input_specs or []self.output_specs: list[IOSpec] = []+ self.execution_data: dict[str, Any] = {}def to_node_spec(self) -> NodeSpec:"""Convert to immutable NodeSpec."""@@ -76,6 +78,7 @@ class _MutableNodeData:node_io=node_io,error=self.error,metrics=self.metrics,+ execution_data=self.execution_data,)@@ -113,6 +116,9 @@ class GraphTracer(GraphTracerProtocol):# The branch_producer_node_id is snapshotted at registration time, before register_controller_output# overrides _stuff_producer_map to point branch stuff codes to the controller nodeself._parallel_combine_map: dict[str, tuple[str, list[tuple[str, str]]]] = {}+ # Registries for pipe and concept data (keyed by pipe_ref and concept_ref)+ self._pipe_registry: dict[str, dict[str, Any]] = {}+ self._concept_registry: dict[str, dict[str, Any]] = {}@propertydef is_active(self) -> bool:@@ -143,6 +149,8 @@ class GraphTracer(GraphTracerProtocol):self._batch_item_map = {}self._batch_aggregate_map = {}self._parallel_combine_map = {}+ self._pipe_registry = {}+ self._concept_registry = {}return GraphContext(graph_id=graph_id,@@ -181,6 +189,8 @@ class GraphTracer(GraphTracerProtocol):pipeline_ref=self._pipeline_ref or PipelineRef(),nodes=nodes,edges=self._edges,+ pipe_registry=dict(self._pipe_registry),+ concept_registry=dict(self._concept_registry),)# Reset internal state@@ -193,6 +203,8 @@ class GraphTracer(GraphTracerProtocol):self._batch_item_map = {}self._batch_aggregate_map = {}self._parallel_combine_map = {}+ self._pipe_registry = {}+ self._concept_registry = {}return graph@@ -409,6 +421,8 @@ class GraphTracer(GraphTracerProtocol):node_kind: NodeKind,started_at: datetime,input_specs: list[IOSpec] | None = None,+ pipe_data: dict[str, Any] | None = None,+ concept_data: list[dict[str, Any]] | None = None,) -> tuple[str, GraphContext]:"""Record the start of a pipe execution."""if not self._is_active:@@ -433,6 +447,18 @@ class GraphTracer(GraphTracerProtocol):)self._nodes[node_id] = node_data+ # Accumulate pipe and concept registry data (deduplicated)+ if graph_context.data_inclusion.pipe_and_concept_registry:+ if pipe_data is not None:+ pipe_ref = f"{pipe_data.get('domain_code', '')}.{pipe_data.get('code', '')}"+ if pipe_ref not in self._pipe_registry:+ self._pipe_registry[pipe_ref] = pipe_data+ if concept_data is not None:+ for concept_item in concept_data:+ concept_ref = f"{concept_item.get('domain_code', '')}.{concept_item.get('code', '')}"+ if concept_ref not in self._concept_registry:+ self._concept_registry[concept_ref] = concept_item+# Add containment edge from parent if this is a child pipeif graph_context.parent_node_id is not None:self.add_edge(@@ -449,6 +475,20 @@ class GraphTracer(GraphTracerProtocol):return node_id, child_context+ @override+ def register_execution_data(+ self,+ node_id: str,+ execution_data: dict[str, Any],+ ) -> None:+ """Register execution metadata for a node."""+ if not self._is_active:+ return+ node_data = self._nodes.get(node_id)+ if node_data is None:+ return+ node_data.execution_data.update(execution_data)+@overridedef on_pipe_end_success(self,@@ -457,6 +497,7 @@ class GraphTracer(GraphTracerProtocol):output_preview: str | None = None,metrics: dict[str, float] | None = None,output_spec: IOSpec | None = None,+ output_concept_data: dict[str, Any] | None = None,) -> None:"""Record successful completion of a pipe execution."""if not self._is_active:@@ -472,6 +513,12 @@ class GraphTracer(GraphTracerProtocol):if metrics:node_data.metrics = metrics+ # Accumulate output concept data (deduplicated)+ if output_concept_data is not None:+ concept_ref = f"{output_concept_data.get('domain_code', '')}.{output_concept_data.get('code', '')}"+ if concept_ref not in self._concept_registry:+ self._concept_registry[concept_ref] = output_concept_data+# Store output spec and register in producer map for data flow trackingif output_spec is not None:# Skip pass-through outputs: if the output digest matches one of the node'sdiff --git a/pipelex/graph/graph_tracer_manager.py b/pipelex/graph/graph_tracer_manager.pyindex 3b4d770df..3f24ba9da 100644--- a/pipelex/graph/graph_tracer_manager.py+++ b/pipelex/graph/graph_tracer_manager.py@@ -1,6 +1,7 @@"""Graph tracer manager with singleton pattern for access from PipeAbstract without hub imports."""from datetime import datetime+from typing import Anyfrom pipelex.graph.graph_config import DataInclusionConfigfrom pipelex.graph.graph_context import GraphContext@@ -175,6 +176,8 @@ class GraphTracerManager(metaclass=ABCSingletonMeta):node_kind: NodeKind,started_at: datetime,input_specs: list[IOSpec] | None = None,+ pipe_data: dict[str, Any] | None = None,+ concept_data: list[dict[str, Any]] | None = None,) -> tuple[str | None, GraphContext | None]:"""Record the start of a pipe execution.@@ -185,6 +188,8 @@ class GraphTracerManager(metaclass=ABCSingletonMeta):node_kind: The kind of node (controller, operator, etc.).started_at: When the pipe started executing.input_specs: Optional list of IOSpec describing the inputs consumed.+ pipe_data: Optional serialized pipe instance for the pipe registry.+ concept_data: Optional list of serialized concept dicts for the concept registry.Returns:Tuple of (node_id, child_graph_context) if tracing is active, (None, None) otherwise.@@ -200,6 +205,8 @@ class GraphTracerManager(metaclass=ABCSingletonMeta):node_kind=node_kind,started_at=started_at,input_specs=input_specs,+ pipe_data=pipe_data,+ concept_data=concept_data,)def on_pipe_end_success(@@ -210,6 +217,7 @@ class GraphTracerManager(metaclass=ABCSingletonMeta):output_preview: str | None = None,metrics: dict[str, float] | None = None,output_spec: IOSpec | None = None,+ output_concept_data: dict[str, Any] | None = None,) -> None:"""Record successful completion of a pipe execution.@@ -220,6 +228,7 @@ class GraphTracerManager(metaclass=ABCSingletonMeta):output_preview: Optional truncated preview of the output.metrics: Optional metrics (e.g., token counts).output_spec: Optional IOSpec describing the output produced.+ output_concept_data: Optional serialized concept dict for the actual output concept."""if node_id is None:return@@ -234,8 +243,29 @@ class GraphTracerManager(metaclass=ABCSingletonMeta):output_preview=output_preview,metrics=metrics,output_spec=output_spec,+ output_concept_data=output_concept_data,)+ def register_execution_data(+ self,+ graph_id: str,+ node_id: str | None,+ execution_data: dict[str, Any],+ ) -> None:+ """Register execution metadata for a node.++ Args:+ graph_id: The graph identifier.+ node_id: The node ID to attach execution data to.+ execution_data: Dictionary of execution metadata.+ """+ if node_id is None:+ return+ tracer = self._get_tracer(graph_id)+ if tracer is None:+ return+ tracer.register_execution_data(node_id=node_id, execution_data=execution_data)+def on_pipe_end_error(self,graph_id: str,diff --git a/pipelex/graph/graph_tracer_protocol.py b/pipelex/graph/graph_tracer_protocol.pyindex adbb217a8..4a8b70d1c 100644--- a/pipelex/graph/graph_tracer_protocol.py+++ b/pipelex/graph/graph_tracer_protocol.py@@ -1,5 +1,5 @@from datetime import datetime-from typing import Protocol+from typing import Any, Protocolfrom typing_extensions import override@@ -51,6 +51,8 @@ class GraphTracerProtocol(Protocol):node_kind: NodeKind,started_at: datetime,input_specs: list[IOSpec] | None = None,+ pipe_data: dict[str, Any] | None = None,+ concept_data: list[dict[str, Any]] | None = None,) -> tuple[str, GraphContext]:"""Record the start of a pipe execution.@@ -61,6 +63,8 @@ class GraphTracerProtocol(Protocol):node_kind: The kind of node (controller, operator, etc.).started_at: When the pipe started executing.input_specs: Optional list of IOSpec describing the inputs consumed.+ pipe_data: Optional serialized pipe instance for the pipe registry.+ concept_data: Optional list of serialized concept dicts for the concept registry.Returns:Tuple of (node_id for this pipe, updated GraphContext for children).@@ -74,6 +78,7 @@ class GraphTracerProtocol(Protocol):output_preview: str | None = None,metrics: dict[str, float] | None = None,output_spec: IOSpec | None = None,+ output_concept_data: dict[str, Any] | None = None,) -> None:"""Record successful completion of a pipe execution.@@ -83,6 +88,7 @@ class GraphTracerProtocol(Protocol):output_preview: Optional truncated preview of the output.metrics: Optional metrics (e.g., token counts).output_spec: Optional IOSpec describing the output produced.+ output_concept_data: Optional serialized concept dict for the actual output concept."""...@@ -197,6 +203,19 @@ class GraphTracerProtocol(Protocol):"""...+ def register_execution_data(+ self,+ node_id: str,+ execution_data: dict[str, Any],+ ) -> None:+ """Register execution metadata for a node.++ Args:+ node_id: The node ID to attach execution data to.+ execution_data: Dictionary of execution metadata (rendered prompts, resolved models, etc.).+ """+ ...+class GraphTracerNoOp(GraphTracerProtocol):"""No-operation implementation of GraphTracerProtocol.@@ -230,6 +249,8 @@ class GraphTracerNoOp(GraphTracerProtocol):node_kind: NodeKind,started_at: datetime,input_specs: list[IOSpec] | None = None,+ pipe_data: dict[str, Any] | None = None,+ concept_data: list[dict[str, Any]] | None = None,) -> tuple[str, GraphContext]:node_id = graph_context.make_node_id()child_context = graph_context.copy_for_child(node_id, graph_context.node_sequence + 1)@@ -243,6 +264,15 @@ class GraphTracerNoOp(GraphTracerProtocol):output_preview: str | None = None,metrics: dict[str, float] | None = None,output_spec: IOSpec | None = None,+ output_concept_data: dict[str, Any] | None = None,+ ) -> None:+ pass++ @override+ def register_execution_data(+ self,+ node_id: str,+ execution_data: dict[str, Any],) -> None:passdiff --git a/pipelex/graph/graphspec.py b/pipelex/graph/graphspec.pyindex 66892c827..d471a9d4a 100644--- a/pipelex/graph/graphspec.py+++ b/pipelex/graph/graphspec.py@@ -244,6 +244,7 @@ class NodeSpec(BaseModel):error: ErrorSpec | None = Nonetags: dict[str, str] = Field(default_factory=dict)metrics: dict[str, float] = Field(default_factory=dict)+ execution_data: dict[str, Any] = Field(default_factory=dict)class EdgeSpec(BaseModel):@@ -277,6 +278,8 @@ class GraphSpec(BaseModel):nodes: list[NodeSpec] = Field(default_factory=empty_list_factory_of(NodeSpec))edges: list[EdgeSpec] = Field(default_factory=empty_list_factory_of(EdgeSpec))meta: dict[str, Any] = Field(default_factory=dict)+ pipe_registry: dict[str, dict[str, Any]] = Field(default_factory=dict)+ concept_registry: dict[str, dict[str, Any]] = Field(default_factory=dict)def to_json(self) -> str:return self.model_dump_json(serialize_as_any=True, by_alias=True, indent=2)diff --git a/pipelex/kit/configs/pipelex.toml b/pipelex/kit/configs/pipelex.tomlindex 9a44d8f78..719a461f3 100644--- a/pipelex/kit/configs/pipelex.toml+++ b/pipelex/kit/configs/pipelex.toml@@ -42,6 +42,7 @@ stuff_json_content = truestuff_text_content = truestuff_html_content = trueerror_stack_traces = true+pipe_and_concept_registry = true[pipelex.pipeline_execution_config.graph_config.graphs_inclusion]# Control which graph outputs are generateddiff --git a/pipelex/pipe_controllers/batch/pipe_batch.py b/pipelex/pipe_controllers/batch/pipe_batch.pyindex b9a0c2896..e4ea1968f 100644--- a/pipelex/pipe_controllers/batch/pipe_batch.py+++ b/pipelex/pipe_controllers/batch/pipe_batch.py@@ -203,6 +203,13 @@ class PipeBatch(PipeController):name=output_name,)+ # Capture execution data for the graph tracer+ execution_data_dict: dict[str, Any] = {+ "item_count": len(input_content.items),+ "branch_pipe_code": self.branch_pipe_code,+ }+ self._register_execution_data(job_metadata, execution_data_dict)+return PipeOutput(working_memory=working_memory,pipeline_run_id=job_metadata.pipeline_run_id,diff --git a/pipelex/pipe_controllers/condition/pipe_condition.py b/pipelex/pipe_controllers/condition/pipe_condition.pyindex e88c7d987..162b7c404 100644--- a/pipelex/pipe_controllers/condition/pipe_condition.py+++ b/pipelex/pipe_controllers/condition/pipe_condition.py@@ -1,4 +1,4 @@-from typing import Literal+from typing import Any, Literalfrom typing_extensions import override@@ -229,12 +229,20 @@ class PipeCondition(PipeController):# Select the outcome based on the evaluated expressionoutcome = self.outcome_map.get(evaluated_expression, self.default_outcome)+ # Capture execution data for the graph tracer+ execution_data_dict: dict[str, Any] = {+ "evaluated_expression": evaluated_expression,+ "selected_outcome": str(outcome),+ }+# Handle continue caseif SpecialOutcome.is_continue(outcome):log.dev(f"PipeCondition '{self.code}' continued with outcome: {outcome}. Evaluated expression: {evaluated_expression}")+ self._register_execution_data(job_metadata, execution_data_dict)return PipeOutput(working_memory=working_memory)if SpecialOutcome.is_fail(outcome):+ self._register_execution_data(job_metadata, execution_data_dict)msg = f"PipeCondition '{self.code}' failed with outcome: {outcome}. Evaluated expression: {evaluated_expression}"raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode, pipe_code=self.code)@@ -254,7 +262,7 @@ class PipeCondition(PipeController):msg = f"Some required stuff(s) not found: {error_details}"raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode, pipe_code=self.code) from exc- return await get_pipe_router().run(+ pipe_output = await get_pipe_router().run(pipe_job=PipeJobFactory.make_pipe_job(pipe=get_required_pipe(pipe_code=outcome),job_metadata=job_metadata,@@ -263,6 +271,8 @@ class PipeCondition(PipeController):output_name=output_name,),)+ self._register_execution_data(job_metadata, execution_data_dict)+ return pipe_output@overrideasync def _dry_run_controller_pipe(@@ -311,4 +321,9 @@ class PipeCondition(PipeController):working_memory=working_memory,pipe_run_params=pipe_run_params,)+ execution_data_dict: dict[str, Any] = {+ "evaluated_expression": "dry_run",+ "selected_outcome": "all_outcomes",+ }+ self._register_execution_data(job_metadata, execution_data_dict)return PipeOutput(working_memory=working_memory)diff --git a/pipelex/pipe_controllers/parallel/pipe_parallel.py b/pipelex/pipe_controllers/parallel/pipe_parallel.pyindex 48579c971..1a279efb9 100644--- a/pipelex/pipe_controllers/parallel/pipe_parallel.py+++ b/pipelex/pipe_controllers/parallel/pipe_parallel.py@@ -208,6 +208,14 @@ class PipeParallel(PipeController):output_stuffs=output_stuffs,)+ # Capture execution data for the graph tracer+ execution_data_dict: dict[str, Any] = {+ "branch_count": len(self.parallel_sub_pipes),+ "add_each_output": self.add_each_output,+ "combined_output_concept": self.combined_output.concept_ref if self.combined_output else None,+ }+ self._register_execution_data(job_metadata, execution_data_dict)+return PipeOutput(working_memory=working_memory,pipeline_run_id=job_metadata.pipeline_run_id,@@ -296,6 +304,13 @@ class PipeParallel(PipeController):output_stuffs=output_stuffs,)+ execution_data_dict: dict[str, Any] = {+ "branch_count": len(self.parallel_sub_pipes),+ "add_each_output": self.add_each_output,+ "combined_output_concept": self.combined_output.concept_ref if self.combined_output else None,+ }+ self._register_execution_data(job_metadata, execution_data_dict)+return PipeOutput(working_memory=working_memory,pipeline_run_id=job_metadata.pipeline_run_id,diff --git a/pipelex/pipe_controllers/sequence/pipe_sequence.py b/pipelex/pipe_controllers/sequence/pipe_sequence.pyindex 6b56e590f..cb5a85d09 100644--- a/pipelex/pipe_controllers/sequence/pipe_sequence.py+++ b/pipelex/pipe_controllers/sequence/pipe_sequence.py@@ -1,4 +1,4 @@-from typing import Literal+from typing import Any, Literalfrom pydantic import field_validatorfrom typing_extensions import override@@ -194,6 +194,12 @@ class PipeSequence(PipeController):sub_pipe_run_params=sub_pipe_run_params,)evolving_memory = pipe_output.working_memory+ # Capture execution data for the graph tracer+ execution_data_dict: dict[str, Any] = {+ "step_count": len(self.sequential_sub_pipes),+ }+ self._register_execution_data(job_metadata, execution_data_dict)+return PipeOutput(working_memory=evolving_memory,pipeline_run_id=job_metadata.pipeline_run_id,diff --git a/pipelex/pipe_operators/compose/pipe_compose.py b/pipelex/pipe_operators/compose/pipe_compose.pyindex 4eb499c52..f6869d9d2 100644--- a/pipelex/pipe_operators/compose/pipe_compose.py+++ b/pipelex/pipe_operators/compose/pipe_compose.py@@ -213,6 +213,13 @@ class PipeCompose(PipeOperator[PipeComposeOutput]):name=output_name,)+ # Capture execution data for the graph tracer+ execution_data_dict: dict[str, Any] = {+ "compose_mode": "template",+ "rendered_text": jinja2_text,+ }+ self._register_execution_data(job_metadata, execution_data_dict)+return PipeComposeOutput(working_memory=working_memory,pipeline_run_id=job_metadata.pipeline_run_id,@@ -262,6 +269,12 @@ class PipeCompose(PipeOperator[PipeComposeOutput]):name=output_name,)+ # Capture execution data for the graph tracer+ execution_data_dict: dict[str, Any] = {+ "compose_mode": "construct",+ }+ self._register_execution_data(job_metadata, execution_data_dict)+return PipeComposeOutput(working_memory=working_memory,pipeline_run_id=job_metadata.pipeline_run_id,diff --git a/pipelex/pipe_operators/extract/pipe_extract.py b/pipelex/pipe_operators/extract/pipe_extract.pyindex 519421ab0..a1b93e8b8 100644--- a/pipelex/pipe_operators/extract/pipe_extract.py+++ b/pipelex/pipe_operators/extract/pipe_extract.py@@ -1,4 +1,4 @@-from typing import TYPE_CHECKING, Literal+from typing import TYPE_CHECKING, Any, Literalfrom pydantic import model_validatorfrom typing_extensions import Self, override@@ -176,6 +176,15 @@ class PipeExtract(PipeOperator[PipeExtractOutput]):name=output_name,)+ # Capture execution data for the graph tracer+ execution_data_dict: dict[str, Any] = {+ "resolved_model": extract_setting.model,+ "document_stuff_name": self.document_stuff_name,+ "should_caption_images": self.should_caption_images,+ "should_include_page_views": self.should_include_page_views,+ }++ self._register_execution_data(job_metadata, execution_data_dict)return PipeExtractOutput(working_memory=working_memory,pipeline_run_id=job_metadata.pipeline_run_id,diff --git a/pipelex/pipe_operators/img_gen/pipe_img_gen.py b/pipelex/pipe_operators/img_gen/pipe_img_gen.pyindex fca79b11e..02b8bfe4c 100644--- a/pipelex/pipe_operators/img_gen/pipe_img_gen.py+++ b/pipelex/pipe_operators/img_gen/pipe_img_gen.py@@ -1,4 +1,4 @@-from typing import TYPE_CHECKING, Literal+from typing import TYPE_CHECKING, Any, Literalfrom pydantic import Fieldfrom typing_extensions import override@@ -233,6 +233,16 @@ class PipeImgGen(PipeOperator[PipeImgGenOutput]):name=output_name,)+ # Capture execution data for the graph tracer+ execution_data_dict: dict[str, Any] = {+ "resolved_model": img_gen_setting.model,+ "rendered_prompt": img_gen_prompt.positive_text,+ "rendered_negative_prompt": img_gen_prompt.negative_text,+ "aspect_ratio": str(img_gen_job_params.aspect_ratio) if img_gen_job_params.aspect_ratio else None,+ "nb_images": nb_images,+ }++ self._register_execution_data(job_metadata, execution_data_dict)return PipeImgGenOutput(working_memory=working_memory,pipeline_run_id=job_metadata.pipeline_run_id,diff --git a/pipelex/pipe_operators/llm/pipe_llm.py b/pipelex/pipe_operators/llm/pipe_llm.pyindex ccb47fc54..e97df2024 100644--- a/pipelex/pipe_operators/llm/pipe_llm.py+++ b/pipelex/pipe_operators/llm/pipe_llm.py@@ -1,4 +1,4 @@-from typing import Literal, cast+from typing import Any, Literal, castfrom pydantic import ValidationError, model_validatorfrom typing_extensions import override@@ -235,6 +235,7 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):# we acknowledge the code here with llm_prompt_1 and llm_prompt_2 is overly complex and should be refactored.the_content: StuffContent+ rendered_llm_prompt: LLMPrompt | None = Noneif (Concept.are_concept_compatible(concept_1=output_stuff_spec.concept, concept_2=get_native_concept(NativeConceptCode.TEXT), strict=True)@@ -246,6 +247,7 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):output_structure_prompt=None,extra_params=llm_prompt_run_params.params,)+ rendered_llm_prompt = llm_prompt_1_for_texttry:generated_text: str = await content_generator.make_llm_text(job_metadata=job_metadata,@@ -309,6 +311,7 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):output_structure_prompt=output_structure_prompt,extra_params=llm_prompt_run_params.params,)+ rendered_llm_prompt = llm_prompt_1_for_objectthe_content = await self._llm_gen_object_stuff_content(job_metadata=job_metadata,pipe_run_params=pipe_run_params,@@ -333,6 +336,27 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):name=output_name,)+ # Capture execution data for the graph tracer+ execution_data_dict: dict[str, Any] = {+ "resolved_model": llm_setting_main.model,+ "resolved_model_for_object": llm_setting_for_object.model,+ "is_multiple_output": is_multiple_output,+ }+ execution_data_dict["rendered_system_prompt"] = rendered_llm_prompt.system_text+ execution_data_dict["rendered_user_prompt"] = rendered_llm_prompt.user_text+ if self.structuring_method is not None:+ execution_data_dict["structuring_method"] = str(self.structuring_method)+ if is_with_preliminary_text:+ execution_data_dict["structuring_path"] = "text_then_object"+ elif is_multiple_output:+ execution_data_dict["structuring_path"] = "object_list"+ else:+ output_is_text = Concept.are_concept_compatible(+ concept_1=output_stuff_spec.concept, concept_2=get_native_concept(NativeConceptCode.TEXT), strict=True+ )+ execution_data_dict["structuring_path"] = "text" if output_is_text else "object_direct"++ self._register_execution_data(job_metadata, execution_data_dict)return PipeLLMOutput(working_memory=working_memory,pipeline_run_id=job_metadata.pipeline_run_id,diff --git a/pipelex/pipe_operators/search/pipe_search.py b/pipelex/pipe_operators/search/pipe_search.pyindex cc389d77b..132e2be50 100644--- a/pipelex/pipe_operators/search/pipe_search.py+++ b/pipelex/pipe_operators/search/pipe_search.py@@ -1,4 +1,4 @@-from typing import TYPE_CHECKING, Literal+from typing import TYPE_CHECKING, Any, Literalfrom typing_extensions import override@@ -143,6 +143,14 @@ class PipeSearch(PipeOperator[PipeSearchOutput]):stuff=output_stuff,name=output_name,)+ # Capture execution data for the graph tracer+ execution_data_dict: dict[str, Any] = {+ "rendered_query": query_text,+ "resolved_model": search_setting.model,+ "is_structured_output": self.is_structured_output,+ }++ self._register_execution_data(job_metadata, execution_data_dict)return PipeSearchOutput(working_memory=working_memory,pipeline_run_id=job_metadata.pipeline_run_id,@@ -176,6 +184,12 @@ class PipeSearch(PipeOperator[PipeSearchOutput]):stuff=output_stuff,name=output_name,)+ execution_data_dict: dict[str, Any] = {+ "rendered_query": "mock",+ "resolved_model": "mock",+ "is_structured_output": self.is_structured_output,+ }+ self._register_execution_data(job_metadata, execution_data_dict)return PipeSearchOutput(working_memory=working_memory,pipeline_run_id=job_metadata.pipeline_run_id,diff --git a/pipelex/pipelex.toml b/pipelex/pipelex.tomlindex 1bdfeba4c..effe3f43f 100644--- a/pipelex/pipelex.toml+++ b/pipelex/pipelex.toml@@ -314,6 +314,7 @@ stuff_json_content = truestuff_text_content = truestuff_html_content = trueerror_stack_traces = true+pipe_and_concept_registry = true[pipelex.pipeline_execution_config.graph_config.graphs_inclusion]graphspec_json = truediff --git a/tests/unit/pipelex/graph/conftest.py b/tests/unit/pipelex/graph/conftest.pyindex d81555a84..b3fee0fb7 100644--- a/tests/unit/pipelex/graph/conftest.py+++ b/tests/unit/pipelex/graph/conftest.py@@ -12,6 +12,7 @@ def make_defaulted_data_inclusion_config(stuff_text_content: bool = False,stuff_html_content: bool = False,error_stack_traces: bool = False,+ pipe_and_concept_registry: bool = False,) -> DataInclusionConfig:"""Create a DataInclusionConfig for testing.@@ -20,6 +21,7 @@ def make_defaulted_data_inclusion_config(stuff_text_content: Whether to include plain text stuff data.stuff_html_content: Whether to include HTML stuff data.error_stack_traces: Whether to include error stack traces.+ pipe_and_concept_registry: Whether to include pipe and concept registries.Returns:A DataInclusionConfig configured for testing.@@ -29,6 +31,7 @@ def make_defaulted_data_inclusion_config(stuff_text_content=stuff_text_content,stuff_html_content=stuff_html_content,error_stack_traces=error_stack_traces,+ pipe_and_concept_registry=pipe_and_concept_registry,)