Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
refactor run pipe (#534)
Thomas Hebrard826a8d72ec21
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex e0661cd4b..c86861598 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -40,6 +40,7 @@- Validation for `PipeCompose` output concepts: must be strictly compatible with the Text concept- Validation for Pipelex Bundle concept keys: cannot create a native concept- Validation for `PipeSequence`: The multiplicity of the output of the sequence must be the same as the multiplicity of the output of the last step.+- Validation for `PipeFunc`: The multiplicity of the output of the pipe must be the same as the multiplicity of the output of the function: If the multiplicity is true, the return type of the function must be a subclass of `ListContent`. If the multiplicity is false, the return type of the function must not be a subclass of `ListContent`.### Changed@@ -84,6 +85,7 @@- `ConceptFactory.make_from_blueprint` method now correctly handles native concepts- The output (and inputs) of a pipe is now a `StuffSpec` object instead, that holds the concept and the multiplicity.- Renamed `domain` into `domain_code` when relevant.+- Refactored the dry run methods of the `PipeAbstract` class.## [v0.17.3] - 2025-12-01diff --git a/pipelex/core/pipes/inputs/exceptions.py b/pipelex/core/pipes/inputs/exceptions.pyindex 8934702be..5bb87b8db 100644--- a/pipelex/core/pipes/inputs/exceptions.py+++ b/pipelex/core/pipes/inputs/exceptions.py@@ -35,7 +35,7 @@ class PipeRunInputsError(PipeRunError):message: str,run_mode: PipeRunMode,pipe_code: str,- missing_inputs: dict[str, str],+ missing_inputs: list[str] | None = None,variable_name: str | None = None,concept_code: str | None = None,):diff --git a/pipelex/core/pipes/pipe_abstract.py b/pipelex/core/pipes/pipe_abstract.pyindex 4027a9073..2f6d57dfa 100644--- a/pipelex/core/pipes/pipe_abstract.py+++ b/pipelex/core/pipes/pipe_abstract.py@@ -17,7 +17,6 @@ from pipelex.core.pipes.pipe_output import PipeOutputfrom pipelex.core.pipes.stuff_spec.stuff_spec import StuffSpecfrom pipelex.pipe_run.pipe_run_mode import PipeRunModefrom pipelex.pipe_run.pipe_run_params import PipeRunParams-from pipelex.pipeline.exceptions import PipeStackOverflowErrorfrom pipelex.pipeline.job_metadata import JobMetadata, OtelContextfrom pipelex.pipeline.pipeline_factory import PipelineFactoryfrom pipelex.system.telemetry.otel_constants import (@@ -33,6 +32,8 @@ from pipelex.tools.misc.package_utils import get_package_versionfrom pipelex.tools.misc.string_utils import is_snake_casefrom pipelex.types import Self+PipeAbstractType = type["PipeAbstract"]+class PipeAbstract(ABC, BaseModel):model_config = ConfigDict(strict=True, extra="forbid")@@ -229,21 +230,66 @@ class PipeAbstract(ABC, BaseModel):def validate_output_static(self):pass+ @finalasync def validate_before_run(self,job_metadata: JobMetadata,working_memory: WorkingMemory,pipe_run_params: PipeRunParams,output_name: str | None = None,- ): ...+ ):+ # Check that all the needed inputs are actually in the working memory+ missing_input_names: list[str] = []++ for named_stuff_spec in self.needed_inputs().named_stuff_specs:+ if not working_memory.get_optional_stuff(named_stuff_spec.variable_name):+ missing_input_names.append(named_stuff_spec.variable_name)++ if missing_input_names:+ msg = f"Dry run of {self.type} '{self.code}': missing required inputs: {', '.join(missing_input_names)}"+ raise PipeRunInputsError(+ message=msg,+ run_mode=pipe_run_params.run_mode,+ pipe_code=self.code,+ missing_inputs=missing_input_names,+ )++ # Specific pipe validation function+ await self._validate_before_run(+ job_metadata=job_metadata, working_memory=working_memory, pipe_run_params=pipe_run_params, output_name=output_name+ )+ @abstractmethod+ async def _validate_before_run(+ self,+ job_metadata: JobMetadata,+ working_memory: WorkingMemory,+ pipe_run_params: PipeRunParams,+ output_name: str | None = None,+ ):+ pass++ @finalasync def validate_after_run(self,job_metadata: JobMetadata,working_memory: WorkingMemory,pipe_run_params: PipeRunParams,output_name: str | None = None,- ): ...+ ):+ await self._validate_after_run(+ job_metadata=job_metadata, working_memory=working_memory, pipe_run_params=pipe_run_params, output_name=output_name+ )++ @abstractmethod+ async def _validate_after_run(+ self,+ job_metadata: JobMetadata,+ working_memory: WorkingMemory,+ pipe_run_params: PipeRunParams,+ output_name: str | None = None,+ ):+ pass@abstractmethoddef required_variables(self) -> set[str]:@@ -269,13 +315,6 @@ class PipeAbstract(ABC, BaseModel):"""- def monitor_pipe_stack(self, pipe_run_params: PipeRunParams):- pipe_stack = pipe_run_params.pipe_stack- limit = pipe_run_params.pipe_stack_limit- if len(pipe_stack) > limit:- msg = f"Exceeded pipe stack limit of {limit}. You can raise that limit in the config. Stack:\n{pipe_stack}"- raise PipeStackOverflowError(message=msg, limit=limit, pipe_stack=pipe_stack)-def _format_pipe_run_info(self, pipe_run_params: PipeRunParams) -> str:indent_level = len(pipe_run_params.pipe_stack) - 1indent = " " * indent_level@@ -292,8 +331,135 @@ class PipeAbstract(ABC, BaseModel):arrow = "[yellow]→[/yellow]"return f"{indent}{pipe_type_label} {pipe_code_label} {arrow} {concept_code_label}"+ @final+ async def run_pipe(+ self,+ job_metadata: JobMetadata,+ working_memory: WorkingMemory,+ pipe_run_params: PipeRunParams,+ output_name: str | None = None,+ ) -> PipeOutput:+ pipe_run_params.push_pipe_to_stack(pipe_code=self.code)++ match pipe_run_params.run_mode:+ case PipeRunMode.LIVE:+ pipe_output = await self.live_run_pipe(+ job_metadata=job_metadata,+ working_memory=working_memory,+ pipe_run_params=pipe_run_params,+ output_name=output_name,+ )+ case PipeRunMode.DRY:+ pipe_output = await self.dry_run_pipe(+ job_metadata=job_metadata,+ working_memory=working_memory,+ pipe_run_params=pipe_run_params,+ output_name=output_name,+ )++ pipe_run_params.pop_pipe_from_stack(pipe_code=self.code)+ return pipe_output++ @final+ async def live_run_pipe(+ self,+ job_metadata: JobMetadata,+ working_memory: WorkingMemory,+ pipe_run_params: PipeRunParams,+ output_name: str | None = None,+ ) -> PipeOutput:+ log.info(self._format_pipe_run_info(pipe_run_params=pipe_run_params))++ # Handle telemetry ------------------------------------------------------------++ # Generate pipe_run_id (business ID, always set)+ this_pipe_run_id = PipelineFactory.make_pipe_run_id()++ # Derive OtelContext if telemetry is enabled (not dry mode and tracer available)+ # The trace_id comes from parent's otel_context (already computed at pipeline start)+ this_otel_context: OtelContext | None = None+ span: Span | None = None+ is_root_span: bool = False++ parent_otel_context = job_metadata.otel_context+ if not pipe_run_params.run_mode.is_dry and parent_otel_context is not None:+ # Start OTel span first+ span, is_root_span = self._start_pipe_span(+ parent_otel_context=parent_otel_context,+ pipeline_run_id=job_metadata.pipeline_run_id,+ working_memory=working_memory,+ )+ # Get the actual span_id from OTel (OTel generates its own span_id)+ if span:+ span_context = span.get_span_context()+ this_otel_context = OtelContext(+ trace_id=parent_otel_context.trace_id,+ trace_name=parent_otel_context.trace_name,+ trace_name_redacted=parent_otel_context.trace_name_redacted,+ span_id=span_context.span_id,+ )++ # Create child metadata with updated pipe_code and pipe_run_id+ # This passes down a modified copy rather than mutating the original+ # otel_context is passed separately because it must always be set explicitly+ # (even when None in dry mode) to avoid inheriting stale parent context+ child_metadata = job_metadata.copy_with_update(+ updated_metadata=JobMetadata(+ pipe_code=self.code,+ pipe_run_id=this_pipe_run_id,+ ),+ otel_context=this_otel_context,+ )++ # Run pipe ------------------------------------------------------------++ try:+ pipe_output = await self._live_run_pipe(+ job_metadata=child_metadata, working_memory=working_memory, pipe_run_params=pipe_run_params, output_name=output_name+ )+ except Exception as exc:+ self._end_pipe_span_error(span, error=exc, is_root_span=is_root_span)+ raise++ # Handle telemetry ------------------------------------------------------------++ self._end_pipe_span_success(span=span, pipe_output=pipe_output, is_root_span=is_root_span)++ return pipe_output++ @final+ async def dry_run_pipe(+ self,+ job_metadata: JobMetadata,+ working_memory: WorkingMemory,+ pipe_run_params: PipeRunParams,+ output_name: str | None = None,+ ) -> PipeOutput:+ log.verbose(f"Dry run of {self.type}: '{self.code}'")+ assert pipe_run_params.run_mode.is_dry, f"Dry run of {self.type} '{self.code}' called with run_mode = {pipe_run_params.run_mode}"+ await self.validate_before_run(+ job_metadata=job_metadata, working_memory=working_memory, pipe_run_params=pipe_run_params, output_name=output_name+ )+ pipe_output = await self._dry_run_pipe(+ job_metadata=job_metadata, working_memory=working_memory, pipe_run_params=pipe_run_params, output_name=output_name+ )+ await self.validate_after_run(+ job_metadata=job_metadata, working_memory=working_memory, pipe_run_params=pipe_run_params, output_name=output_name+ )+ return pipe_output+@abstractmethod- async def _run_pipe(+ async def _live_run_pipe(+ self,+ job_metadata: JobMetadata,+ working_memory: WorkingMemory,+ pipe_run_params: PipeRunParams,+ output_name: str | None = None,+ ) -> PipeOutput:+ pass++ @abstractmethod+ async def _dry_run_pipe(self,job_metadata: JobMetadata,working_memory: WorkingMemory,@@ -475,104 +641,3 @@ class PipeAbstract(ABC, BaseModel):if is_root_span:span.set_attribute(LangfuseSpanAttr.TRACE_OUTCOME, SpanOutcome.FAILURE)span.end()-- @final- async def run_pipe(- self,- job_metadata: JobMetadata,- working_memory: WorkingMemory,- pipe_run_params: PipeRunParams,- output_name: str | None = None,- ) -> PipeOutput:- pipe_run_params.push_pipe_to_stack(pipe_code=self.code)- self.monitor_pipe_stack(pipe_run_params=pipe_run_params)-- # Check inputs -------------------------------------------------------------- # check we have the required inputs in the working memory- missing_inputs: dict[str, str] = {}- for required_stuff_name, stuff_spec in self.needed_inputs().items:- if not working_memory.is_stuff_exists(name=required_stuff_name):- missing_inputs[required_stuff_name] = stuff_spec.concept.code- if missing_inputs:- error = PipeRunInputsError(- message=f"Missing required inputs for pipe '{self.code}': {missing_inputs}",- run_mode=pipe_run_params.run_mode,- pipe_code=self.code,- missing_inputs=missing_inputs,- )- raise error-- pipe_run_info = self._format_pipe_run_info(pipe_run_params=pipe_run_params)- if pipe_run_params.run_mode == PipeRunMode.LIVE:- log.info(pipe_run_info)-- # Handle telemetry -------------------------------------------------------------- # Generate pipe_run_id (business ID, always set)- this_pipe_run_id = PipelineFactory.make_pipe_run_id()-- # Derive OtelContext if telemetry is enabled (not dry mode and tracer available)- # The trace_id comes from parent's otel_context (already computed at pipeline start)- this_otel_context: OtelContext | None = None- span: Span | None = None- is_root_span: bool = False-- parent_otel_context = job_metadata.otel_context- if not pipe_run_params.run_mode.is_dry and parent_otel_context is not None:- # Start OTel span first- span, is_root_span = self._start_pipe_span(- parent_otel_context=parent_otel_context,- pipeline_run_id=job_metadata.pipeline_run_id,- working_memory=working_memory,- )- # Get the actual span_id from OTel (OTel generates its own span_id)- if span:- span_context = span.get_span_context()- this_otel_context = OtelContext(- trace_id=parent_otel_context.trace_id,- trace_name=parent_otel_context.trace_name,- trace_name_redacted=parent_otel_context.trace_name_redacted,- span_id=span_context.span_id,- )-- # Create child metadata with updated pipe_code and pipe_run_id- # This passes down a modified copy rather than mutating the original- # otel_context is passed separately because it must always be set explicitly- # (even when None in dry mode) to avoid inheriting stale parent context- child_metadata = job_metadata.copy_with_update(- updated_metadata=JobMetadata(- pipe_code=self.code,- pipe_run_id=this_pipe_run_id,- ),- otel_context=this_otel_context,- )-- # Run pipe -------------------------------------------------------------- try:- await self.validate_before_run(- job_metadata=child_metadata, working_memory=working_memory, pipe_run_params=pipe_run_params, output_name=output_name- )- pipe_output = await self._run_pipe(- job_metadata=child_metadata, working_memory=working_memory, pipe_run_params=pipe_run_params, output_name=output_name- )- await self.validate_after_run(- job_metadata=child_metadata, working_memory=working_memory, pipe_run_params=pipe_run_params, output_name=output_name- )- except Exception as exc:- self._end_pipe_span_error(span, error=exc, is_root_span=is_root_span)- raise-- # Handle telemetry -------------------------------------------------------------- self._end_pipe_span_success(span=span, pipe_output=pipe_output, is_root_span=is_root_span)-- # Cleanup -------------------------------------------------------------- pipe_run_params.pop_pipe_from_stack(pipe_code=self.code)-- return pipe_output---PipeAbstractType = type[PipeAbstract]diff --git a/pipelex/core/pipes/stuff_spec/stuff_spec.py b/pipelex/core/pipes/stuff_spec/stuff_spec.pyindex 657004f60..f7711af5c 100644--- a/pipelex/core/pipes/stuff_spec/stuff_spec.py+++ b/pipelex/core/pipes/stuff_spec/stuff_spec.py@@ -14,3 +14,10 @@ class StuffSpec(BaseModel):if isinstance(self.multiplicity, bool):return self.multiplicityreturn self.multiplicity > 1++ def to_bundle_representation(self) -> str:+ if self.multiplicity is None:+ return self.concept.concept_ref+ if isinstance(self.multiplicity, bool):+ return f"{self.concept.concept_ref}[]"+ return f"{self.concept.concept_ref}[{self.multiplicity}]"diff --git a/pipelex/core/pipes/variable_multiplicity.py b/pipelex/core/pipes/variable_multiplicity.pyindex 57b0989ad..c8e85882a 100644--- a/pipelex/core/pipes/variable_multiplicity.py+++ b/pipelex/core/pipes/variable_multiplicity.py@@ -58,7 +58,7 @@ class MultiplicityParseResult:self.multiplicity: int | bool | None = multiplicity-def parse_concept_with_multiplicity(concept_spec: str) -> MultiplicityParseResult:+def parse_concept_with_multiplicity(concept_ref: str) -> MultiplicityParseResult:"""Parse a concept specification string to extract concept and multiplicity.Supported formats:@@ -70,7 +70,7 @@ def parse_concept_with_multiplicity(concept_spec: str) -> MultiplicityParseResul- "domain.ConceptName[5]" -> (domain.ConceptName, 5)Args:- concept_spec: Concept specification string with optional multiplicity brackets+ concept_ref: Concept specification string with optional multiplicity bracketsReturns:MultiplicityParseResult with concept (without brackets) and multiplicity value@@ -82,11 +82,11 @@ def parse_concept_with_multiplicity(concept_spec: str) -> MultiplicityParseResul# Use strict pattern to validate identifier syntax# Concept must start with letter/underscore, optional domain prefix, optional bracketspattern = r"^([a-zA-Z_][a-zA-Z0-9_]*(?:\.[a-zA-Z_][a-zA-Z0-9_]*)?)(?:\[(\d*)\])?$"- match = re.match(pattern, concept_spec)+ match = re.match(pattern, concept_ref)if not match:msg = (- f"Invalid concept specification syntax: '{concept_spec}'. "+ f"Invalid concept specification syntax: '{concept_ref}'. "f"Expected format: 'ConceptName', 'ConceptName[]', 'ConceptName[N]', "f"'domain.ConceptName', 'domain.ConceptName[]', or 'domain.ConceptName[N]' "f"where concept and domain names must start with a letter or underscore."@@ -107,7 +107,7 @@ def parse_concept_with_multiplicity(concept_spec: str) -> MultiplicityParseResul# Number in brackets [N] - fixed countmultiplicity = int(bracket_content)if multiplicity <= 0:- msg = f"Invalid multiplicity value in '{concept_spec}': multiplicity must be at least 1. A pipe must produce at least one output."+ msg = f"Invalid multiplicity value in '{concept_ref}': multiplicity must be at least 1. A pipe must produce at least one output."raise PipeVariableMultiplicityError(msg)return MultiplicityParseResult(concept=concept, multiplicity=multiplicity)diff --git a/pipelex/core/stuffs/stuff_factory.py b/pipelex/core/stuffs/stuff_factory.pyindex aedd360e9..b96096cbb 100644--- a/pipelex/core/stuffs/stuff_factory.py+++ b/pipelex/core/stuffs/stuff_factory.py@@ -35,6 +35,10 @@ class StuffBlueprint(BaseModel):class StuffFactory:+ @classmethod+ def make_stuff_code(cls) -> str:+ return shortuuid.uuid()[:5]+@classmethoddef make_stuff_name(cls, concept: Concept) -> str:return Stuff.make_stuff_name(concept=concept)@@ -65,13 +69,17 @@ class StuffFactory:name: str | None = None,code: str | None = None,) -> Stuff:+ if not code:+ code = cls.make_stuff_code()+if not name:name = cls.make_stuff_name(concept=concept)+return Stuff(concept=concept,content=content,stuff_name=name,- stuff_code=code or shortuuid.uuid()[:5],+ stuff_code=code,)@classmethoddiff --git a/pipelex/pipe_controllers/batch/pipe_batch.py b/pipelex/pipe_controllers/batch/pipe_batch.pyindex 4a3cbd0fb..0dd28f444 100644--- a/pipelex/pipe_controllers/batch/pipe_batch.py+++ b/pipelex/pipe_controllers/batch/pipe_batch.py@@ -1,7 +1,6 @@import asynciofrom typing import TYPE_CHECKING, Any, Literal, cast-import shortuuidfrom typing_extensions import overridefrom pipelex.config import get_config@@ -72,7 +71,7 @@ class PipeBatch(PipeController):pass@override- async def validate_before_run(+ async def _validate_before_run(self, job_metadata: JobMetadata, working_memory: WorkingMemory, pipe_run_params: PipeRunParams, output_name: str | None = None) -> None:batch_params = pipe_run_params.batch_params or self.batch_params or BatchParams.make_default()@@ -93,20 +92,19 @@ class PipeBatch(PipeController):)raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode, pipe_code=self.code)- async def _run_batch_pipe(+ @override+ async def _live_run_controller_pipe(self,job_metadata: JobMetadata,working_memory: WorkingMemory,pipe_run_params: PipeRunParams,output_name: str | None = None,) -> PipeOutput:- """Common logic for running or dry-running a pipe in batch mode."""batch_params = pipe_run_params.batch_params or self.batch_params or BatchParams.make_default()input_item_stuff_name = batch_params.input_item_stuff_nameinput_list_stuff_name = batch_params.input_list_stuff_nameif pipe_run_params.final_stuff_code:- method_name = "dry_run_pipe" if pipe_run_params.run_mode == PipeRunMode.DRY else "_run_controller_pipe"pipe_run_params.final_stuff_code = Nonepipe_run_params.push_pipe_layer(pipe_code=self.branch_pipe_code)@@ -117,7 +115,7 @@ class PipeBatch(PipeController):# TODO: Make commented code work when inputing images named "a.b.c"sub_pipe = get_required_pipe(pipe_code=self.branch_pipe_code)nb_history_items_limit = get_config().pipelex.tracker_config.applied_nb_items_limit- batch_output_stuff_code = shortuuid.uuid()+ batch_output_stuff_code = StuffFactory.make_stuff_code()tasks: list[Coroutine[Any, Any, PipeOutput]] = []item_stuffs: list[Stuff] = []required_stuff_lists: list[list[Stuff]] = []@@ -176,21 +174,19 @@ class PipeBatch(PipeController):pipe_outputs = await asyncio.gather(*tasks)output_items: list[StuffContent] = []+output_stuffs: list[Stuff] = []- output_stuff_code = shortuuid.uuid()[:5]+for pipe_output in pipe_outputs:branch_output_stuff = pipe_output.main_stuff- output_stuffs.append(branch_output_stuff)output_items.append(branch_output_stuff.content)list_content: ListContent[StuffContent] = ListContent(items=output_items)output_stuff = StuffFactory.make_stuff(- code=output_stuff_code,concept=self.output.concept,content=list_content,name=output_name,)-method_name = "dry_run_pipe" if pipe_run_params.run_mode == PipeRunMode.DRY else "run_pipe"for branch_index, (required_stuff_list,@@ -235,14 +231,14 @@ class PipeBatch(PipeController):)@override- async def _run_controller_pipe(+ async def _dry_run_controller_pipe(self,job_metadata: JobMetadata,working_memory: WorkingMemory,pipe_run_params: PipeRunParams,output_name: str | None = None,) -> PipeOutput:- return await self._run_batch_pipe(+ return await self._live_run_controller_pipe(job_metadata=job_metadata,working_memory=working_memory,pipe_run_params=pipe_run_params,@@ -250,16 +246,7 @@ class PipeBatch(PipeController):)@override- async def _dry_run_controller_pipe(- self,- job_metadata: JobMetadata,- working_memory: WorkingMemory,- pipe_run_params: PipeRunParams,- output_name: str | None = None,- ) -> PipeOutput:- return await self._run_batch_pipe(- job_metadata=job_metadata,- working_memory=working_memory,- pipe_run_params=pipe_run_params,- output_name=output_name,- )+ async def _validate_after_run(+ self, job_metadata: JobMetadata, working_memory: WorkingMemory, pipe_run_params: PipeRunParams, output_name: str | None = None+ ):+ passdiff --git a/pipelex/pipe_controllers/condition/pipe_condition.py b/pipelex/pipe_controllers/condition/pipe_condition.pyindex 018d47598..cf7c737a3 100644--- a/pipelex/pipe_controllers/condition/pipe_condition.py+++ b/pipelex/pipe_controllers/condition/pipe_condition.py@@ -7,8 +7,7 @@ from pipelex import logfrom pipelex.cogt.templating.template_category import TemplateCategoryfrom pipelex.core.concepts.concept_factory import ConceptFactoryfrom pipelex.core.concepts.native.concept_native import NativeConceptCode-from pipelex.core.memory.exceptions import WorkingMemoryStuffNotFoundError-from pipelex.core.memory.working_memory import WorkingMemory+from pipelex.core.memory.working_memory import WorkingMemory, WorkingMemoryStuffNotFoundErrorfrom pipelex.core.pipes.exceptions import PipeValidationError, PipeValidationErrorTypefrom pipelex.core.pipes.inputs.input_stuff_specs import InputStuffSpecsfrom pipelex.core.pipes.inputs.input_stuff_specs_factory import InputStuffSpecsFactory@@ -84,7 +83,7 @@ class PipeCondition(PipeController):needed_inputs = InputStuffSpecsFactory.make_empty()- # 1. Add the variables from the expression/expression_template+ # Add the variables from the expression/expression_templaterequired_variables = detect_jinja2_required_variables(template_category=TemplateCategory.EXPRESSION,template_source=self.expression,@@ -101,7 +100,7 @@ class PipeCondition(PipeController):),)- # 2. Add the inputs needed by all possible target pipes+ # Add the inputs needed by all possible target pipesfor pipe_code in self.mapped_pipe_codes:pipe = get_required_pipe(pipe_code=pipe_code)# Use the centralized recursion detection@@ -150,7 +149,7 @@ class PipeCondition(PipeController):# TODO: Restore this validation. The problem lies with needed_inputs that construct Anything concepts.# @override- # async def validate_before_run(+ # async def _validate_before_run(# self,# job_metadata: JobMetadata,# working_memory: WorkingMemory,@@ -197,20 +196,14 @@ class PipeCondition(PipeController):return evaluated_expression@override- async def _run_controller_pipe(+ async def _live_run_controller_pipe(self,job_metadata: JobMetadata,working_memory: WorkingMemory,pipe_run_params: PipeRunParams,output_name: str | None = None,) -> PipeOutput:- log.verbose(f"{self.class_name} generating a '{self.output.concept.code}'")-- # TODO: restore pipe_layer feature- # pipe_run_params.push_pipe_code(pipe_code=pipe_code)-evaluated_expression = await self._evaluate_expression(working_memory=working_memory)-# Select the outcome based on the evaluated expressionoutcome = self.outcome_map.get(evaluated_expression, self.default_outcome)@@ -225,7 +218,6 @@ class PipeCondition(PipeController):chosen_pipe = get_required_pipe(pipe_code=outcome)- # Create condition details for trackingcondition_details = self._make_pipe_condition_details(evaluated_expression=self.expression,chosen_pipe_code=chosen_pipe.code,@@ -243,7 +235,6 @@ class PipeCondition(PipeController):msg = f"Some required stuff(s) not found: {error_details}"raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode, pipe_code=self.code) from exc- # Track condition stepsfor required_stuff in required_stuffs:get_pipeline_tracker().add_condition_step(from_stuff=required_stuff,@@ -253,19 +244,15 @@ class PipeCondition(PipeController):comment="PipeCondition required for condition",)- # Execute the chosen pipe- log.verbose(f"Chosen pipe: {chosen_pipe.code}")pipe_output = await get_pipe_router().run(pipe_job=PipeJobFactory.make_pipe_job(- pipe=chosen_pipe,+ pipe=get_required_pipe(pipe_code=outcome),job_metadata=job_metadata,working_memory=working_memory,pipe_run_params=pipe_run_params,output_name=output_name,),)-- # Track choice stepget_pipeline_tracker().add_choice_step(from_condition=condition_details,to_stuff=pipe_output.main_stuff,@@ -282,27 +269,7 @@ class PipeCondition(PipeController):pipe_run_params: PipeRunParams,output_name: str | None = None,) -> PipeOutput:- """Dry run implementation for PipeCondition.- Validates that all required inputs are present, expression is valid, and target pipes exist.- """- log.verbose(f"PipeCondition: dry run controller pipe: {self.code}")- # 1. Validate that all required inputs are present in the working memory- needed_inputs = self.needed_inputs()- missing_input_names: list[str] = []-- for named_stuff_spec in needed_inputs.named_stuff_specs:- if not working_memory.get_optional_stuff(named_stuff_spec.variable_name):- missing_input_names.append(named_stuff_spec.variable_name)-- if missing_input_names:- log.error(f"Dry run failed: missing required inputs: {missing_input_names}")- raise PipeRunError(- message=f"Dry run failed for pipe '{self.code}' (PipeCondition): missing required inputs: {', '.join(missing_input_names)}",- run_mode=pipe_run_params.run_mode,- pipe_code=self.code,- )-- # 2. Validate that the expression template is valid+ # Validate that the expression template is validtry:required_variables = detect_jinja2_required_variables(template_category=TemplateCategory.EXPRESSION,@@ -317,7 +284,7 @@ class PipeCondition(PipeController):)raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode, pipe_code=self.code) from exc- # 3. Validate that all values in the outcomes map (appart from special outcomes) do exist as pipe codes+ # Validate that all values in the outcomes map (appart from special outcomes) do exist as pipe codesall_pipe_codes = set(self.outcome_map.values())if self.default_outcome:all_pipe_codes.add(self.default_outcome)@@ -341,3 +308,15 @@ class PipeCondition(PipeController):pipe_run_params=pipe_run_params,)return PipeOutput(working_memory=working_memory)++ @override+ async def _validate_before_run(+ self, job_metadata: JobMetadata, working_memory: WorkingMemory, pipe_run_params: PipeRunParams, output_name: str | None = None+ ):+ pass++ @override+ async def _validate_after_run(+ self, job_metadata: JobMetadata, working_memory: WorkingMemory, pipe_run_params: PipeRunParams, output_name: str | None = None+ ):+ passdiff --git a/pipelex/pipe_controllers/parallel/pipe_parallel.py b/pipelex/pipe_controllers/parallel/pipe_parallel.pyindex 27d7cd45c..dfe6409aa 100644--- a/pipelex/pipe_controllers/parallel/pipe_parallel.py+++ b/pipelex/pipe_controllers/parallel/pipe_parallel.py@@ -18,7 +18,6 @@ from pipelex.libraries.pipe.exceptions import PipeNotFoundErrorfrom pipelex.pipe_controllers.pipe_controller import PipeControllerfrom pipelex.pipe_controllers.sub_pipe import SubPipefrom pipelex.pipe_run.exceptions import PipeRunError-from pipelex.pipe_run.pipe_run_mode import PipeRunModefrom pipelex.pipe_run.pipe_run_params import PipeRunParamsfrom pipelex.pipeline.job_metadata import JobMetadatafrom pipelex.types import Self@@ -128,7 +127,7 @@ class PipeParallel(PipeController):return {sub_pipe.pipe_code for sub_pipe in self.parallel_sub_pipes}@override- async def _run_controller_pipe(+ async def _live_run_controller_pipe(self,job_metadata: JobMetadata,working_memory: WorkingMemory,@@ -189,6 +188,7 @@ class PipeParallel(PipeController):stuff=combined_output_stuff,name=output_name,)+for stuff in output_stuffs.values():get_pipeline_tracker().add_aggregate_step(from_stuff=stuff,@@ -196,6 +196,7 @@ class PipeParallel(PipeController):pipe_layer=pipe_run_params.pipe_layers,comment="PipeParallel on output_stuffs",)+return PipeOutput(working_memory=working_memory,pipeline_run_id=job_metadata.pipeline_run_id,@@ -209,26 +210,7 @@ class PipeParallel(PipeController):pipe_run_params: PipeRunParams,output_name: str | None = None,) -> PipeOutput:- """Dry run implementation for PipeParallel.- Validates that all required inputs are present and that all parallel sub-pipes can be dry run.- """- log.verbose(f"PipeParallel: dry run controller pipe: {self.code}")- if pipe_run_params.run_mode != PipeRunMode.DRY:- msg = f"PipeSequence._dry_run_controller_pipe() called with run_mode = {pipe_run_params.run_mode} in pipe {self.code}"- raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode, pipe_code=self.code)-- # 1. Validate that all required inputs are present in the working memory- needed_inputs = self.needed_inputs()- missing_input_names: list[str] = []- for named_stuff_spec in needed_inputs.named_stuff_specs:- if not working_memory.get_optional_stuff(named_stuff_spec.variable_name):- missing_input_names.append(named_stuff_spec.variable_name)-- if missing_input_names:- msg = f"Dry run failed for pipe '{self.code}' (PipeParallel): missing required inputs: {missing_input_names}"- raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode, pipe_code=self.code)-- # 2. Validate that all sub-pipes exist+ # 1. Validate that all sub-pipes existfor sub_pipe in self.parallel_sub_pipes:try:get_required_pipe(pipe_code=sub_pipe.pipe_code)@@ -236,7 +218,7 @@ class PipeParallel(PipeController):msg = f"Dry run failed for pipe '{self.code}' (PipeParallel): sub-pipe '{sub_pipe.pipe_code}' not found"raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode, pipe_code=self.code) from exc- # 3. Run all sub-pipes in dry mode+ # 2. Run all sub-pipes in dry modetasks: list[Coroutine[Any, Any, PipeOutput]] = []for sub_pipe in self.parallel_sub_pipes:@@ -251,7 +233,7 @@ class PipeParallel(PipeController):pipe_outputs = await asyncio.gather(*tasks)- # 4. Process outputs as in the regular run+ # 3. Process outputs as in the regular runoutput_stuffs: dict[str, Stuff] = {}output_stuff_contents: dict[str, StuffContent] = {}@@ -276,7 +258,7 @@ class PipeParallel(PipeController):output_stuffs[sub_pipe_output_name] = output_stuffoutput_stuff_contents[sub_pipe_output_name] = output_stuff.content- # 5. Handle combined output if specified+ # 4. Handle combined output if specifiedif self.combined_output:combined_output_stuff = StuffFactory.combine_stuffs(concept=self.combined_output,@@ -291,3 +273,15 @@ class PipeParallel(PipeController):working_memory=working_memory,pipeline_run_id=job_metadata.pipeline_run_id,)++ @override+ async def _validate_before_run(+ self, job_metadata: JobMetadata, working_memory: WorkingMemory, pipe_run_params: PipeRunParams, output_name: str | None = None+ ):+ pass++ @override+ async def _validate_after_run(+ self, job_metadata: JobMetadata, working_memory: WorkingMemory, pipe_run_params: PipeRunParams, output_name: str | None = None+ ):+ passdiff --git a/pipelex/pipe_controllers/pipe_controller.py b/pipelex/pipe_controllers/pipe_controller.pyindex c7ee46a30..444fc15c8 100644--- a/pipelex/pipe_controllers/pipe_controller.py+++ b/pipelex/pipe_controllers/pipe_controller.py@@ -6,7 +6,7 @@ from typing_extensions import overridefrom pipelex.core.memory.working_memory import WorkingMemoryfrom pipelex.core.pipes.pipe_abstract import PipeAbstractfrom pipelex.core.pipes.pipe_output import PipeOutput-from pipelex.pipe_run.pipe_run_params import PipeRunMode, PipeRunParams+from pipelex.pipe_run.pipe_run_params import PipeRunParamsfrom pipelex.pipeline.job_metadata import JobMetadata@@ -27,33 +27,28 @@ class PipeController(PipeAbstract):@final@override- async def _run_pipe(+ async def _live_run_pipe(self,job_metadata: JobMetadata,working_memory: WorkingMemory,pipe_run_params: PipeRunParams,output_name: str | None = None,- print_intermediate_outputs: bool | None = False,) -> PipeOutput:- match pipe_run_params.run_mode:- case PipeRunMode.LIVE:- pipe_output = await self._run_controller_pipe(- job_metadata=job_metadata,- working_memory=working_memory,- pipe_run_params=pipe_run_params,- output_name=output_name,- )- case PipeRunMode.DRY:- pipe_output = await self._dry_run_controller_pipe(- job_metadata=job_metadata,- working_memory=working_memory,- pipe_run_params=pipe_run_params,- output_name=output_name,- )- return pipe_output+ return await self._live_run_controller_pipe(+ job_metadata=job_metadata, working_memory=working_memory, pipe_run_params=pipe_run_params, output_name=output_name+ )++ @final+ @override+ async def _dry_run_pipe(+ self, job_metadata: JobMetadata, working_memory: WorkingMemory, pipe_run_params: PipeRunParams, output_name: str | None = None+ ) -> PipeOutput:+ return await self._dry_run_controller_pipe(+ job_metadata=job_metadata, working_memory=working_memory, pipe_run_params=pipe_run_params, output_name=output_name+ )@abstractmethod- async def _run_controller_pipe(+ async def _live_run_controller_pipe(self,job_metadata: JobMetadata,working_memory: WorkingMemory,diff --git a/pipelex/pipe_controllers/sequence/pipe_sequence.py b/pipelex/pipe_controllers/sequence/pipe_sequence.pyindex 3c8d3d410..b689158de 100644--- a/pipelex/pipe_controllers/sequence/pipe_sequence.py+++ b/pipelex/pipe_controllers/sequence/pipe_sequence.py@@ -11,12 +11,10 @@ from pipelex.core.pipes.inputs.input_stuff_specs_factory import InputStuffSpecsFfrom pipelex.core.pipes.pipe_output import PipeOutputfrom pipelex.core.pipes.variable_multiplicity import is_multiplicity_compatiblefrom pipelex.hub import get_concept_library, get_required_pipe-from pipelex.pipe_controllers.exceptions import PipeControllerOutputConceptMismatchErrorfrom pipelex.pipe_controllers.parallel.pipe_parallel import PipeParallelfrom pipelex.pipe_controllers.pipe_controller import PipeControllerfrom pipelex.pipe_controllers.sequence.exceptions import PipeSequenceValueErrorfrom pipelex.pipe_controllers.sub_pipe import SubPipe-from pipelex.pipe_run.exceptions import PipeRunError, PipeRunParamsErrorfrom pipelex.pipe_run.pipe_run_params import PipeRunParamsfrom pipelex.pipeline.job_metadata import JobMetadata@@ -158,18 +156,13 @@ class PipeSequence(PipeController):return {sub_pipe.pipe_code for sub_pipe in self.sequential_sub_pipes}@override- async def _run_controller_pipe(+ async def _live_run_controller_pipe(self,job_metadata: JobMetadata,working_memory: WorkingMemory,pipe_run_params: PipeRunParams,output_name: str | None = None,) -> PipeOutput:- pipe_run_params.push_pipe_layer(pipe_code=self.code)- if pipe_run_params.is_multiple_output_required:- msg = f"{self.__class__.__name__} does not support multiple outputs, got output_multiplicity = {pipe_run_params.output_multiplicity}"- raise PipeRunParamsError(msg)-evolving_memory = working_memoryfor sub_pipe_index, sub_pipe in enumerate(self.sequential_sub_pipes):@@ -198,24 +191,21 @@ class PipeSequence(PipeController):pipe_run_params: PipeRunParams,output_name: str | None = None,) -> PipeOutput:- if not pipe_run_params.run_mode.is_dry:- msg = f"PipeSequence._dry_run_controller_pipe() called with run_mode = {pipe_run_params.run_mode} in pipe {self.code}"- raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode, pipe_code=self.code)- # Verify the output of this pipe is matching the output of the last step.-- concept_of_last_step = get_required_pipe(pipe_code=self.sequential_sub_pipes[-1].pipe_code).output.concept- # if self.output.concept_ref != concept_ref_of_last_step:- if not get_concept_library().is_compatible(tested_concept=concept_of_last_step, wanted_concept=self.output.concept):- msg = f"""PipeSequence concept mismatch:-the output concept '{concept_of_last_step.concept_ref}' of the last step '{self.sequential_sub_pipes[-1].pipe_code}'-of sequence pipe '{self.code}' is not compatible with the output concept '{self.output.concept.concept_ref}' of the sequence.-"""- raise PipeControllerOutputConceptMismatchError(- message=msg, tested_concept=concept_of_last_step.concept_ref, wanted_concept=self.output.concept.concept_ref- )- return await self._run_controller_pipe(+ return await self._live_run_controller_pipe(job_metadata=job_metadata,working_memory=working_memory,pipe_run_params=pipe_run_params,output_name=output_name,)++ @override+ async def _validate_before_run(+ self, job_metadata: JobMetadata, working_memory: WorkingMemory, pipe_run_params: PipeRunParams, output_name: str | None = None+ ):+ pass++ @override+ async def _validate_after_run(+ self, job_metadata: JobMetadata, working_memory: WorkingMemory, pipe_run_params: PipeRunParams, output_name: str | None = None+ ):+ passdiff --git a/pipelex/pipe_controllers/sub_pipe.py b/pipelex/pipe_controllers/sub_pipe.pyindex f11d3d0a9..ff2736096 100644--- a/pipelex/pipe_controllers/sub_pipe.py+++ b/pipelex/pipe_controllers/sub_pipe.py@@ -29,7 +29,6 @@ class SubPipe(BaseModel):job_metadata: JobMetadata,sub_pipe_run_params: PipeRunParams,) -> PipeOutput:- """Run or dry run a single operation self."""if self.output_multiplicity:sub_pipe_run_params.output_multiplicity = self.output_multiplicity@@ -49,7 +48,6 @@ class SubPipe(BaseModel):message=msg,run_mode=sub_pipe_run_params.run_mode,pipe_code=self.pipe_code,- missing_inputs={},variable_name=batch_params.input_list_stuff_name,concept_code=None,) from exc@@ -65,7 +63,6 @@ class SubPipe(BaseModel):message=msg,run_mode=sub_pipe_run_params.run_mode,pipe_code=self.pipe_code,- missing_inputs={},variable_name=batch_params.input_item_stuff_name,concept_code=None,) from exc@@ -116,7 +113,6 @@ class SubPipe(BaseModel):message=msg,run_mode=sub_pipe_run_params.run_mode,pipe_code=self.pipe_code,- missing_inputs={},variable_name=exc.variable_name,concept_code=None,) from exc@@ -127,6 +123,7 @@ class SubPipe(BaseModel):pipe_run_params=sub_pipe_run_params,output_name=self.output_name,)+if new_output_stuff := pipe_output.working_memory.get_optional_main_stuff():for stuff in required_stuffs:get_pipeline_tracker().add_pipe_step(@@ -136,4 +133,5 @@ class SubPipe(BaseModel):pipe_layer=sub_pipe_run_params.pipe_layers,comment="SubPipe on required_stuff",)+return pipe_outputdiff --git a/pipelex/pipe_operators/compose/pipe_compose.py b/pipelex/pipe_operators/compose/pipe_compose.pyindex 52ccb4fbd..44b70fd0e 100644--- a/pipelex/pipe_operators/compose/pipe_compose.py+++ b/pipelex/pipe_operators/compose/pipe_compose.py@@ -19,7 +19,6 @@ from pipelex.core.stuffs.stuff_content import StuffContentfrom pipelex.core.stuffs.stuff_factory import StuffFactoryfrom pipelex.hub import get_class_registry, get_concept_library, get_content_generator, get_native_conceptfrom pipelex.pipe_operators.pipe_operator import PipeOperator-from pipelex.pipe_run.exceptions import PipeRunParamsErrorfrom pipelex.pipe_run.pipe_run_mode import PipeRunModefrom pipelex.pipe_run.pipe_run_params import PipeRunParamsfrom pipelex.pipe_run.pipe_run_params_factory import PipeRunParamsFactory@@ -102,7 +101,7 @@ class PipeCompose(PipeOperator[PipeComposeOutput]):)@override- async def _run_operator_pipe(+ async def _live_run_operator_pipe(self,job_metadata: JobMetadata,working_memory: WorkingMemory,@@ -111,9 +110,6 @@ class PipeCompose(PipeOperator[PipeComposeOutput]):content_generator: ContentGeneratorProtocol | None = None,) -> PipeComposeOutput:content_generator = content_generator or get_content_generator()- if pipe_run_params.is_multiple_output_required:- msg = f"PipeCompose does not suppport multiple outputs, got output_multiplicity = {pipe_run_params.output_multiplicity}"- raise PipeRunParamsError(msg)context: dict[str, Any] = working_memory.generate_context()if pipe_run_params:@@ -165,10 +161,22 @@ class PipeCompose(PipeOperator[PipeComposeOutput]):log.verbose(f"PipeCompose: using regular operator pipe for jinja2 rendering (dry run not applied to jinja2): {self.code}")content_generator_used = get_content_generator()- return await self._run_operator_pipe(+ return await self._live_run_operator_pipe(job_metadata=job_metadata,working_memory=working_memory,pipe_run_params=pipe_run_params or PipeRunParamsFactory.make_run_params(pipe_run_mode=PipeRunMode.DRY),output_name=output_name,content_generator=content_generator_used,)++ @override+ async def _validate_before_run(+ self, job_metadata: JobMetadata, working_memory: WorkingMemory, pipe_run_params: PipeRunParams, output_name: str | None = None+ ):+ pass++ @override+ async def _validate_after_run(+ self, job_metadata: JobMetadata, working_memory: WorkingMemory, pipe_run_params: PipeRunParams, output_name: str | None = None+ ):+ passdiff --git a/pipelex/pipe_operators/compose/pipe_compose_blueprint.py b/pipelex/pipe_operators/compose/pipe_compose_blueprint.pyindex ccee5f68a..a08e7e9c1 100644--- a/pipelex/pipe_operators/compose/pipe_compose_blueprint.py+++ b/pipelex/pipe_operators/compose/pipe_compose_blueprint.py@@ -7,6 +7,7 @@ from pipelex.cogt.templating.template_category import TemplateCategoryfrom pipelex.cogt.templating.template_preprocessor import preprocess_templatefrom pipelex.cogt.templating.templating_style import TemplatingStylefrom pipelex.core.pipes.pipe_blueprint import PipeBlueprint+from pipelex.core.pipes.variable_multiplicity import parse_concept_with_multiplicityfrom pipelex.tools.jinja2.jinja2_errors import Jinja2TemplateSyntaxErrorfrom pipelex.tools.jinja2.jinja2_parsing import check_jinja2_parsingfrom pipelex.tools.jinja2.jinja2_required_variables import detect_jinja2_required_variables@@ -70,4 +71,10 @@ class PipeComposeBlueprint(PipeBlueprint):@overridedef validate_output(self):- pass+ parsed_output = parse_concept_with_multiplicity(concept_ref=self.output)+ if parsed_output.multiplicity:+ msg = (+ "PipeCompose does not support multiple output generation. The output of PipeCompose must be a single stuff, "+ f"from a concept that refines the native Text concept. Current output: {self.output}"+ )+ raise ValueError(msg)diff --git a/pipelex/pipe_operators/extract/pipe_extract.py b/pipelex/pipe_operators/extract/pipe_extract.pyindex 170c10b70..4a9a78bd7 100644--- a/pipelex/pipe_operators/extract/pipe_extract.py+++ b/pipelex/pipe_operators/extract/pipe_extract.py@@ -28,8 +28,6 @@ from pipelex.hub import (get_native_concept,)from pipelex.pipe_operators.pipe_operator import PipeOperator-from pipelex.pipe_run.exceptions import PipeRunError-from pipelex.pipe_run.pipe_run_mode import PipeRunModefrom pipelex.pipe_run.pipe_run_params import PipeRunParamsfrom pipelex.pipeline.job_metadata import JobMetadatafrom pipelex.tools.pdf.pypdfium2_renderer import pypdfium2_renderer@@ -96,7 +94,7 @@ class PipeExtract(PipeOperator[PipeExtractOutput]):)@override- async def _run_operator_pipe(+ async def _live_run_operator_pipe(self,job_metadata: JobMetadata,working_memory: WorkingMemory,@@ -205,16 +203,22 @@ class PipeExtract(PipeOperator[PipeExtractOutput]):pipe_run_params: PipeRunParams,output_name: str | None = None,) -> PipeExtractOutput:- log.verbose(f"PipeExtract: dry run operator pipe: {self.code}")- if pipe_run_params.run_mode != PipeRunMode.DRY:- msg = f"Running pipe '{self.code}' (PipeExtract) _dry_run_operator_pipe() in non-dry mode is not allowed."- raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode, pipe_code=self.code)-- content_generator_dry = ContentGeneratorDry()- return await self._run_operator_pipe(+ return await self._live_run_operator_pipe(job_metadata=job_metadata,working_memory=working_memory,pipe_run_params=pipe_run_params,output_name=output_name,- content_generator=content_generator_dry,+ content_generator=ContentGeneratorDry(),)++ @override+ async def _validate_before_run(+ self, job_metadata: JobMetadata, working_memory: WorkingMemory, pipe_run_params: PipeRunParams, output_name: str | None = None+ ):+ pass++ @override+ async def _validate_after_run(+ self, job_metadata: JobMetadata, working_memory: WorkingMemory, pipe_run_params: PipeRunParams, output_name: str | None = None+ ):+ passdiff --git a/pipelex/pipe_operators/func/pipe_func.py b/pipelex/pipe_operators/func/pipe_func.pyindex ed65b4dab..4af575e2e 100644--- a/pipelex/pipe_operators/func/pipe_func.py+++ b/pipelex/pipe_operators/func/pipe_func.py@@ -69,18 +69,39 @@ class PipeFunc(PipeOperator[PipeFuncOutput]):@overridedef validate_output_with_library(self):- pass+ function = func_registry.get_required_function(self.function_name)+ return_type = get_type_hints(function).get("return")+ if return_type is None:+ msg = (+ f"PipeFunc '{self.code}' failed to validate output with library: The return type of the function is None. "+ "It should be a subclass of StuffContent."+ )+ raise TypeError(msg)+ if self.output.multiplicity and not issubclass(return_type, ListContent):+ msg = (+ f"PipeFunc '{self.code}' output multiplicity is '{self.output.multiplicity}', but the function '{self.function_name}' "+ f"return type {return_type} is not a subclass of ListContent. The output of your PipeFunc is "+ f"'{self.output.to_bundle_representation()}'. The return type of your function should be a subclass of ListContent."+ )+ raise TypeError(msg)+ if not self.output.multiplicity and issubclass(return_type, ListContent):+ msg = (+ f"PipeFunc '{self.code}' output multiplicity is '{self.output.multiplicity}', but the function '{self.function_name}' "+ f"return type {return_type} is a subclass of ListContent. The output of your PipeFunc is "+ f"'{self.output.concept.concept_ref}{self.output.to_bundle_representation()}' "+ f"when it should be '{self.output.concept.concept_ref}' (no multiplicity)."+ )+ raise TypeError(msg)@override- async def _run_operator_pipe(+ async def _live_run_operator_pipe(self,job_metadata: JobMetadata,working_memory: WorkingMemory,pipe_run_params: PipeRunParams,output_name: str | None = None,) -> PipeFuncOutput:- log.verbose(f"Applying function '{self.function_name}'")-+ log.verbose(f"Running PipeFunc with function '{self.function_name}'")function = func_registry.get_required_function(self.function_name)if asyncio.iscoroutinefunction(function):@@ -124,31 +145,10 @@ class PipeFunc(PipeOperator[PipeFuncOutput]):pipe_run_params: PipeRunParams,output_name: str | None = None,) -> PipeFuncOutput:- log.verbose(f"Dry run for PipeFunc '{self.function_name}'")-function = func_registry.get_required_function(self.function_name)-- # Check that all needed inputs are present in working memory- needed_inputs = self.needed_inputs()-- missing_input_names: list[str] = []- for named_stuff_spec in needed_inputs.named_stuff_specs:- if not working_memory.get_optional_stuff(named_stuff_spec.variable_name):- missing_input_names.append(named_stuff_spec.variable_name)- if missing_input_names:- msg = f"Dry run failed for pipe '{self.code}' (PipeFunc): missing required inputs: {missing_input_names}"- raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode, pipe_code=self.code)-return_type = get_type_hints(function).get("return")-if return_type is None:- msg = f"Dry run failed for pipe '{self.code}' (PipeFunc): function '{self.function_name}' has no return type annotation"- raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode, pipe_code=self.code)- if not issubclass(return_type, StuffContent):- msg = (- f"Dry run failed for pipe '{self.code}' (PipeFunc): "- f"function '{self.function_name}' return type {return_type} is not a subclass of StuffContent"- )+ msg = f"Dry run of {self.type} '{self.code}' failed: The return type of the function is None. It should be a subclass of StuffContent."raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode, pipe_code=self.code)# TODO: Support PipeFunc returning with multiplicity. Create an equivalent of TypedNamedInputRequirement for outputs.@@ -180,3 +180,27 @@ class PipeFunc(PipeOperator[PipeFuncOutput]):working_memory=working_memory,pipeline_run_id=job_metadata.pipeline_run_id,)++ @override+ async def _validate_before_run(+ self, job_metadata: JobMetadata, working_memory: WorkingMemory, pipe_run_params: PipeRunParams, output_name: str | None = None+ ):+ function = func_registry.get_required_function(self.function_name)+ return_type = get_type_hints(function).get("return")+ # TODO: this should not happend ever. The correct way to do this would be to have a unit test making sure+ # that the FuncRegistry DOES CALL the 'is_eligible_function' function, and this function should be unit tested.+ if return_type is None:+ msg = f"Dry run failed for {self.type} '{self.code}': function '{self.function_name}' has no return type annotation"+ raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode, pipe_code=self.code)+ if not issubclass(return_type, StuffContent):+ msg = (+ f"Dry run failed for pipe {self.type} '{self.code}': "+ f"function '{self.function_name}' return type {return_type} is not a subclass of StuffContent"+ )+ raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode, pipe_code=self.code)++ @override+ async def _validate_after_run(+ self, job_metadata: JobMetadata, working_memory: WorkingMemory, pipe_run_params: PipeRunParams, output_name: str | None = None+ ):+ passdiff --git a/pipelex/pipe_operators/img_gen/pipe_img_gen.py b/pipelex/pipe_operators/img_gen/pipe_img_gen.pyindex 1f65309fb..123c77670 100644--- a/pipelex/pipe_operators/img_gen/pipe_img_gen.py+++ b/pipelex/pipe_operators/img_gen/pipe_img_gen.py@@ -99,7 +99,7 @@ class PipeImgGen(PipeOperator[PipeImgGenOutput]):)@override- async def _run_operator_pipe(+ async def _live_run_operator_pipe(self,job_metadata: JobMetadata,working_memory: WorkingMemory,@@ -187,7 +187,7 @@ class PipeImgGen(PipeOperator[PipeImgGenOutput]):else:nb_images = 1- # Get the structure class from the registry (might be a subclass of ImageContent)+ # Get the structure class from the registry (Has to be a subclass of ImageContent)structure_class = get_class_registry().get_required_subclass(name=self.output.concept.structure_class_name,base_class=ImageContent,@@ -238,7 +238,7 @@ class PipeImgGen(PipeOperator[PipeImgGenOutput]):source_prompt=img_gen_prompt_text,base_64=base_64_str,)- log.verbose(the_content, title="Single image content")+ log.verbose(the_content, title=f"output stuff content of PipeImg {self.code}")output_stuff = StuffFactory.make_stuff(name=output_name,@@ -264,11 +264,22 @@ class PipeImgGen(PipeOperator[PipeImgGenOutput]):pipe_run_params: PipeRunParams,output_name: str | None = None,) -> PipeImgGenOutput:- content_generator_dry = ContentGeneratorDry()- return await self._run_operator_pipe(+ return await self._live_run_operator_pipe(job_metadata=job_metadata,working_memory=working_memory,pipe_run_params=pipe_run_params or PipeRunParamsFactory.make_run_params(pipe_run_mode=PipeRunMode.DRY),output_name=output_name,- content_generator=content_generator_dry,+ content_generator=ContentGeneratorDry(),)++ @override+ async def _validate_before_run(+ self, job_metadata: JobMetadata, working_memory: WorkingMemory, pipe_run_params: PipeRunParams, output_name: str | None = None+ ):+ pass++ @override+ async def _validate_after_run(+ self, job_metadata: JobMetadata, working_memory: WorkingMemory, pipe_run_params: PipeRunParams, output_name: str | None = None+ ):+ passdiff --git a/pipelex/pipe_operators/llm/helpers.py b/pipelex/pipe_operators/llm/helpers.pynew file mode 100644index 000000000..4d7b39a9e--- /dev/null+++ b/pipelex/pipe_operators/llm/helpers.py@@ -0,0 +1,35 @@+from pipelex.cogt.templating.template_category import TemplateCategory+from pipelex.config import get_config+from pipelex.core.stuffs.stuff_content import StuffContent+from pipelex.hub import (+ get_class_registry,+ get_content_generator,+ get_required_concept,+)+from pipelex.tools.typing.structure_printer import StructurePrinter+++async def get_output_structure_prompt(concept_ref: str, is_with_preliminary_text: bool) -> str | None:+ concept = get_required_concept(concept_ref=concept_ref)+ output_class = get_class_registry().get_class(concept.structure_class_name)+ if not output_class:+ return None++ class_structure = StructurePrinter().get_type_structure(tp=output_class, base_class=StuffContent)++ if not class_structure:+ return None+ class_structure_str = "\n".join(class_structure)+ llm_config = get_config().cogt.llm_config+ if is_with_preliminary_text:+ template_source = llm_config.get_template(template_name="output_structure_prompt")+ else:+ template_source = llm_config.get_template(template_name="output_structure_prompt_no_preliminary_text")++ return await get_content_generator().make_templated_text(+ context={+ "class_structure_str": class_structure_str,+ },+ template=template_source,+ template_category=TemplateCategory.LLM_PROMPT,+ )diff --git a/pipelex/pipe_operators/llm/pipe_llm.py b/pipelex/pipe_operators/llm/pipe_llm.pyindex 489929113..c6b964ed5 100644--- a/pipelex/pipe_operators/llm/pipe_llm.py+++ b/pipelex/pipe_operators/llm/pipe_llm.py@@ -12,7 +12,6 @@ from pipelex.cogt.llm.llm_prompt_factory_abstract import LLMPromptFactoryAbstracfrom pipelex.cogt.llm.llm_prompt_template import LLMPromptTemplatefrom pipelex.cogt.llm.llm_setting import LLMModelChoice, LLMSetting, LLMSettingChoicesfrom pipelex.cogt.models.model_deck_check import check_llm_choice_with_deck-from pipelex.cogt.templating.template_category import TemplateCategoryfrom pipelex.config import get_configfrom pipelex.core.concepts.concept import Conceptfrom pipelex.core.concepts.concept_factory import ConceptFactory@@ -21,7 +20,6 @@ from pipelex.core.domains.domain import SpecialDomainfrom pipelex.core.memory.working_memory import WorkingMemoryfrom pipelex.core.pipes.exceptions import PipeValidationError, PipeValidationErrorTypefrom pipelex.core.pipes.inputs.input_stuff_specs import InputStuffSpecs-from pipelex.core.pipes.inputs.input_stuff_specs_factory import InputStuffSpecsFactoryfrom pipelex.core.pipes.pipe_output import PipeOutputfrom pipelex.core.pipes.variable_multiplicity import VariableMultiplicityfrom pipelex.core.stuffs.list_content import ListContent@@ -35,6 +33,7 @@ from pipelex.hub import (get_native_concept,get_required_concept,)+from pipelex.pipe_operators.llm.helpers import get_output_structure_promptfrom pipelex.pipe_operators.llm.llm_prompt_blueprint import LLMPromptBlueprintfrom pipelex.pipe_operators.llm.pipe_llm_blueprint import StructuringMethodfrom pipelex.pipe_operators.pipe_operator import PipeOperator@@ -46,7 +45,6 @@ from pipelex.pipe_run.pipe_run_params import ()from pipelex.pipeline.job_metadata import JobMetadatafrom pipelex.tools.typing.pydantic_utils import format_pydantic_validation_error-from pipelex.tools.typing.structure_printer import StructurePrinterfrom pipelex.types import Self@@ -112,20 +110,14 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):@overridedef needed_inputs(self, visited_pipes: set[str] | None = None) -> InputStuffSpecs:- """Needed inputs are the inputs needed to run the pipe, specified in the inputs attribute of the pipe"""- needed_inputs = InputStuffSpecsFactory.make_empty()-- for input_name, stuff_spec in self.inputs.items:- needed_inputs.add_stuff_spec(variable_name=input_name, concept=stuff_spec.concept, multiplicity=stuff_spec.multiplicity)- return needed_inputs+ return self.inputs@overridedef required_variables(self) -> set[str]:- """Required variables are the variables that are used in the current prompt template or system prompt"""return {variable_name for variable_name in self.llm_prompt_spec.required_variables() if not variable_name.startswith("_")}@override- async def _run_operator_pipe(+ async def _live_run_operator_pipe(self,job_metadata: JobMetadata,working_memory: WorkingMemory,@@ -133,7 +125,6 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):output_name: str | None = None,content_generator: ContentGeneratorProtocol | None = None,) -> PipeLLMOutput:- llm_config = get_config().cogt.llm_configcontent_generator = content_generator or get_content_generator()# interpret / unwrap the argumentsoutput_stuff_spec = self.output@@ -271,8 +262,8 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):llm_prompt_2_factory = Noneoutput_structure_prompt: str | None = None- if llm_config.is_structure_prompt_enabled:- output_structure_prompt = await PipeLLM.get_output_structure_prompt(+ if get_config().cogt.llm_config.is_structure_prompt_enabled:+ output_structure_prompt = await get_output_structure_prompt(concept_ref=pipe_run_params.dynamic_output_concept_code or output_stuff_spec.concept.concept_ref,is_with_preliminary_text=is_with_preliminary_text,)@@ -440,37 +431,22 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):pipe_run_params: PipeRunParams,output_name: str | None = None,) -> PipeLLMOutput:- content_generator_dry = ContentGeneratorDry()- return await self._run_operator_pipe(+ return await self._live_run_operator_pipe(job_metadata=job_metadata,working_memory=working_memory,pipe_run_params=pipe_run_params,output_name=output_name,- content_generator=content_generator_dry,+ content_generator=ContentGeneratorDry(),)- @staticmethod- async def get_output_structure_prompt(concept_ref: str, is_with_preliminary_text: bool) -> str | None:- concept = get_required_concept(concept_ref=concept_ref)- output_class = get_class_registry().get_class(concept.structure_class_name)- if not output_class:- return None-- class_structure = StructurePrinter().get_type_structure(tp=output_class, base_class=StuffContent)-- if not class_structure:- return None- class_structure_str = "\n".join(class_structure)- llm_config = get_config().cogt.llm_config- if is_with_preliminary_text:- template_source = llm_config.get_template(template_name="output_structure_prompt")- else:- template_source = llm_config.get_template(template_name="output_structure_prompt_no_preliminary_text")-- return await get_content_generator().make_templated_text(- context={- "class_structure_str": class_structure_str,- },- template=template_source,- template_category=TemplateCategory.LLM_PROMPT,- )+ @override+ async def _validate_before_run(+ self, job_metadata: JobMetadata, working_memory: WorkingMemory, pipe_run_params: PipeRunParams, output_name: str | None = None+ ):+ pass++ @override+ async def _validate_after_run(+ self, job_metadata: JobMetadata, working_memory: WorkingMemory, pipe_run_params: PipeRunParams, output_name: str | None = None+ ):+ passdiff --git a/pipelex/pipe_operators/pipe_operator.py b/pipelex/pipe_operators/pipe_operator.pyindex 8453349f9..c8b2a94f0 100644--- a/pipelex/pipe_operators/pipe_operator.py+++ b/pipelex/pipe_operators/pipe_operator.py@@ -8,7 +8,6 @@ from pipelex.core.memory.working_memory import WorkingMemoryfrom pipelex.core.pipes.pipe_abstract import PipeAbstractfrom pipelex.core.pipes.pipe_output import PipeOutputfrom pipelex.pipe_operators.exceptions import PipeOperatorModelAvailabilityError-from pipelex.pipe_run.pipe_run_mode import PipeRunModefrom pipelex.pipe_run.pipe_run_params import PipeRunParamsfrom pipelex.pipeline.job_metadata import JobMetadata@@ -28,7 +27,7 @@ class PipeOperator(PipeAbstract, Generic[PipeOperatorOutputType]):@final@override- async def _run_pipe(+ async def _live_run_pipe(self,job_metadata: JobMetadata,working_memory: WorkingMemory,@@ -36,33 +35,23 @@ class PipeOperator(PipeAbstract, Generic[PipeOperatorOutputType]):output_name: str | None = None,) -> PipeOutput:try:- match pipe_run_params.run_mode:- case PipeRunMode.LIVE:- pipe_output = await self._run_operator_pipe(- job_metadata=job_metadata,- working_memory=working_memory,- pipe_run_params=pipe_run_params,- output_name=output_name,- )- main_stuff = pipe_output.main_stuff- output_concept_code = self.output.concept.code- output_concept_with_multiplicity = f"[bold green]{output_concept_code}[/bold green]"- if main_stuff.is_list:- list_content: ListContent[StuffContent] = main_stuff.as_list_content() # pyright: ignore[reportUnknownVariableType, reportUnknownMemberType]- nb_items = len(list_content.items)- if nb_items == 1:- output_concept_with_multiplicity += " [1 item]"- else:- output_concept_with_multiplicity += f" [{nb_items} items]"- title = f"Output of pipe [red]{self.code}[/red] [yellow]→[/yellow] {output_concept_with_multiplicity}"- main_stuff.pretty_print_stuff(title=title)- case PipeRunMode.DRY:- pipe_output = await self._dry_run_operator_pipe(- job_metadata=job_metadata,- working_memory=working_memory,- pipe_run_params=pipe_run_params,- output_name=output_name,- )+ pipe_output = await self._live_run_operator_pipe(+ job_metadata=job_metadata, working_memory=working_memory, pipe_run_params=pipe_run_params, output_name=output_name+ )++ main_stuff = pipe_output.main_stuff+ output_concept_code = self.output.concept.code+ output_concept_with_multiplicity = f"[bold green]{output_concept_code}[/bold green]"+ if main_stuff.is_list:+ list_content: ListContent[StuffContent] = main_stuff.as_list_content() # pyright: ignore[reportUnknownVariableType, reportUnknownMemberType]+ nb_items = len(list_content.items)+ if nb_items == 1:+ output_concept_with_multiplicity += " [1 item]"+ else:+ output_concept_with_multiplicity += f" [{nb_items} items]"+ title = f"Output of pipe [red]{self.code}[/red] [yellow]→[/yellow] {output_concept_with_multiplicity}"+ main_stuff.pretty_print_stuff(title=title)+except ModelWaterfallError as model_waterfall_error:raise PipeOperatorModelAvailabilityError(message=model_waterfall_error.message,@@ -85,8 +74,17 @@ class PipeOperator(PipeAbstract, Generic[PipeOperatorOutputType]):) from model_not_found_errorreturn pipe_output+ @final+ @override+ async def _dry_run_pipe(+ self, job_metadata: JobMetadata, working_memory: WorkingMemory, pipe_run_params: PipeRunParams, output_name: str | None = None+ ) -> PipeOutput:+ return await self._dry_run_operator_pipe(+ job_metadata=job_metadata, working_memory=working_memory, pipe_run_params=pipe_run_params, output_name=output_name+ )+@abstractmethod- async def _run_operator_pipe(+ async def _live_run_operator_pipe(self,job_metadata: JobMetadata,working_memory: WorkingMemory,diff --git a/pipelex/pipe_run/pipe_run_params.py b/pipelex/pipe_run/pipe_run_params.pyindex 817db8690..1ae739dfc 100644--- a/pipelex/pipe_run/pipe_run_params.py+++ b/pipelex/pipe_run/pipe_run_params.py@@ -8,6 +8,7 @@ from pipelex import logfrom pipelex.core.memory.working_memory import BATCH_ITEM_STUFF_NAME, MAIN_STUFF_NAMEfrom pipelex.core.pipes.variable_multiplicity import VariableMultiplicity, VariableMultiplicityResolutionfrom pipelex.pipe_run.pipe_run_mode import PipeRunMode+from pipelex.pipeline.exceptions import PipeStackOverflowErrorfrom pipelex.types import Self, StrEnum@@ -189,10 +190,14 @@ class PipeRunParams(BaseModel):@propertydef is_multiple_output_required(self) -> bool:- return isinstance(self.output_multiplicity, int) and self.output_multiplicity > 1 # pyright: ignore[reportUnnecessaryIsInstance]+ return isinstance(self.output_multiplicity, int) and self.output_multiplicity > 1def push_pipe_to_stack(self, pipe_code: str) -> None:self.pipe_stack.append(pipe_code)+ limit = self.pipe_stack_limit+ if len(self.pipe_stack) > limit:+ msg = f"Exceeded pipe stack limit of {limit}. You can raise that limit in the config. Stack:\n{self.pipe_stack}"+ raise PipeStackOverflowError(message=msg, limit=limit, pipe_stack=self.pipe_stack)def pop_pipe_from_stack(self, pipe_code: str) -> None:popped_pipe_code = self.pipe_stack.pop()diff --git a/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.py b/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.pyindex 92da2f747..a62a56503 100644--- a/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.py+++ b/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.py@@ -56,8 +56,9 @@ class TestPipeConditionSimple:# Verify it failed for the right reasonerror = exc_info.valueassert error.pipe_code == "test_condition_fail"+ assert error.missing_inputs is not Noneassert "user_category" in error.missing_inputs- assert "Missing required inputs" in str(error)+ assert "missing required inputs" in str(error)async def test_direct_pipe_condition_should_succeed(self, load_test_library: Callable[[list[Path]], None]):"""Test a PipeCondition created directly in code that should SUCCEED dry run."""diff --git a/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_complex.py b/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_complex.pyindex 1c3f9299d..b719f6dac 100644--- a/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_complex.py+++ b/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_complex.py@@ -300,7 +300,7 @@ class TestPipeConditionComplex:error = exc_info.valueassert error.pipe_code == "complex_document_processor"- assert "Missing required inputs" in str(error)+ assert "missing required inputs" in str(error)# Check the underlying cause is PipeRunInputsError with missing_inputs detailscause = error.__cause__diff --git a/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_simple.py b/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_simple.pyindex a8c5bad0e..1740588c1 100644--- a/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_simple.py+++ b/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_simple.py@@ -242,7 +242,7 @@ class TestPipeConditionSimple:# Verify the error detailserror = exc_info.valueassert error.pipe_code == "basic_condition_by_category_2"- assert "Missing required inputs" in str(error)+ assert "missing required inputs" in str(error)# Check the underlying cause is PipeRunInputsError with missing_inputs detailscause = error.__cause__diff --git a/tests/unit/pipelex/core/pipes/test_stuff_spec.py b/tests/unit/pipelex/core/pipes/test_stuff_spec.pynew file mode 100644index 000000000..222ea900f--- /dev/null+++ b/tests/unit/pipelex/core/pipes/test_stuff_spec.py@@ -0,0 +1,85 @@+import pytest++from pipelex.core.concepts.concept_factory import ConceptFactory+from pipelex.core.concepts.native.concept_native import NativeConceptCode+from pipelex.core.pipes.stuff_spec.stuff_spec import StuffSpec+from pipelex.core.pipes.variable_multiplicity import VariableMultiplicity+++class TestStuffSpecToBundleRepresentation:+ """Test the StuffSpec.to_bundle_representation method."""++ @pytest.mark.parametrize(+ ("multiplicity", "expected_suffix"),+ [+ # No multiplicity - returns just concept_ref+ (None, ""),+ # Boolean multiplicity (variable-length list)+ (True, "[]"),+ # Integer multiplicity (fixed-length list)+ (1, "[1]"),+ (3, "[3]"),+ (10, "[10]"),+ (999, "[999]"),+ ],+ )+ def test_to_bundle_representation_with_native_text(+ self,+ multiplicity: VariableMultiplicity | None,+ expected_suffix: str,+ ):+ """Test to_bundle_representation with native.Text concept and various multiplicities."""+ concept = ConceptFactory.make_native_concept(native_concept_code=NativeConceptCode.TEXT)+ stuff_spec = StuffSpec(concept=concept, multiplicity=multiplicity)++ result = stuff_spec.to_bundle_representation()++ expected = f"{concept.concept_ref}{expected_suffix}"+ assert result == expected+ assert result == f"native.Text{expected_suffix}"++ @pytest.mark.parametrize(+ ("native_concept_code", "multiplicity", "expected"),+ [+ # Various native concepts without multiplicity+ (NativeConceptCode.TEXT, None, "native.Text"),+ (NativeConceptCode.IMAGE, None, "native.Image"),+ (NativeConceptCode.PDF, None, "native.PDF"),+ (NativeConceptCode.NUMBER, None, "native.Number"),+ # Various native concepts with variable-length list+ (NativeConceptCode.TEXT, True, "native.Text[]"),+ (NativeConceptCode.IMAGE, True, "native.Image[]"),+ # Various native concepts with fixed-length list+ (NativeConceptCode.TEXT, 5, "native.Text[5]"),+ (NativeConceptCode.IMAGE, 3, "native.Image[3]"),+ ],+ )+ def test_to_bundle_representation_with_various_native_concepts(+ self,+ native_concept_code: NativeConceptCode,+ multiplicity: VariableMultiplicity | None,+ expected: str,+ ):+ """Test to_bundle_representation with different native concept types."""+ concept = ConceptFactory.make_native_concept(native_concept_code=native_concept_code)+ stuff_spec = StuffSpec(concept=concept, multiplicity=multiplicity)++ result = stuff_spec.to_bundle_representation()++ assert result == expected++ def test_to_bundle_representation_with_custom_domain_concept(self):+ """Test to_bundle_representation with a custom domain concept."""+ concept = ConceptFactory.make(+ domain_code="my_domain",+ concept_code="MyCustomConcept",+ description="A custom concept for testing",+ structure_class_name="TextContent",+ )+ stuff_spec_single = StuffSpec(concept=concept, multiplicity=None)+ stuff_spec_list = StuffSpec(concept=concept, multiplicity=True)+ stuff_spec_fixed = StuffSpec(concept=concept, multiplicity=7)++ assert stuff_spec_single.to_bundle_representation() == "my_domain.MyCustomConcept"+ assert stuff_spec_list.to_bundle_representation() == "my_domain.MyCustomConcept[]"+ assert stuff_spec_fixed.to_bundle_representation() == "my_domain.MyCustomConcept[7]"diff --git a/tests/unit/pipelex/tools/test.plx b/tests/unit/pipelex/tools/test.plxindex d57a23de1..eab2abf75 100644--- a/tests/unit/pipelex/tools/test.plx+++ b/tests/unit/pipelex/tools/test.plx@@ -9,6 +9,6 @@ FilePath = "The path of a file"type = "PipeFunc"description = "Read the content of related codebase files"inputs = { related_file_paths = "FilePath" }-output = "CodebaseFileContent"+output = "CodebaseFileContent[]"function_name = "read_file_content"