Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

refactor run pipe (#534)

Thomas Hebrard826a8d72ec21
diff --git a/CHANGELOG.md b/CHANGELOG.md
index e0661cd4b..c86861598 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -40,6 +40,7 @@
- Validation for `PipeCompose` output concepts: must be strictly compatible with the Text concept
- Validation for Pipelex Bundle concept keys: cannot create a native concept
- Validation for `PipeSequence`: The multiplicity of the output of the sequence must be the same as the multiplicity of the output of the last step.
+- Validation for `PipeFunc`: The multiplicity of the output of the pipe must be the same as the multiplicity of the output of the function: If the multiplicity is true, the return type of the function must be a subclass of `ListContent`. If the multiplicity is false, the return type of the function must not be a subclass of `ListContent`.
### Changed
@@ -84,6 +85,7 @@
- `ConceptFactory.make_from_blueprint` method now correctly handles native concepts
- The output (and inputs) of a pipe is now a `StuffSpec` object instead, that holds the concept and the multiplicity.
- Renamed `domain` into `domain_code` when relevant.
+- Refactored the dry run methods of the `PipeAbstract` class.
## [v0.17.3] - 2025-12-01
diff --git a/pipelex/core/pipes/inputs/exceptions.py b/pipelex/core/pipes/inputs/exceptions.py
index 8934702be..5bb87b8db 100644
--- a/pipelex/core/pipes/inputs/exceptions.py
+++ b/pipelex/core/pipes/inputs/exceptions.py
@@ -35,7 +35,7 @@ class PipeRunInputsError(PipeRunError):
message: str,
run_mode: PipeRunMode,
pipe_code: str,
- missing_inputs: dict[str, str],
+ missing_inputs: list[str] | None = None,
variable_name: str | None = None,
concept_code: str | None = None,
):
diff --git a/pipelex/core/pipes/pipe_abstract.py b/pipelex/core/pipes/pipe_abstract.py
index 4027a9073..2f6d57dfa 100644
--- a/pipelex/core/pipes/pipe_abstract.py
+++ b/pipelex/core/pipes/pipe_abstract.py
@@ -17,7 +17,6 @@ from pipelex.core.pipes.pipe_output import PipeOutput
from pipelex.core.pipes.stuff_spec.stuff_spec import StuffSpec
from pipelex.pipe_run.pipe_run_mode import PipeRunMode
from pipelex.pipe_run.pipe_run_params import PipeRunParams
-from pipelex.pipeline.exceptions import PipeStackOverflowError
from pipelex.pipeline.job_metadata import JobMetadata, OtelContext
from pipelex.pipeline.pipeline_factory import PipelineFactory
from pipelex.system.telemetry.otel_constants import (
@@ -33,6 +32,8 @@ from pipelex.tools.misc.package_utils import get_package_version
from pipelex.tools.misc.string_utils import is_snake_case
from pipelex.types import Self
+PipeAbstractType = type["PipeAbstract"]
+
class PipeAbstract(ABC, BaseModel):
model_config = ConfigDict(strict=True, extra="forbid")
@@ -229,21 +230,66 @@ class PipeAbstract(ABC, BaseModel):
def validate_output_static(self):
pass
+ @final
async def validate_before_run(
self,
job_metadata: JobMetadata,
working_memory: WorkingMemory,
pipe_run_params: PipeRunParams,
output_name: str | None = None,
- ): ...
+ ):
+ # Check that all the needed inputs are actually in the working memory
+ missing_input_names: list[str] = []
+
+ for named_stuff_spec in self.needed_inputs().named_stuff_specs:
+ if not working_memory.get_optional_stuff(named_stuff_spec.variable_name):
+ missing_input_names.append(named_stuff_spec.variable_name)
+
+ if missing_input_names:
+ msg = f"Dry run of {self.type} '{self.code}': missing required inputs: {', '.join(missing_input_names)}"
+ raise PipeRunInputsError(
+ message=msg,
+ run_mode=pipe_run_params.run_mode,
+ pipe_code=self.code,
+ missing_inputs=missing_input_names,
+ )
+
+ # Specific pipe validation function
+ await self._validate_before_run(
+ job_metadata=job_metadata, working_memory=working_memory, pipe_run_params=pipe_run_params, output_name=output_name
+ )
+ @abstractmethod
+ async def _validate_before_run(
+ self,
+ job_metadata: JobMetadata,
+ working_memory: WorkingMemory,
+ pipe_run_params: PipeRunParams,
+ output_name: str | None = None,
+ ):
+ pass
+
+ @final
async def validate_after_run(
self,
job_metadata: JobMetadata,
working_memory: WorkingMemory,
pipe_run_params: PipeRunParams,
output_name: str | None = None,
- ): ...
+ ):
+ await self._validate_after_run(
+ job_metadata=job_metadata, working_memory=working_memory, pipe_run_params=pipe_run_params, output_name=output_name
+ )
+
+ @abstractmethod
+ async def _validate_after_run(
+ self,
+ job_metadata: JobMetadata,
+ working_memory: WorkingMemory,
+ pipe_run_params: PipeRunParams,
+ output_name: str | None = None,
+ ):
+ pass
@abstractmethod
def required_variables(self) -> set[str]:
@@ -269,13 +315,6 @@ class PipeAbstract(ABC, BaseModel):
"""
- def monitor_pipe_stack(self, pipe_run_params: PipeRunParams):
- pipe_stack = pipe_run_params.pipe_stack
- limit = pipe_run_params.pipe_stack_limit
- if len(pipe_stack) > limit:
- msg = f"Exceeded pipe stack limit of {limit}. You can raise that limit in the config. Stack:\n{pipe_stack}"
- raise PipeStackOverflowError(message=msg, limit=limit, pipe_stack=pipe_stack)
-
def _format_pipe_run_info(self, pipe_run_params: PipeRunParams) -> str:
indent_level = len(pipe_run_params.pipe_stack) - 1
indent = " " * indent_level
@@ -292,8 +331,135 @@ class PipeAbstract(ABC, BaseModel):
arrow = "[yellow]→[/yellow]"
return f"{indent}{pipe_type_label} {pipe_code_label} {arrow} {concept_code_label}"
+ @final
+ async def run_pipe(
+ self,
+ job_metadata: JobMetadata,
+ working_memory: WorkingMemory,
+ pipe_run_params: PipeRunParams,
+ output_name: str | None = None,
+ ) -> PipeOutput:
+ pipe_run_params.push_pipe_to_stack(pipe_code=self.code)
+
+ match pipe_run_params.run_mode:
+ case PipeRunMode.LIVE:
+ pipe_output = await self.live_run_pipe(
+ job_metadata=job_metadata,
+ working_memory=working_memory,
+ pipe_run_params=pipe_run_params,
+ output_name=output_name,
+ )
+ case PipeRunMode.DRY:
+ pipe_output = await self.dry_run_pipe(
+ job_metadata=job_metadata,
+ working_memory=working_memory,
+ pipe_run_params=pipe_run_params,
+ output_name=output_name,
+ )
+
+ pipe_run_params.pop_pipe_from_stack(pipe_code=self.code)
+ return pipe_output
+
+ @final
+ async def live_run_pipe(
+ self,
+ job_metadata: JobMetadata,
+ working_memory: WorkingMemory,
+ pipe_run_params: PipeRunParams,
+ output_name: str | None = None,
+ ) -> PipeOutput:
+ log.info(self._format_pipe_run_info(pipe_run_params=pipe_run_params))
+
+ # Handle telemetry ------------------------------------------------------------
+
+ # Generate pipe_run_id (business ID, always set)
+ this_pipe_run_id = PipelineFactory.make_pipe_run_id()
+
+ # Derive OtelContext if telemetry is enabled (not dry mode and tracer available)
+ # The trace_id comes from parent's otel_context (already computed at pipeline start)
+ this_otel_context: OtelContext | None = None
+ span: Span | None = None
+ is_root_span: bool = False
+
+ parent_otel_context = job_metadata.otel_context
+ if not pipe_run_params.run_mode.is_dry and parent_otel_context is not None:
+ # Start OTel span first
+ span, is_root_span = self._start_pipe_span(
+ parent_otel_context=parent_otel_context,
+ pipeline_run_id=job_metadata.pipeline_run_id,
+ working_memory=working_memory,
+ )
+ # Get the actual span_id from OTel (OTel generates its own span_id)
+ if span:
+ span_context = span.get_span_context()
+ this_otel_context = OtelContext(
+ trace_id=parent_otel_context.trace_id,
+ trace_name=parent_otel_context.trace_name,
+ trace_name_redacted=parent_otel_context.trace_name_redacted,
+ span_id=span_context.span_id,
+ )
+
+ # Create child metadata with updated pipe_code and pipe_run_id
+ # This passes down a modified copy rather than mutating the original
+ # otel_context is passed separately because it must always be set explicitly
+ # (even when None in dry mode) to avoid inheriting stale parent context
+ child_metadata = job_metadata.copy_with_update(
+ updated_metadata=JobMetadata(
+ pipe_code=self.code,
+ pipe_run_id=this_pipe_run_id,
+ ),
+ otel_context=this_otel_context,
+ )
+
+ # Run pipe ------------------------------------------------------------
+
+ try:
+ pipe_output = await self._live_run_pipe(
+ job_metadata=child_metadata, working_memory=working_memory, pipe_run_params=pipe_run_params, output_name=output_name
+ )
+ except Exception as exc:
+ self._end_pipe_span_error(span, error=exc, is_root_span=is_root_span)
+ raise
+
+ # Handle telemetry ------------------------------------------------------------
+
+ self._end_pipe_span_success(span=span, pipe_output=pipe_output, is_root_span=is_root_span)
+
+ return pipe_output
+
+ @final
+ async def dry_run_pipe(
+ self,
+ job_metadata: JobMetadata,
+ working_memory: WorkingMemory,
+ pipe_run_params: PipeRunParams,
+ output_name: str | None = None,
+ ) -> PipeOutput:
+ log.verbose(f"Dry run of {self.type}: '{self.code}'")
+ assert pipe_run_params.run_mode.is_dry, f"Dry run of {self.type} '{self.code}' called with run_mode = {pipe_run_params.run_mode}"
+ await self.validate_before_run(
+ job_metadata=job_metadata, working_memory=working_memory, pipe_run_params=pipe_run_params, output_name=output_name
+ )
+ pipe_output = await self._dry_run_pipe(
+ job_metadata=job_metadata, working_memory=working_memory, pipe_run_params=pipe_run_params, output_name=output_name
+ )
+ await self.validate_after_run(
+ job_metadata=job_metadata, working_memory=working_memory, pipe_run_params=pipe_run_params, output_name=output_name
+ )
+ return pipe_output
+
@abstractmethod
- async def _run_pipe(
+ async def _live_run_pipe(
+ self,
+ job_metadata: JobMetadata,
+ working_memory: WorkingMemory,
+ pipe_run_params: PipeRunParams,
+ output_name: str | None = None,
+ ) -> PipeOutput:
+ pass
+
+ @abstractmethod
+ async def _dry_run_pipe(
self,
job_metadata: JobMetadata,
working_memory: WorkingMemory,
@@ -475,104 +641,3 @@ class PipeAbstract(ABC, BaseModel):
if is_root_span:
span.set_attribute(LangfuseSpanAttr.TRACE_OUTCOME, SpanOutcome.FAILURE)
span.end()
-
- @final
- async def run_pipe(
- self,
- job_metadata: JobMetadata,
- working_memory: WorkingMemory,
- pipe_run_params: PipeRunParams,
- output_name: str | None = None,
- ) -> PipeOutput:
- pipe_run_params.push_pipe_to_stack(pipe_code=self.code)
- self.monitor_pipe_stack(pipe_run_params=pipe_run_params)
-
- # Check inputs ------------------------------------------------------------
-
- # check we have the required inputs in the working memory
- missing_inputs: dict[str, str] = {}
- for required_stuff_name, stuff_spec in self.needed_inputs().items:
- if not working_memory.is_stuff_exists(name=required_stuff_name):
- missing_inputs[required_stuff_name] = stuff_spec.concept.code
- if missing_inputs:
- error = PipeRunInputsError(
- message=f"Missing required inputs for pipe '{self.code}': {missing_inputs}",
- run_mode=pipe_run_params.run_mode,
- pipe_code=self.code,
- missing_inputs=missing_inputs,
- )
- raise error
-
- pipe_run_info = self._format_pipe_run_info(pipe_run_params=pipe_run_params)
- if pipe_run_params.run_mode == PipeRunMode.LIVE:
- log.info(pipe_run_info)
-
- # Handle telemetry ------------------------------------------------------------
-
- # Generate pipe_run_id (business ID, always set)
- this_pipe_run_id = PipelineFactory.make_pipe_run_id()
-
- # Derive OtelContext if telemetry is enabled (not dry mode and tracer available)
- # The trace_id comes from parent's otel_context (already computed at pipeline start)
- this_otel_context: OtelContext | None = None
- span: Span | None = None
- is_root_span: bool = False
-
- parent_otel_context = job_metadata.otel_context
- if not pipe_run_params.run_mode.is_dry and parent_otel_context is not None:
- # Start OTel span first
- span, is_root_span = self._start_pipe_span(
- parent_otel_context=parent_otel_context,
- pipeline_run_id=job_metadata.pipeline_run_id,
- working_memory=working_memory,
- )
- # Get the actual span_id from OTel (OTel generates its own span_id)
- if span:
- span_context = span.get_span_context()
- this_otel_context = OtelContext(
- trace_id=parent_otel_context.trace_id,
- trace_name=parent_otel_context.trace_name,
- trace_name_redacted=parent_otel_context.trace_name_redacted,
- span_id=span_context.span_id,
- )
-
- # Create child metadata with updated pipe_code and pipe_run_id
- # This passes down a modified copy rather than mutating the original
- # otel_context is passed separately because it must always be set explicitly
- # (even when None in dry mode) to avoid inheriting stale parent context
- child_metadata = job_metadata.copy_with_update(
- updated_metadata=JobMetadata(
- pipe_code=self.code,
- pipe_run_id=this_pipe_run_id,
- ),
- otel_context=this_otel_context,
- )
-
- # Run pipe ------------------------------------------------------------
-
- try:
- await self.validate_before_run(
- job_metadata=child_metadata, working_memory=working_memory, pipe_run_params=pipe_run_params, output_name=output_name
- )
- pipe_output = await self._run_pipe(
- job_metadata=child_metadata, working_memory=working_memory, pipe_run_params=pipe_run_params, output_name=output_name
- )
- await self.validate_after_run(
- job_metadata=child_metadata, working_memory=working_memory, pipe_run_params=pipe_run_params, output_name=output_name
- )
- except Exception as exc:
- self._end_pipe_span_error(span, error=exc, is_root_span=is_root_span)
- raise
-
- # Handle telemetry ------------------------------------------------------------
-
- self._end_pipe_span_success(span=span, pipe_output=pipe_output, is_root_span=is_root_span)
-
- # Cleanup ------------------------------------------------------------
-
- pipe_run_params.pop_pipe_from_stack(pipe_code=self.code)
-
- return pipe_output
-
-
-PipeAbstractType = type[PipeAbstract]
diff --git a/pipelex/core/pipes/stuff_spec/stuff_spec.py b/pipelex/core/pipes/stuff_spec/stuff_spec.py
index 657004f60..f7711af5c 100644
--- a/pipelex/core/pipes/stuff_spec/stuff_spec.py
+++ b/pipelex/core/pipes/stuff_spec/stuff_spec.py
@@ -14,3 +14,10 @@ class StuffSpec(BaseModel):
if isinstance(self.multiplicity, bool):
return self.multiplicity
return self.multiplicity > 1
+
+ def to_bundle_representation(self) -> str:
+ if self.multiplicity is None:
+ return self.concept.concept_ref
+ if isinstance(self.multiplicity, bool):
+ return f"{self.concept.concept_ref}[]"
+ return f"{self.concept.concept_ref}[{self.multiplicity}]"
diff --git a/pipelex/core/pipes/variable_multiplicity.py b/pipelex/core/pipes/variable_multiplicity.py
index 57b0989ad..c8e85882a 100644
--- a/pipelex/core/pipes/variable_multiplicity.py
+++ b/pipelex/core/pipes/variable_multiplicity.py
@@ -58,7 +58,7 @@ class MultiplicityParseResult:
self.multiplicity: int | bool | None = multiplicity
-def parse_concept_with_multiplicity(concept_spec: str) -> MultiplicityParseResult:
+def parse_concept_with_multiplicity(concept_ref: str) -> MultiplicityParseResult:
"""Parse a concept specification string to extract concept and multiplicity.
Supported formats:
@@ -70,7 +70,7 @@ def parse_concept_with_multiplicity(concept_spec: str) -> MultiplicityParseResul
- "domain.ConceptName[5]" -> (domain.ConceptName, 5)
Args:
- concept_spec: Concept specification string with optional multiplicity brackets
+ concept_ref: Concept specification string with optional multiplicity brackets
Returns:
MultiplicityParseResult with concept (without brackets) and multiplicity value
@@ -82,11 +82,11 @@ def parse_concept_with_multiplicity(concept_spec: str) -> MultiplicityParseResul
# Use strict pattern to validate identifier syntax
# Concept must start with letter/underscore, optional domain prefix, optional brackets
pattern = r"^([a-zA-Z_][a-zA-Z0-9_]*(?:\.[a-zA-Z_][a-zA-Z0-9_]*)?)(?:\[(\d*)\])?$"
- match = re.match(pattern, concept_spec)
+ match = re.match(pattern, concept_ref)
if not match:
msg = (
- f"Invalid concept specification syntax: '{concept_spec}'. "
+ f"Invalid concept specification syntax: '{concept_ref}'. "
f"Expected format: 'ConceptName', 'ConceptName[]', 'ConceptName[N]', "
f"'domain.ConceptName', 'domain.ConceptName[]', or 'domain.ConceptName[N]' "
f"where concept and domain names must start with a letter or underscore."
@@ -107,7 +107,7 @@ def parse_concept_with_multiplicity(concept_spec: str) -> MultiplicityParseResul
# Number in brackets [N] - fixed count
multiplicity = int(bracket_content)
if multiplicity <= 0:
- msg = f"Invalid multiplicity value in '{concept_spec}': multiplicity must be at least 1. A pipe must produce at least one output."
+ msg = f"Invalid multiplicity value in '{concept_ref}': multiplicity must be at least 1. A pipe must produce at least one output."
raise PipeVariableMultiplicityError(msg)
return MultiplicityParseResult(concept=concept, multiplicity=multiplicity)
diff --git a/pipelex/core/stuffs/stuff_factory.py b/pipelex/core/stuffs/stuff_factory.py
index aedd360e9..b96096cbb 100644
--- a/pipelex/core/stuffs/stuff_factory.py
+++ b/pipelex/core/stuffs/stuff_factory.py
@@ -35,6 +35,10 @@ class StuffBlueprint(BaseModel):
class StuffFactory:
+ @classmethod
+ def make_stuff_code(cls) -> str:
+ return shortuuid.uuid()[:5]
+
@classmethod
def make_stuff_name(cls, concept: Concept) -> str:
return Stuff.make_stuff_name(concept=concept)
@@ -65,13 +69,17 @@ class StuffFactory:
name: str | None = None,
code: str | None = None,
) -> Stuff:
+ if not code:
+ code = cls.make_stuff_code()
+
if not name:
name = cls.make_stuff_name(concept=concept)
+
return Stuff(
concept=concept,
content=content,
stuff_name=name,
- stuff_code=code or shortuuid.uuid()[:5],
+ stuff_code=code,
)
@classmethod
diff --git a/pipelex/pipe_controllers/batch/pipe_batch.py b/pipelex/pipe_controllers/batch/pipe_batch.py
index 4a3cbd0fb..0dd28f444 100644
--- a/pipelex/pipe_controllers/batch/pipe_batch.py
+++ b/pipelex/pipe_controllers/batch/pipe_batch.py
@@ -1,7 +1,6 @@
import asyncio
from typing import TYPE_CHECKING, Any, Literal, cast
-import shortuuid
from typing_extensions import override
from pipelex.config import get_config
@@ -72,7 +71,7 @@ class PipeBatch(PipeController):
pass
@override
- async def validate_before_run(
+ async def _validate_before_run(
self, job_metadata: JobMetadata, working_memory: WorkingMemory, pipe_run_params: PipeRunParams, output_name: str | None = None
) -> None:
batch_params = pipe_run_params.batch_params or self.batch_params or BatchParams.make_default()
@@ -93,20 +92,19 @@ class PipeBatch(PipeController):
)
raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode, pipe_code=self.code)
- async def _run_batch_pipe(
+ @override
+ async def _live_run_controller_pipe(
self,
job_metadata: JobMetadata,
working_memory: WorkingMemory,
pipe_run_params: PipeRunParams,
output_name: str | None = None,
) -> PipeOutput:
- """Common logic for running or dry-running a pipe in batch mode."""
batch_params = pipe_run_params.batch_params or self.batch_params or BatchParams.make_default()
input_item_stuff_name = batch_params.input_item_stuff_name
input_list_stuff_name = batch_params.input_list_stuff_name
if pipe_run_params.final_stuff_code:
- method_name = "dry_run_pipe" if pipe_run_params.run_mode == PipeRunMode.DRY else "_run_controller_pipe"
pipe_run_params.final_stuff_code = None
pipe_run_params.push_pipe_layer(pipe_code=self.branch_pipe_code)
@@ -117,7 +115,7 @@ class PipeBatch(PipeController):
# TODO: Make commented code work when inputing images named "a.b.c"
sub_pipe = get_required_pipe(pipe_code=self.branch_pipe_code)
nb_history_items_limit = get_config().pipelex.tracker_config.applied_nb_items_limit
- batch_output_stuff_code = shortuuid.uuid()
+ batch_output_stuff_code = StuffFactory.make_stuff_code()
tasks: list[Coroutine[Any, Any, PipeOutput]] = []
item_stuffs: list[Stuff] = []
required_stuff_lists: list[list[Stuff]] = []
@@ -176,21 +174,19 @@ class PipeBatch(PipeController):
pipe_outputs = await asyncio.gather(*tasks)
output_items: list[StuffContent] = []
+
output_stuffs: list[Stuff] = []
- output_stuff_code = shortuuid.uuid()[:5]
+
for pipe_output in pipe_outputs:
branch_output_stuff = pipe_output.main_stuff
- output_stuffs.append(branch_output_stuff)
output_items.append(branch_output_stuff.content)
list_content: ListContent[StuffContent] = ListContent(items=output_items)
output_stuff = StuffFactory.make_stuff(
- code=output_stuff_code,
concept=self.output.concept,
content=list_content,
name=output_name,
)
-
method_name = "dry_run_pipe" if pipe_run_params.run_mode == PipeRunMode.DRY else "run_pipe"
for branch_index, (
required_stuff_list,
@@ -235,14 +231,14 @@ class PipeBatch(PipeController):
)
@override
- async def _run_controller_pipe(
+ async def _dry_run_controller_pipe(
self,
job_metadata: JobMetadata,
working_memory: WorkingMemory,
pipe_run_params: PipeRunParams,
output_name: str | None = None,
) -> PipeOutput:
- return await self._run_batch_pipe(
+ return await self._live_run_controller_pipe(
job_metadata=job_metadata,
working_memory=working_memory,
pipe_run_params=pipe_run_params,
@@ -250,16 +246,7 @@ class PipeBatch(PipeController):
)
@override
- async def _dry_run_controller_pipe(
- self,
- job_metadata: JobMetadata,
- working_memory: WorkingMemory,
- pipe_run_params: PipeRunParams,
- output_name: str | None = None,
- ) -> PipeOutput:
- return await self._run_batch_pipe(
- job_metadata=job_metadata,
- working_memory=working_memory,
- pipe_run_params=pipe_run_params,
- output_name=output_name,
- )
+ async def _validate_after_run(
+ self, job_metadata: JobMetadata, working_memory: WorkingMemory, pipe_run_params: PipeRunParams, output_name: str | None = None
+ ):
+ pass
diff --git a/pipelex/pipe_controllers/condition/pipe_condition.py b/pipelex/pipe_controllers/condition/pipe_condition.py
index 018d47598..cf7c737a3 100644
--- a/pipelex/pipe_controllers/condition/pipe_condition.py
+++ b/pipelex/pipe_controllers/condition/pipe_condition.py
@@ -7,8 +7,7 @@ from pipelex import log
from pipelex.cogt.templating.template_category import TemplateCategory
from pipelex.core.concepts.concept_factory import ConceptFactory
from pipelex.core.concepts.native.concept_native import NativeConceptCode
-from pipelex.core.memory.exceptions import WorkingMemoryStuffNotFoundError
-from pipelex.core.memory.working_memory import WorkingMemory
+from pipelex.core.memory.working_memory import WorkingMemory, WorkingMemoryStuffNotFoundError
from pipelex.core.pipes.exceptions import PipeValidationError, PipeValidationErrorType
from pipelex.core.pipes.inputs.input_stuff_specs import InputStuffSpecs
from pipelex.core.pipes.inputs.input_stuff_specs_factory import InputStuffSpecsFactory
@@ -84,7 +83,7 @@ class PipeCondition(PipeController):
needed_inputs = InputStuffSpecsFactory.make_empty()
- # 1. Add the variables from the expression/expression_template
+ # Add the variables from the expression/expression_template
required_variables = detect_jinja2_required_variables(
template_category=TemplateCategory.EXPRESSION,
template_source=self.expression,
@@ -101,7 +100,7 @@ class PipeCondition(PipeController):
),
)
- # 2. Add the inputs needed by all possible target pipes
+ # Add the inputs needed by all possible target pipes
for pipe_code in self.mapped_pipe_codes:
pipe = get_required_pipe(pipe_code=pipe_code)
# Use the centralized recursion detection
@@ -150,7 +149,7 @@ class PipeCondition(PipeController):
# TODO: Restore this validation. The problem lies with needed_inputs that construct Anything concepts.
# @override
- # async def validate_before_run(
+ # async def _validate_before_run(
# self,
# job_metadata: JobMetadata,
# working_memory: WorkingMemory,
@@ -197,20 +196,14 @@ class PipeCondition(PipeController):
return evaluated_expression
@override
- async def _run_controller_pipe(
+ async def _live_run_controller_pipe(
self,
job_metadata: JobMetadata,
working_memory: WorkingMemory,
pipe_run_params: PipeRunParams,
output_name: str | None = None,
) -> PipeOutput:
- log.verbose(f"{self.class_name} generating a '{self.output.concept.code}'")
-
- # TODO: restore pipe_layer feature
- # pipe_run_params.push_pipe_code(pipe_code=pipe_code)
-
evaluated_expression = await self._evaluate_expression(working_memory=working_memory)
-
# Select the outcome based on the evaluated expression
outcome = self.outcome_map.get(evaluated_expression, self.default_outcome)
@@ -225,7 +218,6 @@ class PipeCondition(PipeController):
chosen_pipe = get_required_pipe(pipe_code=outcome)
- # Create condition details for tracking
condition_details = self._make_pipe_condition_details(
evaluated_expression=self.expression,
chosen_pipe_code=chosen_pipe.code,
@@ -243,7 +235,6 @@ class PipeCondition(PipeController):
msg = f"Some required stuff(s) not found: {error_details}"
raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode, pipe_code=self.code) from exc
- # Track condition steps
for required_stuff in required_stuffs:
get_pipeline_tracker().add_condition_step(
from_stuff=required_stuff,
@@ -253,19 +244,15 @@ class PipeCondition(PipeController):
comment="PipeCondition required for condition",
)
- # Execute the chosen pipe
- log.verbose(f"Chosen pipe: {chosen_pipe.code}")
pipe_output = await get_pipe_router().run(
pipe_job=PipeJobFactory.make_pipe_job(
- pipe=chosen_pipe,
+ pipe=get_required_pipe(pipe_code=outcome),
job_metadata=job_metadata,
working_memory=working_memory,
pipe_run_params=pipe_run_params,
output_name=output_name,
),
)
-
- # Track choice step
get_pipeline_tracker().add_choice_step(
from_condition=condition_details,
to_stuff=pipe_output.main_stuff,
@@ -282,27 +269,7 @@ class PipeCondition(PipeController):
pipe_run_params: PipeRunParams,
output_name: str | None = None,
) -> PipeOutput:
- """Dry run implementation for PipeCondition.
- Validates that all required inputs are present, expression is valid, and target pipes exist.
- """
- log.verbose(f"PipeCondition: dry run controller pipe: {self.code}")
- # 1. Validate that all required inputs are present in the working memory
- needed_inputs = self.needed_inputs()
- missing_input_names: list[str] = []
-
- for named_stuff_spec in needed_inputs.named_stuff_specs:
- if not working_memory.get_optional_stuff(named_stuff_spec.variable_name):
- missing_input_names.append(named_stuff_spec.variable_name)
-
- if missing_input_names:
- log.error(f"Dry run failed: missing required inputs: {missing_input_names}")
- raise PipeRunError(
- message=f"Dry run failed for pipe '{self.code}' (PipeCondition): missing required inputs: {', '.join(missing_input_names)}",
- run_mode=pipe_run_params.run_mode,
- pipe_code=self.code,
- )
-
- # 2. Validate that the expression template is valid
+ # Validate that the expression template is valid
try:
required_variables = detect_jinja2_required_variables(
template_category=TemplateCategory.EXPRESSION,
@@ -317,7 +284,7 @@ class PipeCondition(PipeController):
)
raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode, pipe_code=self.code) from exc
- # 3. Validate that all values in the outcomes map (appart from special outcomes) do exist as pipe codes
+ # Validate that all values in the outcomes map (appart from special outcomes) do exist as pipe codes
all_pipe_codes = set(self.outcome_map.values())
if self.default_outcome:
all_pipe_codes.add(self.default_outcome)
@@ -341,3 +308,15 @@ class PipeCondition(PipeController):
pipe_run_params=pipe_run_params,
)
return PipeOutput(working_memory=working_memory)
+
+ @override
+ async def _validate_before_run(
+ self, job_metadata: JobMetadata, working_memory: WorkingMemory, pipe_run_params: PipeRunParams, output_name: str | None = None
+ ):
+ pass
+
+ @override
+ async def _validate_after_run(
+ self, job_metadata: JobMetadata, working_memory: WorkingMemory, pipe_run_params: PipeRunParams, output_name: str | None = None
+ ):
+ pass
diff --git a/pipelex/pipe_controllers/parallel/pipe_parallel.py b/pipelex/pipe_controllers/parallel/pipe_parallel.py
index 27d7cd45c..dfe6409aa 100644
--- a/pipelex/pipe_controllers/parallel/pipe_parallel.py
+++ b/pipelex/pipe_controllers/parallel/pipe_parallel.py
@@ -18,7 +18,6 @@ from pipelex.libraries.pipe.exceptions import PipeNotFoundError
from pipelex.pipe_controllers.pipe_controller import PipeController
from pipelex.pipe_controllers.sub_pipe import SubPipe
from pipelex.pipe_run.exceptions import PipeRunError
-from pipelex.pipe_run.pipe_run_mode import PipeRunMode
from pipelex.pipe_run.pipe_run_params import PipeRunParams
from pipelex.pipeline.job_metadata import JobMetadata
from pipelex.types import Self
@@ -128,7 +127,7 @@ class PipeParallel(PipeController):
return {sub_pipe.pipe_code for sub_pipe in self.parallel_sub_pipes}
@override
- async def _run_controller_pipe(
+ async def _live_run_controller_pipe(
self,
job_metadata: JobMetadata,
working_memory: WorkingMemory,
@@ -189,6 +188,7 @@ class PipeParallel(PipeController):
stuff=combined_output_stuff,
name=output_name,
)
+
for stuff in output_stuffs.values():
get_pipeline_tracker().add_aggregate_step(
from_stuff=stuff,
@@ -196,6 +196,7 @@ class PipeParallel(PipeController):
pipe_layer=pipe_run_params.pipe_layers,
comment="PipeParallel on output_stuffs",
)
+
return PipeOutput(
working_memory=working_memory,
pipeline_run_id=job_metadata.pipeline_run_id,
@@ -209,26 +210,7 @@ class PipeParallel(PipeController):
pipe_run_params: PipeRunParams,
output_name: str | None = None,
) -> PipeOutput:
- """Dry run implementation for PipeParallel.
- Validates that all required inputs are present and that all parallel sub-pipes can be dry run.
- """
- log.verbose(f"PipeParallel: dry run controller pipe: {self.code}")
- if pipe_run_params.run_mode != PipeRunMode.DRY:
- msg = f"PipeSequence._dry_run_controller_pipe() called with run_mode = {pipe_run_params.run_mode} in pipe {self.code}"
- raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode, pipe_code=self.code)
-
- # 1. Validate that all required inputs are present in the working memory
- needed_inputs = self.needed_inputs()
- missing_input_names: list[str] = []
- for named_stuff_spec in needed_inputs.named_stuff_specs:
- if not working_memory.get_optional_stuff(named_stuff_spec.variable_name):
- missing_input_names.append(named_stuff_spec.variable_name)
-
- if missing_input_names:
- msg = f"Dry run failed for pipe '{self.code}' (PipeParallel): missing required inputs: {missing_input_names}"
- raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode, pipe_code=self.code)
-
- # 2. Validate that all sub-pipes exist
+ # 1. Validate that all sub-pipes exist
for sub_pipe in self.parallel_sub_pipes:
try:
get_required_pipe(pipe_code=sub_pipe.pipe_code)
@@ -236,7 +218,7 @@ class PipeParallel(PipeController):
msg = f"Dry run failed for pipe '{self.code}' (PipeParallel): sub-pipe '{sub_pipe.pipe_code}' not found"
raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode, pipe_code=self.code) from exc
- # 3. Run all sub-pipes in dry mode
+ # 2. Run all sub-pipes in dry mode
tasks: list[Coroutine[Any, Any, PipeOutput]] = []
for sub_pipe in self.parallel_sub_pipes:
@@ -251,7 +233,7 @@ class PipeParallel(PipeController):
pipe_outputs = await asyncio.gather(*tasks)
- # 4. Process outputs as in the regular run
+ # 3. Process outputs as in the regular run
output_stuffs: dict[str, Stuff] = {}
output_stuff_contents: dict[str, StuffContent] = {}
@@ -276,7 +258,7 @@ class PipeParallel(PipeController):
output_stuffs[sub_pipe_output_name] = output_stuff
output_stuff_contents[sub_pipe_output_name] = output_stuff.content
- # 5. Handle combined output if specified
+ # 4. Handle combined output if specified
if self.combined_output:
combined_output_stuff = StuffFactory.combine_stuffs(
concept=self.combined_output,
@@ -291,3 +273,15 @@ class PipeParallel(PipeController):
working_memory=working_memory,
pipeline_run_id=job_metadata.pipeline_run_id,
)
+
+ @override
+ async def _validate_before_run(
+ self, job_metadata: JobMetadata, working_memory: WorkingMemory, pipe_run_params: PipeRunParams, output_name: str | None = None
+ ):
+ pass
+
+ @override
+ async def _validate_after_run(
+ self, job_metadata: JobMetadata, working_memory: WorkingMemory, pipe_run_params: PipeRunParams, output_name: str | None = None
+ ):
+ pass
diff --git a/pipelex/pipe_controllers/pipe_controller.py b/pipelex/pipe_controllers/pipe_controller.py
index c7ee46a30..444fc15c8 100644
--- a/pipelex/pipe_controllers/pipe_controller.py
+++ b/pipelex/pipe_controllers/pipe_controller.py
@@ -6,7 +6,7 @@ from typing_extensions import override
from pipelex.core.memory.working_memory import WorkingMemory
from pipelex.core.pipes.pipe_abstract import PipeAbstract
from pipelex.core.pipes.pipe_output import PipeOutput
-from pipelex.pipe_run.pipe_run_params import PipeRunMode, PipeRunParams
+from pipelex.pipe_run.pipe_run_params import PipeRunParams
from pipelex.pipeline.job_metadata import JobMetadata
@@ -27,33 +27,28 @@ class PipeController(PipeAbstract):
@final
@override
- async def _run_pipe(
+ async def _live_run_pipe(
self,
job_metadata: JobMetadata,
working_memory: WorkingMemory,
pipe_run_params: PipeRunParams,
output_name: str | None = None,
- print_intermediate_outputs: bool | None = False,
) -> PipeOutput:
- match pipe_run_params.run_mode:
- case PipeRunMode.LIVE:
- pipe_output = await self._run_controller_pipe(
- job_metadata=job_metadata,
- working_memory=working_memory,
- pipe_run_params=pipe_run_params,
- output_name=output_name,
- )
- case PipeRunMode.DRY:
- pipe_output = await self._dry_run_controller_pipe(
- job_metadata=job_metadata,
- working_memory=working_memory,
- pipe_run_params=pipe_run_params,
- output_name=output_name,
- )
- return pipe_output
+ return await self._live_run_controller_pipe(
+ job_metadata=job_metadata, working_memory=working_memory, pipe_run_params=pipe_run_params, output_name=output_name
+ )
+
+ @final
+ @override
+ async def _dry_run_pipe(
+ self, job_metadata: JobMetadata, working_memory: WorkingMemory, pipe_run_params: PipeRunParams, output_name: str | None = None
+ ) -> PipeOutput:
+ return await self._dry_run_controller_pipe(
+ job_metadata=job_metadata, working_memory=working_memory, pipe_run_params=pipe_run_params, output_name=output_name
+ )
@abstractmethod
- async def _run_controller_pipe(
+ async def _live_run_controller_pipe(
self,
job_metadata: JobMetadata,
working_memory: WorkingMemory,
diff --git a/pipelex/pipe_controllers/sequence/pipe_sequence.py b/pipelex/pipe_controllers/sequence/pipe_sequence.py
index 3c8d3d410..b689158de 100644
--- a/pipelex/pipe_controllers/sequence/pipe_sequence.py
+++ b/pipelex/pipe_controllers/sequence/pipe_sequence.py
@@ -11,12 +11,10 @@ from pipelex.core.pipes.inputs.input_stuff_specs_factory import InputStuffSpecsF
from pipelex.core.pipes.pipe_output import PipeOutput
from pipelex.core.pipes.variable_multiplicity import is_multiplicity_compatible
from pipelex.hub import get_concept_library, get_required_pipe
-from pipelex.pipe_controllers.exceptions import PipeControllerOutputConceptMismatchError
from pipelex.pipe_controllers.parallel.pipe_parallel import PipeParallel
from pipelex.pipe_controllers.pipe_controller import PipeController
from pipelex.pipe_controllers.sequence.exceptions import PipeSequenceValueError
from pipelex.pipe_controllers.sub_pipe import SubPipe
-from pipelex.pipe_run.exceptions import PipeRunError, PipeRunParamsError
from pipelex.pipe_run.pipe_run_params import PipeRunParams
from pipelex.pipeline.job_metadata import JobMetadata
@@ -158,18 +156,13 @@ class PipeSequence(PipeController):
return {sub_pipe.pipe_code for sub_pipe in self.sequential_sub_pipes}
@override
- async def _run_controller_pipe(
+ async def _live_run_controller_pipe(
self,
job_metadata: JobMetadata,
working_memory: WorkingMemory,
pipe_run_params: PipeRunParams,
output_name: str | None = None,
) -> PipeOutput:
- pipe_run_params.push_pipe_layer(pipe_code=self.code)
- if pipe_run_params.is_multiple_output_required:
- msg = f"{self.__class__.__name__} does not support multiple outputs, got output_multiplicity = {pipe_run_params.output_multiplicity}"
- raise PipeRunParamsError(msg)
-
evolving_memory = working_memory
for sub_pipe_index, sub_pipe in enumerate(self.sequential_sub_pipes):
@@ -198,24 +191,21 @@ class PipeSequence(PipeController):
pipe_run_params: PipeRunParams,
output_name: str | None = None,
) -> PipeOutput:
- if not pipe_run_params.run_mode.is_dry:
- msg = f"PipeSequence._dry_run_controller_pipe() called with run_mode = {pipe_run_params.run_mode} in pipe {self.code}"
- raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode, pipe_code=self.code)
- # Verify the output of this pipe is matching the output of the last step.
-
- concept_of_last_step = get_required_pipe(pipe_code=self.sequential_sub_pipes[-1].pipe_code).output.concept
- # if self.output.concept_ref != concept_ref_of_last_step:
- if not get_concept_library().is_compatible(tested_concept=concept_of_last_step, wanted_concept=self.output.concept):
- msg = f"""PipeSequence concept mismatch:
-the output concept '{concept_of_last_step.concept_ref}' of the last step '{self.sequential_sub_pipes[-1].pipe_code}'
-of sequence pipe '{self.code}' is not compatible with the output concept '{self.output.concept.concept_ref}' of the sequence.
-"""
- raise PipeControllerOutputConceptMismatchError(
- message=msg, tested_concept=concept_of_last_step.concept_ref, wanted_concept=self.output.concept.concept_ref
- )
- return await self._run_controller_pipe(
+ return await self._live_run_controller_pipe(
job_metadata=job_metadata,
working_memory=working_memory,
pipe_run_params=pipe_run_params,
output_name=output_name,
)
+
+ @override
+ async def _validate_before_run(
+ self, job_metadata: JobMetadata, working_memory: WorkingMemory, pipe_run_params: PipeRunParams, output_name: str | None = None
+ ):
+ pass
+
+ @override
+ async def _validate_after_run(
+ self, job_metadata: JobMetadata, working_memory: WorkingMemory, pipe_run_params: PipeRunParams, output_name: str | None = None
+ ):
+ pass
diff --git a/pipelex/pipe_controllers/sub_pipe.py b/pipelex/pipe_controllers/sub_pipe.py
index f11d3d0a9..ff2736096 100644
--- a/pipelex/pipe_controllers/sub_pipe.py
+++ b/pipelex/pipe_controllers/sub_pipe.py
@@ -29,7 +29,6 @@ class SubPipe(BaseModel):
job_metadata: JobMetadata,
sub_pipe_run_params: PipeRunParams,
) -> PipeOutput:
- """Run or dry run a single operation self."""
if self.output_multiplicity:
sub_pipe_run_params.output_multiplicity = self.output_multiplicity
@@ -49,7 +48,6 @@ class SubPipe(BaseModel):
message=msg,
run_mode=sub_pipe_run_params.run_mode,
pipe_code=self.pipe_code,
- missing_inputs={},
variable_name=batch_params.input_list_stuff_name,
concept_code=None,
) from exc
@@ -65,7 +63,6 @@ class SubPipe(BaseModel):
message=msg,
run_mode=sub_pipe_run_params.run_mode,
pipe_code=self.pipe_code,
- missing_inputs={},
variable_name=batch_params.input_item_stuff_name,
concept_code=None,
) from exc
@@ -116,7 +113,6 @@ class SubPipe(BaseModel):
message=msg,
run_mode=sub_pipe_run_params.run_mode,
pipe_code=self.pipe_code,
- missing_inputs={},
variable_name=exc.variable_name,
concept_code=None,
) from exc
@@ -127,6 +123,7 @@ class SubPipe(BaseModel):
pipe_run_params=sub_pipe_run_params,
output_name=self.output_name,
)
+
if new_output_stuff := pipe_output.working_memory.get_optional_main_stuff():
for stuff in required_stuffs:
get_pipeline_tracker().add_pipe_step(
@@ -136,4 +133,5 @@ class SubPipe(BaseModel):
pipe_layer=sub_pipe_run_params.pipe_layers,
comment="SubPipe on required_stuff",
)
+
return pipe_output
diff --git a/pipelex/pipe_operators/compose/pipe_compose.py b/pipelex/pipe_operators/compose/pipe_compose.py
index 52ccb4fbd..44b70fd0e 100644
--- a/pipelex/pipe_operators/compose/pipe_compose.py
+++ b/pipelex/pipe_operators/compose/pipe_compose.py
@@ -19,7 +19,6 @@ from pipelex.core.stuffs.stuff_content import StuffContent
from pipelex.core.stuffs.stuff_factory import StuffFactory
from pipelex.hub import get_class_registry, get_concept_library, get_content_generator, get_native_concept
from pipelex.pipe_operators.pipe_operator import PipeOperator
-from pipelex.pipe_run.exceptions import PipeRunParamsError
from pipelex.pipe_run.pipe_run_mode import PipeRunMode
from pipelex.pipe_run.pipe_run_params import PipeRunParams
from pipelex.pipe_run.pipe_run_params_factory import PipeRunParamsFactory
@@ -102,7 +101,7 @@ class PipeCompose(PipeOperator[PipeComposeOutput]):
)
@override
- async def _run_operator_pipe(
+ async def _live_run_operator_pipe(
self,
job_metadata: JobMetadata,
working_memory: WorkingMemory,
@@ -111,9 +110,6 @@ class PipeCompose(PipeOperator[PipeComposeOutput]):
content_generator: ContentGeneratorProtocol | None = None,
) -> PipeComposeOutput:
content_generator = content_generator or get_content_generator()
- if pipe_run_params.is_multiple_output_required:
- msg = f"PipeCompose does not suppport multiple outputs, got output_multiplicity = {pipe_run_params.output_multiplicity}"
- raise PipeRunParamsError(msg)
context: dict[str, Any] = working_memory.generate_context()
if pipe_run_params:
@@ -165,10 +161,22 @@ class PipeCompose(PipeOperator[PipeComposeOutput]):
log.verbose(f"PipeCompose: using regular operator pipe for jinja2 rendering (dry run not applied to jinja2): {self.code}")
content_generator_used = get_content_generator()
- return await self._run_operator_pipe(
+ return await self._live_run_operator_pipe(
job_metadata=job_metadata,
working_memory=working_memory,
pipe_run_params=pipe_run_params or PipeRunParamsFactory.make_run_params(pipe_run_mode=PipeRunMode.DRY),
output_name=output_name,
content_generator=content_generator_used,
)
+
+ @override
+ async def _validate_before_run(
+ self, job_metadata: JobMetadata, working_memory: WorkingMemory, pipe_run_params: PipeRunParams, output_name: str | None = None
+ ):
+ pass
+
+ @override
+ async def _validate_after_run(
+ self, job_metadata: JobMetadata, working_memory: WorkingMemory, pipe_run_params: PipeRunParams, output_name: str | None = None
+ ):
+ pass
diff --git a/pipelex/pipe_operators/compose/pipe_compose_blueprint.py b/pipelex/pipe_operators/compose/pipe_compose_blueprint.py
index ccee5f68a..a08e7e9c1 100644
--- a/pipelex/pipe_operators/compose/pipe_compose_blueprint.py
+++ b/pipelex/pipe_operators/compose/pipe_compose_blueprint.py
@@ -7,6 +7,7 @@ from pipelex.cogt.templating.template_category import TemplateCategory
from pipelex.cogt.templating.template_preprocessor import preprocess_template
from pipelex.cogt.templating.templating_style import TemplatingStyle
from pipelex.core.pipes.pipe_blueprint import PipeBlueprint
+from pipelex.core.pipes.variable_multiplicity import parse_concept_with_multiplicity
from pipelex.tools.jinja2.jinja2_errors import Jinja2TemplateSyntaxError
from pipelex.tools.jinja2.jinja2_parsing import check_jinja2_parsing
from pipelex.tools.jinja2.jinja2_required_variables import detect_jinja2_required_variables
@@ -70,4 +71,10 @@ class PipeComposeBlueprint(PipeBlueprint):
@override
def validate_output(self):
- pass
+ parsed_output = parse_concept_with_multiplicity(concept_ref=self.output)
+ if parsed_output.multiplicity:
+ msg = (
+ "PipeCompose does not support multiple output generation. The output of PipeCompose must be a single stuff, "
+ f"from a concept that refines the native Text concept. Current output: {self.output}"
+ )
+ raise ValueError(msg)
diff --git a/pipelex/pipe_operators/extract/pipe_extract.py b/pipelex/pipe_operators/extract/pipe_extract.py
index 170c10b70..4a9a78bd7 100644
--- a/pipelex/pipe_operators/extract/pipe_extract.py
+++ b/pipelex/pipe_operators/extract/pipe_extract.py
@@ -28,8 +28,6 @@ from pipelex.hub import (
get_native_concept,
)
from pipelex.pipe_operators.pipe_operator import PipeOperator
-from pipelex.pipe_run.exceptions import PipeRunError
-from pipelex.pipe_run.pipe_run_mode import PipeRunMode
from pipelex.pipe_run.pipe_run_params import PipeRunParams
from pipelex.pipeline.job_metadata import JobMetadata
from pipelex.tools.pdf.pypdfium2_renderer import pypdfium2_renderer
@@ -96,7 +94,7 @@ class PipeExtract(PipeOperator[PipeExtractOutput]):
)
@override
- async def _run_operator_pipe(
+ async def _live_run_operator_pipe(
self,
job_metadata: JobMetadata,
working_memory: WorkingMemory,
@@ -205,16 +203,22 @@ class PipeExtract(PipeOperator[PipeExtractOutput]):
pipe_run_params: PipeRunParams,
output_name: str | None = None,
) -> PipeExtractOutput:
- log.verbose(f"PipeExtract: dry run operator pipe: {self.code}")
- if pipe_run_params.run_mode != PipeRunMode.DRY:
- msg = f"Running pipe '{self.code}' (PipeExtract) _dry_run_operator_pipe() in non-dry mode is not allowed."
- raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode, pipe_code=self.code)
-
- content_generator_dry = ContentGeneratorDry()
- return await self._run_operator_pipe(
+ return await self._live_run_operator_pipe(
job_metadata=job_metadata,
working_memory=working_memory,
pipe_run_params=pipe_run_params,
output_name=output_name,
- content_generator=content_generator_dry,
+ content_generator=ContentGeneratorDry(),
)
+
+ @override
+ async def _validate_before_run(
+ self, job_metadata: JobMetadata, working_memory: WorkingMemory, pipe_run_params: PipeRunParams, output_name: str | None = None
+ ):
+ pass
+
+ @override
+ async def _validate_after_run(
+ self, job_metadata: JobMetadata, working_memory: WorkingMemory, pipe_run_params: PipeRunParams, output_name: str | None = None
+ ):
+ pass
diff --git a/pipelex/pipe_operators/func/pipe_func.py b/pipelex/pipe_operators/func/pipe_func.py
index ed65b4dab..4af575e2e 100644
--- a/pipelex/pipe_operators/func/pipe_func.py
+++ b/pipelex/pipe_operators/func/pipe_func.py
@@ -69,18 +69,39 @@ class PipeFunc(PipeOperator[PipeFuncOutput]):
@override
def validate_output_with_library(self):
- pass
+ function = func_registry.get_required_function(self.function_name)
+ return_type = get_type_hints(function).get("return")
+ if return_type is None:
+ msg = (
+ f"PipeFunc '{self.code}' failed to validate output with library: The return type of the function is None. "
+ "It should be a subclass of StuffContent."
+ )
+ raise TypeError(msg)
+ if self.output.multiplicity and not issubclass(return_type, ListContent):
+ msg = (
+ f"PipeFunc '{self.code}' output multiplicity is '{self.output.multiplicity}', but the function '{self.function_name}' "
+ f"return type {return_type} is not a subclass of ListContent. The output of your PipeFunc is "
+ f"'{self.output.to_bundle_representation()}'. The return type of your function should be a subclass of ListContent."
+ )
+ raise TypeError(msg)
+ if not self.output.multiplicity and issubclass(return_type, ListContent):
+ msg = (
+ f"PipeFunc '{self.code}' output multiplicity is '{self.output.multiplicity}', but the function '{self.function_name}' "
+ f"return type {return_type} is a subclass of ListContent. The output of your PipeFunc is "
+ f"'{self.output.concept.concept_ref}{self.output.to_bundle_representation()}' "
+ f"when it should be '{self.output.concept.concept_ref}' (no multiplicity)."
+ )
+ raise TypeError(msg)
@override
- async def _run_operator_pipe(
+ async def _live_run_operator_pipe(
self,
job_metadata: JobMetadata,
working_memory: WorkingMemory,
pipe_run_params: PipeRunParams,
output_name: str | None = None,
) -> PipeFuncOutput:
- log.verbose(f"Applying function '{self.function_name}'")
-
+ log.verbose(f"Running PipeFunc with function '{self.function_name}'")
function = func_registry.get_required_function(self.function_name)
if asyncio.iscoroutinefunction(function):
@@ -124,31 +145,10 @@ class PipeFunc(PipeOperator[PipeFuncOutput]):
pipe_run_params: PipeRunParams,
output_name: str | None = None,
) -> PipeFuncOutput:
- log.verbose(f"Dry run for PipeFunc '{self.function_name}'")
-
function = func_registry.get_required_function(self.function_name)
-
- # Check that all needed inputs are present in working memory
- needed_inputs = self.needed_inputs()
-
- missing_input_names: list[str] = []
- for named_stuff_spec in needed_inputs.named_stuff_specs:
- if not working_memory.get_optional_stuff(named_stuff_spec.variable_name):
- missing_input_names.append(named_stuff_spec.variable_name)
- if missing_input_names:
- msg = f"Dry run failed for pipe '{self.code}' (PipeFunc): missing required inputs: {missing_input_names}"
- raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode, pipe_code=self.code)
-
return_type = get_type_hints(function).get("return")
-
if return_type is None:
- msg = f"Dry run failed for pipe '{self.code}' (PipeFunc): function '{self.function_name}' has no return type annotation"
- raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode, pipe_code=self.code)
- if not issubclass(return_type, StuffContent):
- msg = (
- f"Dry run failed for pipe '{self.code}' (PipeFunc): "
- f"function '{self.function_name}' return type {return_type} is not a subclass of StuffContent"
- )
+ msg = f"Dry run of {self.type} '{self.code}' failed: The return type of the function is None. It should be a subclass of StuffContent."
raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode, pipe_code=self.code)
# TODO: Support PipeFunc returning with multiplicity. Create an equivalent of TypedNamedInputRequirement for outputs.
@@ -180,3 +180,27 @@ class PipeFunc(PipeOperator[PipeFuncOutput]):
working_memory=working_memory,
pipeline_run_id=job_metadata.pipeline_run_id,
)
+
+ @override
+ async def _validate_before_run(
+ self, job_metadata: JobMetadata, working_memory: WorkingMemory, pipe_run_params: PipeRunParams, output_name: str | None = None
+ ):
+ function = func_registry.get_required_function(self.function_name)
+ return_type = get_type_hints(function).get("return")
+ # TODO: this should not happend ever. The correct way to do this would be to have a unit test making sure
+ # that the FuncRegistry DOES CALL the 'is_eligible_function' function, and this function should be unit tested.
+ if return_type is None:
+ msg = f"Dry run failed for {self.type} '{self.code}': function '{self.function_name}' has no return type annotation"
+ raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode, pipe_code=self.code)
+ if not issubclass(return_type, StuffContent):
+ msg = (
+ f"Dry run failed for pipe {self.type} '{self.code}': "
+ f"function '{self.function_name}' return type {return_type} is not a subclass of StuffContent"
+ )
+ raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode, pipe_code=self.code)
+
+ @override
+ async def _validate_after_run(
+ self, job_metadata: JobMetadata, working_memory: WorkingMemory, pipe_run_params: PipeRunParams, output_name: str | None = None
+ ):
+ pass
diff --git a/pipelex/pipe_operators/img_gen/pipe_img_gen.py b/pipelex/pipe_operators/img_gen/pipe_img_gen.py
index 1f65309fb..123c77670 100644
--- a/pipelex/pipe_operators/img_gen/pipe_img_gen.py
+++ b/pipelex/pipe_operators/img_gen/pipe_img_gen.py
@@ -99,7 +99,7 @@ class PipeImgGen(PipeOperator[PipeImgGenOutput]):
)
@override
- async def _run_operator_pipe(
+ async def _live_run_operator_pipe(
self,
job_metadata: JobMetadata,
working_memory: WorkingMemory,
@@ -187,7 +187,7 @@ class PipeImgGen(PipeOperator[PipeImgGenOutput]):
else:
nb_images = 1
- # Get the structure class from the registry (might be a subclass of ImageContent)
+ # Get the structure class from the registry (Has to be a subclass of ImageContent)
structure_class = get_class_registry().get_required_subclass(
name=self.output.concept.structure_class_name,
base_class=ImageContent,
@@ -238,7 +238,7 @@ class PipeImgGen(PipeOperator[PipeImgGenOutput]):
source_prompt=img_gen_prompt_text,
base_64=base_64_str,
)
- log.verbose(the_content, title="Single image content")
+ log.verbose(the_content, title=f"output stuff content of PipeImg {self.code}")
output_stuff = StuffFactory.make_stuff(
name=output_name,
@@ -264,11 +264,22 @@ class PipeImgGen(PipeOperator[PipeImgGenOutput]):
pipe_run_params: PipeRunParams,
output_name: str | None = None,
) -> PipeImgGenOutput:
- content_generator_dry = ContentGeneratorDry()
- return await self._run_operator_pipe(
+ return await self._live_run_operator_pipe(
job_metadata=job_metadata,
working_memory=working_memory,
pipe_run_params=pipe_run_params or PipeRunParamsFactory.make_run_params(pipe_run_mode=PipeRunMode.DRY),
output_name=output_name,
- content_generator=content_generator_dry,
+ content_generator=ContentGeneratorDry(),
)
+
+ @override
+ async def _validate_before_run(
+ self, job_metadata: JobMetadata, working_memory: WorkingMemory, pipe_run_params: PipeRunParams, output_name: str | None = None
+ ):
+ pass
+
+ @override
+ async def _validate_after_run(
+ self, job_metadata: JobMetadata, working_memory: WorkingMemory, pipe_run_params: PipeRunParams, output_name: str | None = None
+ ):
+ pass
diff --git a/pipelex/pipe_operators/llm/helpers.py b/pipelex/pipe_operators/llm/helpers.py
new file mode 100644
index 000000000..4d7b39a9e
--- /dev/null
+++ b/pipelex/pipe_operators/llm/helpers.py
@@ -0,0 +1,35 @@
+from pipelex.cogt.templating.template_category import TemplateCategory
+from pipelex.config import get_config
+from pipelex.core.stuffs.stuff_content import StuffContent
+from pipelex.hub import (
+ get_class_registry,
+ get_content_generator,
+ get_required_concept,
+)
+from pipelex.tools.typing.structure_printer import StructurePrinter
+
+
+async def get_output_structure_prompt(concept_ref: str, is_with_preliminary_text: bool) -> str | None:
+ concept = get_required_concept(concept_ref=concept_ref)
+ output_class = get_class_registry().get_class(concept.structure_class_name)
+ if not output_class:
+ return None
+
+ class_structure = StructurePrinter().get_type_structure(tp=output_class, base_class=StuffContent)
+
+ if not class_structure:
+ return None
+ class_structure_str = "\n".join(class_structure)
+ llm_config = get_config().cogt.llm_config
+ if is_with_preliminary_text:
+ template_source = llm_config.get_template(template_name="output_structure_prompt")
+ else:
+ template_source = llm_config.get_template(template_name="output_structure_prompt_no_preliminary_text")
+
+ return await get_content_generator().make_templated_text(
+ context={
+ "class_structure_str": class_structure_str,
+ },
+ template=template_source,
+ template_category=TemplateCategory.LLM_PROMPT,
+ )
diff --git a/pipelex/pipe_operators/llm/pipe_llm.py b/pipelex/pipe_operators/llm/pipe_llm.py
index 489929113..c6b964ed5 100644
--- a/pipelex/pipe_operators/llm/pipe_llm.py
+++ b/pipelex/pipe_operators/llm/pipe_llm.py
@@ -12,7 +12,6 @@ from pipelex.cogt.llm.llm_prompt_factory_abstract import LLMPromptFactoryAbstrac
from pipelex.cogt.llm.llm_prompt_template import LLMPromptTemplate
from pipelex.cogt.llm.llm_setting import LLMModelChoice, LLMSetting, LLMSettingChoices
from pipelex.cogt.models.model_deck_check import check_llm_choice_with_deck
-from pipelex.cogt.templating.template_category import TemplateCategory
from pipelex.config import get_config
from pipelex.core.concepts.concept import Concept
from pipelex.core.concepts.concept_factory import ConceptFactory
@@ -21,7 +20,6 @@ from pipelex.core.domains.domain import SpecialDomain
from pipelex.core.memory.working_memory import WorkingMemory
from pipelex.core.pipes.exceptions import PipeValidationError, PipeValidationErrorType
from pipelex.core.pipes.inputs.input_stuff_specs import InputStuffSpecs
-from pipelex.core.pipes.inputs.input_stuff_specs_factory import InputStuffSpecsFactory
from pipelex.core.pipes.pipe_output import PipeOutput
from pipelex.core.pipes.variable_multiplicity import VariableMultiplicity
from pipelex.core.stuffs.list_content import ListContent
@@ -35,6 +33,7 @@ from pipelex.hub import (
get_native_concept,
get_required_concept,
)
+from pipelex.pipe_operators.llm.helpers import get_output_structure_prompt
from pipelex.pipe_operators.llm.llm_prompt_blueprint import LLMPromptBlueprint
from pipelex.pipe_operators.llm.pipe_llm_blueprint import StructuringMethod
from pipelex.pipe_operators.pipe_operator import PipeOperator
@@ -46,7 +45,6 @@ from pipelex.pipe_run.pipe_run_params import (
)
from pipelex.pipeline.job_metadata import JobMetadata
from pipelex.tools.typing.pydantic_utils import format_pydantic_validation_error
-from pipelex.tools.typing.structure_printer import StructurePrinter
from pipelex.types import Self
@@ -112,20 +110,14 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):
@override
def needed_inputs(self, visited_pipes: set[str] | None = None) -> InputStuffSpecs:
- """Needed inputs are the inputs needed to run the pipe, specified in the inputs attribute of the pipe"""
- needed_inputs = InputStuffSpecsFactory.make_empty()
-
- for input_name, stuff_spec in self.inputs.items:
- needed_inputs.add_stuff_spec(variable_name=input_name, concept=stuff_spec.concept, multiplicity=stuff_spec.multiplicity)
- return needed_inputs
+ return self.inputs
@override
def required_variables(self) -> set[str]:
- """Required variables are the variables that are used in the current prompt template or system prompt"""
return {variable_name for variable_name in self.llm_prompt_spec.required_variables() if not variable_name.startswith("_")}
@override
- async def _run_operator_pipe(
+ async def _live_run_operator_pipe(
self,
job_metadata: JobMetadata,
working_memory: WorkingMemory,
@@ -133,7 +125,6 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):
output_name: str | None = None,
content_generator: ContentGeneratorProtocol | None = None,
) -> PipeLLMOutput:
- llm_config = get_config().cogt.llm_config
content_generator = content_generator or get_content_generator()
# interpret / unwrap the arguments
output_stuff_spec = self.output
@@ -271,8 +262,8 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):
llm_prompt_2_factory = None
output_structure_prompt: str | None = None
- if llm_config.is_structure_prompt_enabled:
- output_structure_prompt = await PipeLLM.get_output_structure_prompt(
+ if get_config().cogt.llm_config.is_structure_prompt_enabled:
+ output_structure_prompt = await get_output_structure_prompt(
concept_ref=pipe_run_params.dynamic_output_concept_code or output_stuff_spec.concept.concept_ref,
is_with_preliminary_text=is_with_preliminary_text,
)
@@ -440,37 +431,22 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):
pipe_run_params: PipeRunParams,
output_name: str | None = None,
) -> PipeLLMOutput:
- content_generator_dry = ContentGeneratorDry()
- return await self._run_operator_pipe(
+ return await self._live_run_operator_pipe(
job_metadata=job_metadata,
working_memory=working_memory,
pipe_run_params=pipe_run_params,
output_name=output_name,
- content_generator=content_generator_dry,
+ content_generator=ContentGeneratorDry(),
)
- @staticmethod
- async def get_output_structure_prompt(concept_ref: str, is_with_preliminary_text: bool) -> str | None:
- concept = get_required_concept(concept_ref=concept_ref)
- output_class = get_class_registry().get_class(concept.structure_class_name)
- if not output_class:
- return None
-
- class_structure = StructurePrinter().get_type_structure(tp=output_class, base_class=StuffContent)
-
- if not class_structure:
- return None
- class_structure_str = "\n".join(class_structure)
- llm_config = get_config().cogt.llm_config
- if is_with_preliminary_text:
- template_source = llm_config.get_template(template_name="output_structure_prompt")
- else:
- template_source = llm_config.get_template(template_name="output_structure_prompt_no_preliminary_text")
-
- return await get_content_generator().make_templated_text(
- context={
- "class_structure_str": class_structure_str,
- },
- template=template_source,
- template_category=TemplateCategory.LLM_PROMPT,
- )
+ @override
+ async def _validate_before_run(
+ self, job_metadata: JobMetadata, working_memory: WorkingMemory, pipe_run_params: PipeRunParams, output_name: str | None = None
+ ):
+ pass
+
+ @override
+ async def _validate_after_run(
+ self, job_metadata: JobMetadata, working_memory: WorkingMemory, pipe_run_params: PipeRunParams, output_name: str | None = None
+ ):
+ pass
diff --git a/pipelex/pipe_operators/pipe_operator.py b/pipelex/pipe_operators/pipe_operator.py
index 8453349f9..c8b2a94f0 100644
--- a/pipelex/pipe_operators/pipe_operator.py
+++ b/pipelex/pipe_operators/pipe_operator.py
@@ -8,7 +8,6 @@ from pipelex.core.memory.working_memory import WorkingMemory
from pipelex.core.pipes.pipe_abstract import PipeAbstract
from pipelex.core.pipes.pipe_output import PipeOutput
from pipelex.pipe_operators.exceptions import PipeOperatorModelAvailabilityError
-from pipelex.pipe_run.pipe_run_mode import PipeRunMode
from pipelex.pipe_run.pipe_run_params import PipeRunParams
from pipelex.pipeline.job_metadata import JobMetadata
@@ -28,7 +27,7 @@ class PipeOperator(PipeAbstract, Generic[PipeOperatorOutputType]):
@final
@override
- async def _run_pipe(
+ async def _live_run_pipe(
self,
job_metadata: JobMetadata,
working_memory: WorkingMemory,
@@ -36,33 +35,23 @@ class PipeOperator(PipeAbstract, Generic[PipeOperatorOutputType]):
output_name: str | None = None,
) -> PipeOutput:
try:
- match pipe_run_params.run_mode:
- case PipeRunMode.LIVE:
- pipe_output = await self._run_operator_pipe(
- job_metadata=job_metadata,
- working_memory=working_memory,
- pipe_run_params=pipe_run_params,
- output_name=output_name,
- )
- main_stuff = pipe_output.main_stuff
- output_concept_code = self.output.concept.code
- output_concept_with_multiplicity = f"[bold green]{output_concept_code}[/bold green]"
- if main_stuff.is_list:
- list_content: ListContent[StuffContent] = main_stuff.as_list_content() # pyright: ignore[reportUnknownVariableType, reportUnknownMemberType]
- nb_items = len(list_content.items)
- if nb_items == 1:
- output_concept_with_multiplicity += " [1 item]"
- else:
- output_concept_with_multiplicity += f" [{nb_items} items]"
- title = f"Output of pipe [red]{self.code}[/red] [yellow]→[/yellow] {output_concept_with_multiplicity}"
- main_stuff.pretty_print_stuff(title=title)
- case PipeRunMode.DRY:
- pipe_output = await self._dry_run_operator_pipe(
- job_metadata=job_metadata,
- working_memory=working_memory,
- pipe_run_params=pipe_run_params,
- output_name=output_name,
- )
+ pipe_output = await self._live_run_operator_pipe(
+ job_metadata=job_metadata, working_memory=working_memory, pipe_run_params=pipe_run_params, output_name=output_name
+ )
+
+ main_stuff = pipe_output.main_stuff
+ output_concept_code = self.output.concept.code
+ output_concept_with_multiplicity = f"[bold green]{output_concept_code}[/bold green]"
+ if main_stuff.is_list:
+ list_content: ListContent[StuffContent] = main_stuff.as_list_content() # pyright: ignore[reportUnknownVariableType, reportUnknownMemberType]
+ nb_items = len(list_content.items)
+ if nb_items == 1:
+ output_concept_with_multiplicity += " [1 item]"
+ else:
+ output_concept_with_multiplicity += f" [{nb_items} items]"
+ title = f"Output of pipe [red]{self.code}[/red] [yellow]→[/yellow] {output_concept_with_multiplicity}"
+ main_stuff.pretty_print_stuff(title=title)
+
except ModelWaterfallError as model_waterfall_error:
raise PipeOperatorModelAvailabilityError(
message=model_waterfall_error.message,
@@ -85,8 +74,17 @@ class PipeOperator(PipeAbstract, Generic[PipeOperatorOutputType]):
) from model_not_found_error
return pipe_output
+ @final
+ @override
+ async def _dry_run_pipe(
+ self, job_metadata: JobMetadata, working_memory: WorkingMemory, pipe_run_params: PipeRunParams, output_name: str | None = None
+ ) -> PipeOutput:
+ return await self._dry_run_operator_pipe(
+ job_metadata=job_metadata, working_memory=working_memory, pipe_run_params=pipe_run_params, output_name=output_name
+ )
+
@abstractmethod
- async def _run_operator_pipe(
+ async def _live_run_operator_pipe(
self,
job_metadata: JobMetadata,
working_memory: WorkingMemory,
diff --git a/pipelex/pipe_run/pipe_run_params.py b/pipelex/pipe_run/pipe_run_params.py
index 817db8690..1ae739dfc 100644
--- a/pipelex/pipe_run/pipe_run_params.py
+++ b/pipelex/pipe_run/pipe_run_params.py
@@ -8,6 +8,7 @@ from pipelex import log
from pipelex.core.memory.working_memory import BATCH_ITEM_STUFF_NAME, MAIN_STUFF_NAME
from pipelex.core.pipes.variable_multiplicity import VariableMultiplicity, VariableMultiplicityResolution
from pipelex.pipe_run.pipe_run_mode import PipeRunMode
+from pipelex.pipeline.exceptions import PipeStackOverflowError
from pipelex.types import Self, StrEnum
@@ -189,10 +190,14 @@ class PipeRunParams(BaseModel):
@property
def is_multiple_output_required(self) -> bool:
- return isinstance(self.output_multiplicity, int) and self.output_multiplicity > 1 # pyright: ignore[reportUnnecessaryIsInstance]
+ return isinstance(self.output_multiplicity, int) and self.output_multiplicity > 1
def push_pipe_to_stack(self, pipe_code: str) -> None:
self.pipe_stack.append(pipe_code)
+ limit = self.pipe_stack_limit
+ if len(self.pipe_stack) > limit:
+ msg = f"Exceeded pipe stack limit of {limit}. You can raise that limit in the config. Stack:\n{self.pipe_stack}"
+ raise PipeStackOverflowError(message=msg, limit=limit, pipe_stack=self.pipe_stack)
def pop_pipe_from_stack(self, pipe_code: str) -> None:
popped_pipe_code = self.pipe_stack.pop()
diff --git a/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.py b/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.py
index 92da2f747..a62a56503 100644
--- a/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.py
+++ b/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.py
@@ -56,8 +56,9 @@ class TestPipeConditionSimple:
# Verify it failed for the right reason
error = exc_info.value
assert error.pipe_code == "test_condition_fail"
+ assert error.missing_inputs is not None
assert "user_category" in error.missing_inputs
- assert "Missing required inputs" in str(error)
+ assert "missing required inputs" in str(error)
async def test_direct_pipe_condition_should_succeed(self, load_test_library: Callable[[list[Path]], None]):
"""Test a PipeCondition created directly in code that should SUCCEED dry run."""
diff --git a/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_complex.py b/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_complex.py
index 1c3f9299d..b719f6dac 100644
--- a/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_complex.py
+++ b/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_complex.py
@@ -300,7 +300,7 @@ class TestPipeConditionComplex:
error = exc_info.value
assert error.pipe_code == "complex_document_processor"
- assert "Missing required inputs" in str(error)
+ assert "missing required inputs" in str(error)
# Check the underlying cause is PipeRunInputsError with missing_inputs details
cause = error.__cause__
diff --git a/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_simple.py b/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_simple.py
index a8c5bad0e..1740588c1 100644
--- a/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_simple.py
+++ b/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_simple.py
@@ -242,7 +242,7 @@ class TestPipeConditionSimple:
# Verify the error details
error = exc_info.value
assert error.pipe_code == "basic_condition_by_category_2"
- assert "Missing required inputs" in str(error)
+ assert "missing required inputs" in str(error)
# Check the underlying cause is PipeRunInputsError with missing_inputs details
cause = error.__cause__
diff --git a/tests/unit/pipelex/core/pipes/test_stuff_spec.py b/tests/unit/pipelex/core/pipes/test_stuff_spec.py
new file mode 100644
index 000000000..222ea900f
--- /dev/null
+++ b/tests/unit/pipelex/core/pipes/test_stuff_spec.py
@@ -0,0 +1,85 @@
+import pytest
+
+from pipelex.core.concepts.concept_factory import ConceptFactory
+from pipelex.core.concepts.native.concept_native import NativeConceptCode
+from pipelex.core.pipes.stuff_spec.stuff_spec import StuffSpec
+from pipelex.core.pipes.variable_multiplicity import VariableMultiplicity
+
+
+class TestStuffSpecToBundleRepresentation:
+ """Test the StuffSpec.to_bundle_representation method."""
+
+ @pytest.mark.parametrize(
+ ("multiplicity", "expected_suffix"),
+ [
+ # No multiplicity - returns just concept_ref
+ (None, ""),
+ # Boolean multiplicity (variable-length list)
+ (True, "[]"),
+ # Integer multiplicity (fixed-length list)
+ (1, "[1]"),
+ (3, "[3]"),
+ (10, "[10]"),
+ (999, "[999]"),
+ ],
+ )
+ def test_to_bundle_representation_with_native_text(
+ self,
+ multiplicity: VariableMultiplicity | None,
+ expected_suffix: str,
+ ):
+ """Test to_bundle_representation with native.Text concept and various multiplicities."""
+ concept = ConceptFactory.make_native_concept(native_concept_code=NativeConceptCode.TEXT)
+ stuff_spec = StuffSpec(concept=concept, multiplicity=multiplicity)
+
+ result = stuff_spec.to_bundle_representation()
+
+ expected = f"{concept.concept_ref}{expected_suffix}"
+ assert result == expected
+ assert result == f"native.Text{expected_suffix}"
+
+ @pytest.mark.parametrize(
+ ("native_concept_code", "multiplicity", "expected"),
+ [
+ # Various native concepts without multiplicity
+ (NativeConceptCode.TEXT, None, "native.Text"),
+ (NativeConceptCode.IMAGE, None, "native.Image"),
+ (NativeConceptCode.PDF, None, "native.PDF"),
+ (NativeConceptCode.NUMBER, None, "native.Number"),
+ # Various native concepts with variable-length list
+ (NativeConceptCode.TEXT, True, "native.Text[]"),
+ (NativeConceptCode.IMAGE, True, "native.Image[]"),
+ # Various native concepts with fixed-length list
+ (NativeConceptCode.TEXT, 5, "native.Text[5]"),
+ (NativeConceptCode.IMAGE, 3, "native.Image[3]"),
+ ],
+ )
+ def test_to_bundle_representation_with_various_native_concepts(
+ self,
+ native_concept_code: NativeConceptCode,
+ multiplicity: VariableMultiplicity | None,
+ expected: str,
+ ):
+ """Test to_bundle_representation with different native concept types."""
+ concept = ConceptFactory.make_native_concept(native_concept_code=native_concept_code)
+ stuff_spec = StuffSpec(concept=concept, multiplicity=multiplicity)
+
+ result = stuff_spec.to_bundle_representation()
+
+ assert result == expected
+
+ def test_to_bundle_representation_with_custom_domain_concept(self):
+ """Test to_bundle_representation with a custom domain concept."""
+ concept = ConceptFactory.make(
+ domain_code="my_domain",
+ concept_code="MyCustomConcept",
+ description="A custom concept for testing",
+ structure_class_name="TextContent",
+ )
+ stuff_spec_single = StuffSpec(concept=concept, multiplicity=None)
+ stuff_spec_list = StuffSpec(concept=concept, multiplicity=True)
+ stuff_spec_fixed = StuffSpec(concept=concept, multiplicity=7)
+
+ assert stuff_spec_single.to_bundle_representation() == "my_domain.MyCustomConcept"
+ assert stuff_spec_list.to_bundle_representation() == "my_domain.MyCustomConcept[]"
+ assert stuff_spec_fixed.to_bundle_representation() == "my_domain.MyCustomConcept[7]"
diff --git a/tests/unit/pipelex/tools/test.plx b/tests/unit/pipelex/tools/test.plx
index d57a23de1..eab2abf75 100644
--- a/tests/unit/pipelex/tools/test.plx
+++ b/tests/unit/pipelex/tools/test.plx
@@ -9,6 +9,6 @@ FilePath = "The path of a file"
type = "PipeFunc"
description = "Read the content of related codebase files"
inputs = { related_file_paths = "FilePath" }
-output = "CodebaseFileContent"
+output = "CodebaseFileContent[]"
function_name = "read_file_content"