Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

Fix/dry run pipefunc (#201)

Thomas Hebrard545c15013a33
diff --git a/pipelex/core/stuff.py b/pipelex/core/stuff.py
index f7c3361a2..208c51701 100644
--- a/pipelex/core/stuff.py
+++ b/pipelex/core/stuff.py
@@ -40,8 +40,9 @@ class Stuff(CustomBaseModel):
if key in artefact_dict:
stuff_name = self.stuff_name or f"unnamed using concept code {self.concept_code}"
raise StuffError(
- f"""Cannot create stuff artefact for stuff {stuff_name} because reserved field {key} already exists in the stuff content.
- Forbidden fields are: `stuff_name`, `content_class`, `concept_code`, `stuff_code`, `content`"""
+ f"""Cannot create stuff artefact for stuff '{stuff_name}' of concept '{self.concept_code}' because reserved field '{key}'
+in the structured output '{self.content.__class__.__name__}' already exists in the stuff content.
+Forbidden fields are: 'stuff_name', 'content_class', 'concept_code', 'stuff_code', 'content'."""
)
artefact_dict[key] = value
diff --git a/pipelex/core/working_memory_factory.py b/pipelex/core/working_memory_factory.py
index 8cf20f118..adaf6f287 100644
--- a/pipelex/core/working_memory_factory.py
+++ b/pipelex/core/working_memory_factory.py
@@ -167,7 +167,7 @@ class WorkingMemoryFactory(BaseModel):
return working_memory
@classmethod
- def _create_mock_content(cls, requirement: TypedNamedInputRequirement) -> StuffContent:
+ def create_mock_content(cls, requirement: TypedNamedInputRequirement) -> StuffContent:
"""Helper method to create mock content for a requirement."""
if requirement.structure_class:
# Create mock object using polyfactory
@@ -204,7 +204,7 @@ class WorkingMemoryFactory(BaseModel):
try:
if not requirement.multiplicity:
- mock_content = cls._create_mock_content(requirement)
+ mock_content = cls.create_mock_content(requirement)
# Create stuff with mock content
mock_stuff = Stuff(
@@ -226,7 +226,7 @@ class WorkingMemoryFactory(BaseModel):
items: List[StuffContent] = []
for _ in range(nb_stuffs):
- item_mock_content = cls._create_mock_content(requirement)
+ item_mock_content = cls.create_mock_content(requirement)
items.append(item_mock_content)
mock_list_content = ListContent[StuffContent](items=items)
diff --git a/pipelex/pipe_operators/pipe_func.py b/pipelex/pipe_operators/pipe_func.py
index 15d139db6..0336ca214 100644
--- a/pipelex/pipe_operators/pipe_func.py
+++ b/pipelex/pipe_operators/pipe_func.py
@@ -1,14 +1,16 @@
-from typing import List, Optional, Set, cast
+from typing import List, Optional, Set, cast, get_type_hints
from typing_extensions import override
from pipelex import log
-from pipelex.core.pipe_input_spec import PipeInputSpec
+from pipelex.core.pipe_input_spec import PipeInputSpec, TypedNamedInputRequirement
from pipelex.core.pipe_output import PipeOutput
from pipelex.core.pipe_run_params import PipeRunParams
from pipelex.core.stuff_content import ListContent, StuffContent, TextContent
from pipelex.core.stuff_factory import StuffFactory
from pipelex.core.working_memory import WorkingMemory
+from pipelex.core.working_memory_factory import WorkingMemoryFactory
+from pipelex.exceptions import DryRunError
from pipelex.pipe_operators.pipe_operator import PipeOperator
from pipelex.pipeline.job_metadata import JobMetadata
from pipelex.tools.func_registry import func_registry
@@ -27,7 +29,7 @@ class PipeFunc(PipeOperator):
@override
def needed_inputs(self) -> PipeInputSpec:
- return PipeInputSpec.make_empty()
+ return self.inputs
@override
async def _run_operator_pipe(
@@ -80,7 +82,48 @@ class PipeFunc(PipeOperator):
pipe_run_params: PipeRunParams,
output_name: Optional[str] = None,
) -> PipeOutput:
- log.warning("Dry run not yet implemented for PipeFunc")
+ log.debug(f"Dry run for PipeFunc '{self.function_name}'")
+
+ function = func_registry.get_required_function(self.function_name)
+ if not callable(function):
+ raise ValueError(f"Function '{self.function_name}' is not callable")
+
+ # Check that all needed inputs are present in working memory
+ needed_inputs = self.needed_inputs()
+ for input_name, _ in needed_inputs.items:
+ if input_name not in working_memory.root:
+ raise DryRunError(
+ f"Required input '{input_name}' not found in working memory for function '{self.function_name}' in pipe '{self.code}'"
+ )
+
+ try:
+ return_type = get_type_hints(function).get("return")
+
+ if return_type is None:
+ raise DryRunError(f"Function '{self.function_name}' has no return type annotation")
+ else:
+ if not issubclass(return_type, StuffContent):
+ raise ValueError(f"Function '{self.function_name}' return type {return_type} is not a subclass of StuffContent")
+
+ requirement = TypedNamedInputRequirement(
+ variable_name="mock_output", concept_code=f"mock.{return_type.__name__}", structure_class=return_type, multiplicity=False
+ )
+ mock_content = WorkingMemoryFactory.create_mock_content(requirement)
+
+ except Exception as e:
+ raise DryRunError(f"Failed to get type hints for function '{self.function_name}' in pipe '{self.code}': {e}")
+
+ output_stuff = StuffFactory.make_stuff(
+ name=output_name,
+ concept_str=self.output_concept_code,
+ content=mock_content,
+ )
+
+ working_memory.set_new_main_stuff(
+ stuff=output_stuff,
+ name=output_name,
+ )
+
return PipeFuncOutput(
working_memory=working_memory,
pipeline_run_id=job_metadata.pipeline_run_id,
diff --git a/pipelex/pipe_works/pipe_dry.py b/pipelex/pipe_works/pipe_dry.py
index ddfb741a1..def6f0dbd 100644
--- a/pipelex/pipe_works/pipe_dry.py
+++ b/pipelex/pipe_works/pipe_dry.py
@@ -127,11 +127,11 @@ async def dry_run_pipes(pipes: List[PipeAbstract]) -> Dict[str, str]:
# Filter out pipes that are allowed to fail
unexpected_failures = {pipe_code: results[pipe_code] for pipe_code in failed_pipes if pipe_code not in allowed_to_fail_pipes}
- log.info(f"Dry run completed: {len(successful_pipes)} successful, {len(failed_pipes)} failed, in {time.time() - start_time:.2f} seconds")
+ log.info(f"Dry run completed: '{len(successful_pipes)}' successful, '{len(failed_pipes)}' failed, in '{time.time() - start_time:.2f}' seconds")
if unexpected_failures:
- unexpected_failures_details = "\n".join([f"{pipe_code}: {results[pipe_code]}" for pipe_code in unexpected_failures])
- raise Exception(f"Dry run failed with {len(unexpected_failures)} unexpected pipe failures: {unexpected_failures_details}")
+ unexpected_failures_details = "\n".join([f"'{pipe_code}': {results[pipe_code]}" for pipe_code in unexpected_failures])
+ raise Exception(f"Dry run failed with '{len(unexpected_failures)}' unexpected pipe failures:\n{unexpected_failures_details}")
if failed_pipes and not unexpected_failures:
log.info("All failures were expected (allowed by config)")
diff --git a/pyproject.toml b/pyproject.toml
index f4c4799ae..2eb105ab5 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
[project]
name = "pipelex"
-version = "0.6.7"
+version = "0.6.8"
description = "Pipelex is an open-source dev tool based on a simple declarative language that lets you define replicable, structured, composable LLM pipelines."
authors = [{ name = "Evotis S.A.S.", email = "[email protected]" }]
maintainers = [{ name = "Pipelex staff", email = "[email protected]" }]