Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

Make PipeParallel enforce "add each output" or "combined output" or both Stronger error handling in library loading.

Louis Choquelf6a55398baa0
diff --git a/pipelex/cli/commands/build_cmd.py b/pipelex/cli/commands/build_cmd.py
index c4e0dda4c..66e70c02a 100644
--- a/pipelex/cli/commands/build_cmd.py
+++ b/pipelex/cli/commands/build_cmd.py
@@ -32,7 +32,7 @@ def build_pipe_cmd(
output_path: Annotated[
str | None,
typer.Option("--output", "-o", help="Path to save the generated PLX file (use --output='' to skip saving)"),
- ] = "./generated_pipeline.plx",
+ ] = "./results/generated_pipeline.plx",
) -> None:
Pipelex.make(relative_config_folder_path="../../../pipelex/libraries", from_file=True)
typer.echo("=" * 70)
diff --git a/pipelex/core/bundles/pipelex_bundle_blueprint.py b/pipelex/core/bundles/pipelex_bundle_blueprint.py
index 66ce37b84..91cb86c39 100644
--- a/pipelex/core/bundles/pipelex_bundle_blueprint.py
+++ b/pipelex/core/bundles/pipelex_bundle_blueprint.py
@@ -31,6 +31,7 @@ PipeBlueprintUnion = Annotated[
class PipelexBundleBlueprint(BaseModel):
model_config = ConfigDict(extra="forbid")
+ source: str | None = None
domain: str
definition: str | None = None
system_prompt: str | None = None
diff --git a/pipelex/core/concepts/concept_factory.py b/pipelex/core/concepts/concept_factory.py
index f501974be..b80758831 100644
--- a/pipelex/core/concepts/concept_factory.py
+++ b/pipelex/core/concepts/concept_factory.py
@@ -13,7 +13,7 @@ from pipelex.core.concepts.concept_native import NativeConceptEnumData, NativeCo
from pipelex.core.concepts.structure_generator import StructureGenerator
from pipelex.core.domains.domain import SpecialDomain
from pipelex.core.stuffs.stuff_content import TextContent
-from pipelex.exceptions import ConceptFactoryError, StructureClassError
+from pipelex.exceptions import ConceptCodeError, ConceptDefinitionError, ConceptRefineError, StructureClassError
class DomainAndConceptCode(BaseModel):
@@ -106,16 +106,10 @@ class ConceptFactory:
@classmethod
def make_refine(cls, refine: str) -> str:
- if NativeConceptManager.is_native_concept(concept_string_or_code=refine):
- return NativeConceptManager.get_native_concept_string(concept_string_or_code=refine)
- msg = f"Refine '{refine}' is not a native concept"
- raise ConceptFactoryError(msg)
-
- @classmethod
- def make_refines(cls, blueprint: ConceptBlueprint) -> str | None:
- if blueprint.refines:
- return cls.make_refine(refine=blueprint.refines)
- return None
+ if not NativeConceptManager.is_native_concept(concept_string_or_code=refine):
+ msg = f"Refine '{refine}' is not a native concept"
+ raise ConceptRefineError(msg)
+ return NativeConceptManager.get_native_concept_string(concept_string_or_code=refine)
@classmethod
def make_from_blueprint(
@@ -125,7 +119,11 @@ class ConceptFactory:
blueprint: ConceptBlueprint,
concept_codes_from_the_same_domain: list[str] | None = None,
) -> Concept:
- ConceptBlueprint.validate_concept_code(concept_code=concept_code)
+ try:
+ ConceptBlueprint.validate_concept_code(concept_code=concept_code)
+ except ConceptCodeError as exc:
+ msg = f"Could not validate concept code '{concept_code}' in domain '{domain}': {exc}"
+ raise ConceptDefinitionError(msg) from exc
structure_class_name: str
current_refine: str | None = None
@@ -163,7 +161,7 @@ class ConceptFactory:
except Exception as exc:
msg = f"Error generating structure class for concept '{concept_code}' in domain '{domain}': {exc}"
- raise ConceptFactoryError(msg) from exc
+ raise ConceptDefinitionError(msg) from exc
# Handle refines definition
elif blueprint.refines:
@@ -173,8 +171,12 @@ class ConceptFactory:
f"Concept '{concept_code}' in domain '{domain}' has refines but also has a structure class registered. "
"A concept cannot have both structure and refines."
)
- raise ConceptFactoryError(msg)
- current_refine = cls.make_refines(blueprint=blueprint)
+ raise ConceptDefinitionError(msg)
+ try:
+ current_refine = cls.make_refine(refine=blueprint.refines)
+ except ConceptRefineError as exc:
+ msg = f"Could not validate refine '{blueprint.refines}' for concept '{concept_code}' in domain '{domain}': {exc}"
+ raise ConceptDefinitionError(msg) from exc
structure_class_name = current_refine.split(".")[1] + "Content" if current_refine else TextContent.__name__
# Handle neither structure nor refines - check the class registry
# If there is a class, use it. structure_class_name is then the concept_code
diff --git a/pipelex/core/domains/domain_factory.py b/pipelex/core/domains/domain_factory.py
index 449349ea2..bc9dfc7f6 100644
--- a/pipelex/core/domains/domain_factory.py
+++ b/pipelex/core/domains/domain_factory.py
@@ -1,14 +1,23 @@
+from pydantic import ValidationError
+
from pipelex.core.domains.domain import Domain
from pipelex.core.domains.domain_blueprint import DomainBlueprint
+from pipelex.exceptions import DomainDefinitionError
+from pipelex.tools.typing.pydantic_utils import format_pydantic_validation_error
class DomainFactory:
@classmethod
def make_from_blueprint(cls, blueprint: DomainBlueprint) -> Domain:
- return Domain(
- code=blueprint.code,
- definition=blueprint.definition,
- system_prompt=blueprint.system_prompt,
- system_prompt_to_structure=blueprint.system_prompt_to_structure,
- prompt_template_to_structure=blueprint.prompt_template_to_structure,
- )
+ try:
+ return Domain(
+ code=blueprint.code,
+ definition=blueprint.definition,
+ system_prompt=blueprint.system_prompt,
+ system_prompt_to_structure=blueprint.system_prompt_to_structure,
+ prompt_template_to_structure=blueprint.prompt_template_to_structure,
+ )
+ except ValidationError as exc:
+ formatted_error_msg = format_pydantic_validation_error(exc)
+ msg = f"Could not make domain from blueprint: {formatted_error_msg}"
+ raise DomainDefinitionError(msg) from exc
diff --git a/pipelex/core/pipes/pipe_blueprint.py b/pipelex/core/pipes/pipe_blueprint.py
index d684c82ed..07e1c5fe8 100644
--- a/pipelex/core/pipes/pipe_blueprint.py
+++ b/pipelex/core/pipes/pipe_blueprint.py
@@ -37,6 +37,7 @@ class AllowedPipeTypes(StrEnum):
class PipeBlueprint(BaseModel):
+ source: str | None = None
category: Any
type: Any # TODO: Find a better way to handle this.
definition: str | None = None
diff --git a/pipelex/core/pipes/pipe_run_params.py b/pipelex/core/pipes/pipe_run_params.py
index 2a71d40d4..037c2b501 100644
--- a/pipelex/core/pipes/pipe_run_params.py
+++ b/pipelex/core/pipes/pipe_run_params.py
@@ -177,19 +177,14 @@ class BatchParams(BaseModel):
@classmethod
def make_optional_batch_params(
cls,
- input_list_name: bool | str,
+ input_list_name: str | None,
input_item_name: str | None = None,
) -> BatchParams | None:
the_batch_params: BatchParams | None = None
- if input_list_name or input_item_name:
- input_list_stuff_name: str
- if isinstance(input_list_name, str):
- input_list_stuff_name = input_list_name
- else:
- input_list_stuff_name = MAIN_STUFF_NAME
+ if input_list_name:
input_item_stuff_name = input_item_name or BATCH_ITEM_STUFF_NAME
the_batch_params = BatchParams(
- input_list_stuff_name=input_list_stuff_name,
+ input_list_stuff_name=input_list_name,
input_item_stuff_name=input_item_stuff_name,
)
return the_batch_params
diff --git a/pipelex/exceptions.py b/pipelex/exceptions.py
index 1b83b4d3e..f16a8b59d 100644
--- a/pipelex/exceptions.py
+++ b/pipelex/exceptions.py
@@ -104,19 +104,15 @@ class ClientAuthenticationError(PipelexError):
pass
-class DomainDefinitionError(PipelexError):
- pass
-
-
class ConceptLibraryConceptNotFoundError(PipelexError):
pass
-class ConceptFactoryError(PipelexError):
+class LibraryError(PipelexError):
pass
-class LibraryError(PipelexError):
+class LibraryLoadingError(LibraryError):
pass
@@ -144,10 +140,48 @@ class LibraryParsingError(LibraryError):
pass
-class PipeDefinitionError(PipelexError):
+# TODO: add details from all cases raising this error
+class DomainDefinitionError(PipelexError):
+ pass
+
+
+# TODO: add details from all cases raising this error
+class ConceptDefinitionError(PipelexError):
pass
+# TODO: add details from all cases raising this error
+class PipeDefinitionError(PipelexError):
+ def __init__(
+ self,
+ message: str,
+ domain: str | None = None,
+ pipe_code: str | None = None,
+ description: str | None = None,
+ source: str | None = None,
+ ):
+ self.domain = domain
+ self.pipe_code = pipe_code
+ self.description = description
+ self.source = source
+ message = message + " • " + self.pipe_details()
+ super().__init__(message)
+
+ def pipe_details(self) -> str:
+ if not self.domain and not self.pipe_code and not self.description and not self.source:
+ return "No pipe details provided"
+ details = "Pipe details:"
+ if self.domain:
+ details += f" • domain='{self.domain}'"
+ if self.pipe_code:
+ details += f" • pipe='{self.pipe_code}'"
+ if self.description:
+ details += f" • description='{self.description}'"
+ if self.source:
+ details += f" • source='{self.source}'"
+ return details
+
+
class UnexpectedPipeDefinitionError(PipeDefinitionError):
pass
@@ -239,6 +273,10 @@ class ConceptCodeError(ConceptError):
pass
+class ConceptRefineError(ConceptError):
+ pass
+
+
class PipelineManagerNotFoundError(PipelexError):
pass
diff --git a/pipelex/libraries/library_config.py b/pipelex/libraries/library_config.py
index bfb5650b4..65e176f9b 100644
--- a/pipelex/libraries/library_config.py
+++ b/pipelex/libraries/library_config.py
@@ -1,3 +1,4 @@
+from pathlib import Path
from typing import ClassVar
from pipelex.tools.config.config_model import ConfigModel
@@ -27,8 +28,8 @@ class LibraryConfig(ConfigModel):
return "tests/test_pipelines"
@property
- def failing_pipelines_file_paths(self) -> list[str]:
- return ["tests/test_pipelines/failing_pipelines.plx"]
+ def failing_pipelines_file_paths(self) -> set[Path]:
+ return {Path("tests/test_pipelines/failing_pipelines.plx")}
def get_templates_paths(self) -> list[str]:
return [str(path) for path in find_files_in_dir(dir_path=self.templates_dir_path, pattern="*.toml", is_recursive=True)]
diff --git a/pipelex/libraries/library_manager.py b/pipelex/libraries/library_manager.py
index 164fb76b5..171439ab1 100644
--- a/pipelex/libraries/library_manager.py
+++ b/pipelex/libraries/library_manager.py
@@ -21,9 +21,12 @@ from pipelex.core.pipes.pipe_abstract import PipeAbstract
from pipelex.core.pipes.pipe_factory import PipeFactory
from pipelex.core.pipes.pipe_library import PipeLibrary
from pipelex.exceptions import (
+ ConceptDefinitionError,
ConceptLibraryError,
+ DomainDefinitionError,
LibraryError,
- PipelexSetupError,
+ LibraryLoadingError,
+ PipeDefinitionError,
PipeLibraryError,
)
from pipelex.libraries.library_config import LibraryConfig
@@ -140,29 +143,45 @@ class LibraryManager(LibraryManagerAbstract):
return all_plx_paths
- @override
- def load_from_file(self, plx_path: Path) -> None:
- """Load a single file - this method is kept for compatibility."""
- if not PipelexInterpreter.is_pipelex_file(plx_path):
- msg = f"File is not a valid Pipelex PLX file: {plx_path}"
- raise LibraryError(msg)
+ # @override
+ # def load_from_file(self, plx_path: Path) -> None:
+ # """Load a single file - this method is kept for compatibility."""
+ # if not PipelexInterpreter.is_pipelex_file(plx_path):
+ # msg = f"File is not a valid Pipelex PLX file: {plx_path}"
+ # raise LibraryError(msg)
- blueprint = PipelexInterpreter(file_path=plx_path).make_pipelex_bundle_blueprint()
- self.load_from_blueprint(blueprint)
+ # blueprint = PipelexInterpreter(file_path=plx_path).make_pipelex_bundle_blueprint()
+ # try:
+ # self.load_from_blueprint(blueprint)
+ # except PipeDefinitionError as exc:
+ # msg = f"Could not load PLX blueprint from '{plx_path}': {exc}"
+ # raise LibraryError(msg) from exc
@override
def load_from_blueprint(self, blueprint: PipelexBundleBlueprint) -> list[PipeAbstract]:
"""Load a blueprint."""
# Create and load domain
- domain = self._load_domain_from_blueprint(blueprint)
+ try:
+ domain = self._load_domain_from_blueprint(blueprint)
+ except DomainDefinitionError as exc:
+ msg = f"Could not load domain from PLX blueprint at '{blueprint.source}', domain code: '{blueprint.domain}': {exc}"
+ raise LibraryLoadingError(msg) from exc
self.domain_library.add_domain(domain=domain)
# Create and load concepts
- concepts = self._load_concepts_from_blueprint(blueprint)
+ try:
+ concepts = self._load_concepts_from_blueprint(blueprint)
+ except ConceptDefinitionError as exc:
+ msg = f"Could not load concepts from PLX blueprint at '{blueprint.source}', domain code: '{blueprint.domain}': {exc}"
+ raise LibraryLoadingError(msg) from exc
self.concept_library.add_concepts(concepts=concepts)
# Create and load pipes
- pipes = self._load_pipes_from_blueprint(blueprint)
+ try:
+ pipes = self._load_pipes_from_blueprint(blueprint)
+ except PipeDefinitionError as exc:
+ msg = f"Could not load pipes from PLX blueprint at '{blueprint.source}', domain code: '{blueprint.domain}': {exc}"
+ raise LibraryLoadingError(msg) from exc
self.pipe_library.add_pipes(pipes=pipes)
return pipes
@@ -194,19 +213,23 @@ class LibraryManager(LibraryManagerAbstract):
)
def _load_concepts_from_blueprint(self, blueprint: PipelexBundleBlueprint) -> list[Concept]:
- concepts: list[Concept] = []
+ if blueprint.concept is None:
+ return []
- if blueprint.concept is not None:
- for concept_code, concept_blueprint_or_str in blueprint.concept.items():
- concept = ConceptFactory.make_from_blueprint(
- domain=blueprint.domain,
- concept_code=concept_code,
- concept_codes_from_the_same_domain=list(blueprint.concept.keys()),
- blueprint=ConceptBlueprint(definition=concept_blueprint_or_str)
- if isinstance(concept_blueprint_or_str, str)
- else concept_blueprint_or_str,
- )
- concepts.append(concept)
+ concepts: list[Concept] = []
+ for concept_code, concept_blueprint_or_str in blueprint.concept.items():
+ concept_blueprint: ConceptBlueprint
+ if isinstance(concept_blueprint_or_str, str):
+ concept_blueprint = ConceptBlueprint(definition=concept_blueprint_or_str)
+ else:
+ concept_blueprint = concept_blueprint_or_str
+ concept = ConceptFactory.make_from_blueprint(
+ domain=blueprint.domain,
+ concept_code=concept_code,
+ concept_codes_from_the_same_domain=list(blueprint.concept.keys()),
+ blueprint=concept_blueprint,
+ )
+ concepts.append(concept)
return concepts
def _load_pipes_from_blueprint(self, blueprint: PipelexBundleBlueprint) -> list[PipeAbstract]:
@@ -228,43 +251,48 @@ class LibraryManager(LibraryManagerAbstract):
library_dirs: list[Path] | None = None,
library_file_paths: list[Path] | None = None,
) -> None:
- dirs_to_use: list[Path] = self._get_pipeline_library_dirs()
- all_plx_paths: list[Path] = self._get_pipelex_plx_files_from_dirs(dirs_to_use)
-
- # Remove failing pipelines from the list
- failing_pipelines_file_paths = get_config().pipelex.library_config.failing_pipelines_file_paths
- failing_paths_set = {Path(fp) for fp in failing_pipelines_file_paths}
- all_plx_paths = [path for path in all_plx_paths if path not in failing_paths_set]
+ dirs_to_use = library_dirs or self._get_pipeline_library_dirs()
- if library_dirs is not None:
- dirs_to_use = library_dirs
+ valid_plx_paths: list[Path]
+ if library_file_paths:
+ valid_plx_paths = library_file_paths
+ else:
+ all_plx_paths: list[Path] = self._get_pipelex_plx_files_from_dirs(dirs_to_use)
+ # Remove failing pipelines from the list
+ failing_pipelines_file_paths = get_config().pipelex.library_config.failing_pipelines_file_paths
+ valid_plx_paths = [path for path in all_plx_paths if path not in failing_pipelines_file_paths]
# Register classes in the directories
for library_dir in dirs_to_use:
ClassRegistryUtils.register_classes_in_folder(folder_path=str(library_dir))
FuncRegistryUtils.register_funcs_in_folder(folder_path=str(library_dir))
- if library_file_paths is not None:
- all_plx_paths = library_file_paths
-
# Parse all blueprints first
blueprints: list[PipelexBundleBlueprint] = []
- for plx_file_path in all_plx_paths:
+ for plx_file_path in valid_plx_paths:
try:
blueprint = PipelexInterpreter(file_path=plx_file_path).make_pipelex_bundle_blueprint()
except FileNotFoundError as exc:
msg = f"Could not find PLX blueprint at '{plx_file_path}'"
- raise PipelexSetupError(msg) from exc
+ raise LibraryLoadingError(msg) from exc
except ValidationError as exc:
formatted_error_msg = format_pydantic_validation_error(exc)
msg = f"Could not load PLX blueprint from '{plx_file_path}' because of: {formatted_error_msg}"
- raise PipelexSetupError(msg) from exc
+ raise LibraryLoadingError(msg) from exc
+ except PipeDefinitionError as exc:
+ msg = f"Could not load PLX blueprint from '{plx_file_path}': {exc}"
+ raise LibraryLoadingError(msg) from exc
+ blueprint.source = str(plx_file_path)
blueprints.append(blueprint)
# Load all domains first
all_domains: list[Domain] = []
for blueprint in blueprints:
- domain = self._load_domain_from_blueprint(blueprint)
+ try:
+ domain = self._load_domain_from_blueprint(blueprint)
+ except DomainDefinitionError as exc:
+ msg = f"Could not load domain from PLX blueprint at '{blueprint.source}', domain code: '{blueprint.domain}': {exc}"
+ raise LibraryLoadingError(msg) from exc
all_domains.append(domain)
for domain in all_domains:
self.domain_library.add_domain(domain=domain)
@@ -272,13 +300,21 @@ class LibraryManager(LibraryManagerAbstract):
# Load all concepts second
all_concepts: list[Concept] = []
for blueprint in blueprints:
- concepts = self._load_concepts_from_blueprint(blueprint)
+ try:
+ concepts = self._load_concepts_from_blueprint(blueprint)
+ except ConceptDefinitionError as exc:
+ msg = f"Could not load concepts from PLX blueprint at '{blueprint.source}', domain code: '{blueprint.domain}': {exc}"
+ raise LibraryLoadingError(msg) from exc
all_concepts.extend(concepts)
self.concept_library.add_concepts(concepts=all_concepts)
# Load all pipes third
all_pipes: list[PipeAbstract] = []
for blueprint in blueprints:
- pipes = self._load_pipes_from_blueprint(blueprint)
+ try:
+ pipes = self._load_pipes_from_blueprint(blueprint)
+ except PipeDefinitionError as exc:
+ msg = f"Could not load pipes from PLX blueprint at '{blueprint.source}', domain code: '{blueprint.domain}': {exc}"
+ raise LibraryLoadingError(msg) from exc
all_pipes.extend(pipes)
self.pipe_library.add_pipes(pipes=all_pipes)
diff --git a/pipelex/libraries/library_manager_abstract.py b/pipelex/libraries/library_manager_abstract.py
index 3f93d87ed..dfc6840b8 100644
--- a/pipelex/libraries/library_manager_abstract.py
+++ b/pipelex/libraries/library_manager_abstract.py
@@ -26,9 +26,9 @@ class LibraryManagerAbstract(ABC):
def load_libraries(self, library_dirs: list[Path] | None = None, library_file_paths: list[Path] | None = None) -> None:
pass
- @abstractmethod
- def load_from_file(self, plx_path: Path) -> None:
- pass
+ # @abstractmethod
+ # def load_from_file(self, plx_path: Path) -> None:
+ # pass
@abstractmethod
def load_from_blueprint(self, blueprint: PipelexBundleBlueprint) -> list[PipeAbstract]:
diff --git a/pipelex/libraries/pipelines/builder/builder.plx b/pipelex/libraries/pipelines/builder/builder.plx
index d610ba2bf..2289a482e 100644
--- a/pipelex/libraries/pipelines/builder/builder.plx
+++ b/pipelex/libraries/pipelines/builder/builder.plx
@@ -102,6 +102,7 @@ parallels = [
{ pipe = "draft_to_conceptspecs_text", result = "concept_spec_drafts_text" },
{ pipe = "draft_to_pipesignatures_text", result = "pipe_signatures_text" },
]
+add_each_output = true
[pipe.draft_to_conceptspecs_text]
type = "PipeLLM"
diff --git a/pipelex/libraries/pipelines/builder/pipe/pipe_parallel_spec.py b/pipelex/libraries/pipelines/builder/pipe/pipe_parallel_spec.py
index 2afdf8b6e..f3e481410 100644
--- a/pipelex/libraries/pipelines/builder/pipe/pipe_parallel_spec.py
+++ b/pipelex/libraries/pipelines/builder/pipe/pipe_parallel_spec.py
@@ -1,12 +1,14 @@
from typing import Literal
-from pydantic import Field, field_validator
+from pydantic import Field, field_validator, model_validator
from typing_extensions import override
+from pipelex.exceptions import PipeDefinitionError
from pipelex.libraries.pipelines.builder.concept.concept_spec import ConceptBlueprint
from pipelex.libraries.pipelines.builder.pipe.pipe_signature import PipeSpec
from pipelex.libraries.pipelines.builder.pipe.sub_pipe_spec import SubPipeSpec
from pipelex.pipe_controllers.parallel.pipe_parallel_blueprint import PipeParallelBlueprint
+from pipelex.types import Self
class PipeParallelSpec(PipeSpec):
@@ -38,7 +40,7 @@ class PipeParallelSpec(PipeSpec):
category: Literal["PipeController"] = "PipeController"
the_pipe_code: str = Field(description="Pipe code. Must be snake_case.")
parallels: list[SubPipeSpec]
- add_each_output: bool = True
+ add_each_output: bool = False
combined_output: str | None = None
@field_validator("combined_output", mode="before")
@@ -48,6 +50,16 @@ class PipeParallelSpec(PipeSpec):
ConceptBlueprint.validate_concept_string_or_code(concept_string_or_code=combined_output)
return combined_output
+ @model_validator(mode="after")
+ def validate_output_options(self) -> Self:
+ if not self.add_each_output and not self.combined_output:
+ msg = (
+ "PipeParallel requires either add_each_output to be True or combined_output to be set, "
+ "or both, otherwise the pipe won't output anything"
+ )
+ raise PipeDefinitionError(msg)
+ return self
+
@override
def to_blueprint(self) -> PipeParallelBlueprint:
base_blueprint = super().to_blueprint()
diff --git a/pipelex/libraries/pipelines/builder/pipe/sub_pipe_spec.py b/pipelex/libraries/pipelines/builder/pipe/sub_pipe_spec.py
index 42423bd3b..ce0cd7ab1 100644
--- a/pipelex/libraries/pipelines/builder/pipe/sub_pipe_spec.py
+++ b/pipelex/libraries/pipelines/builder/pipe/sub_pipe_spec.py
@@ -42,7 +42,7 @@ class SubPipeSpec(StructuredContent):
result: str | None = None
nb_output: int | None = None
multiple_output: bool | None = None
- batch_over: bool | str = False
+ batch_over: str | None = None
batch_as: str | None = None
@model_validator(mode="after")
@@ -54,14 +54,11 @@ class SubPipeSpec(StructuredContent):
@model_validator(mode="after")
def validate_batch_params(self) -> Self:
- batch_over_is_specified = self.batch_over is not False and self.batch_over != ""
- batch_as_is_specified = self.batch_as is not None and self.batch_as != ""
-
- if batch_over_is_specified and not batch_as_is_specified:
+ if self.batch_over and not self.batch_as:
msg = f"In pipe '{self.the_pipe_code}': When 'batch_over' is specified, 'batch_as' must also be provided"
raise PipeDefinitionError(msg)
- if batch_as_is_specified and not batch_over_is_specified:
+ if self.batch_as and not self.batch_over:
msg = f"In pipe '{self.the_pipe_code}': When 'batch_as' is specified, 'batch_over' must also be provided"
raise PipeDefinitionError(msg)
diff --git a/pipelex/pipe_controllers/batch/pipe_batch_factory.py b/pipelex/pipe_controllers/batch/pipe_batch_factory.py
index 3eeb04bb2..469bccaea 100644
--- a/pipelex/pipe_controllers/batch/pipe_batch_factory.py
+++ b/pipelex/pipe_controllers/batch/pipe_batch_factory.py
@@ -41,7 +41,7 @@ class PipeBatchFactory(PipeFactoryProtocol[PipeBatchBlueprint, PipeBatch]):
),
branch_pipe_code=blueprint.branch_pipe_code,
batch_params=BatchParams.make_optional_batch_params(
- input_list_name=blueprint.input_list_name or False,
+ input_list_name=blueprint.input_list_name,
input_item_name=blueprint.input_item_name,
),
)
diff --git a/pipelex/pipe_controllers/parallel/pipe_parallel_blueprint.py b/pipelex/pipe_controllers/parallel/pipe_parallel_blueprint.py
index f9f5599a0..9351f0814 100644
--- a/pipelex/pipe_controllers/parallel/pipe_parallel_blueprint.py
+++ b/pipelex/pipe_controllers/parallel/pipe_parallel_blueprint.py
@@ -1,17 +1,19 @@
from typing import Literal
-from pydantic import field_validator
+from pydantic import field_validator, model_validator
from pipelex.core.concepts.concept_blueprint import ConceptBlueprint
from pipelex.core.pipes.pipe_blueprint import PipeBlueprint
+from pipelex.exceptions import PipeDefinitionError
from pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprint
+from pipelex.types import Self
class PipeParallelBlueprint(PipeBlueprint):
type: Literal["PipeParallel"] = "PipeParallel"
category: Literal["PipeController"] = "PipeController"
parallels: list[SubPipeBlueprint]
- add_each_output: bool = True
+ add_each_output: bool = False
combined_output: str | None = None
@field_validator("combined_output", mode="before")
@@ -20,3 +22,13 @@ class PipeParallelBlueprint(PipeBlueprint):
if combined_output:
ConceptBlueprint.validate_concept_string_or_code(concept_string_or_code=combined_output)
return combined_output
+
+ @model_validator(mode="after")
+ def validate_output_options(self) -> Self:
+ if not self.add_each_output and not self.combined_output:
+ msg = (
+ "PipeParallel requires either add_each_output to be True or combined_output to be set, "
+ "or both, otherwise the pipe won't output anything"
+ )
+ raise PipeDefinitionError(message=msg, description=self.definition, source=self.source)
+ return self
diff --git a/pipelex/pipe_controllers/parallel/pipe_parallel_factory.py b/pipelex/pipe_controllers/parallel/pipe_parallel_factory.py
index af322d650..b43fa0423 100644
--- a/pipelex/pipe_controllers/parallel/pipe_parallel_factory.py
+++ b/pipelex/pipe_controllers/parallel/pipe_parallel_factory.py
@@ -6,7 +6,7 @@ from pipelex.core.concepts.concept_factory import ConceptFactory
from pipelex.core.pipes.pipe_factory import PipeFactoryProtocol
from pipelex.core.pipes.pipe_input_factory import PipeInputSpecFactory
from pipelex.exceptions import PipeDefinitionError
-from pipelex.hub import get_concept_provider
+from pipelex.hub import get_required_concept
from pipelex.pipe_controllers.parallel.pipe_parallel import PipeParallel
from pipelex.pipe_controllers.parallel.pipe_parallel_blueprint import PipeParallelBlueprint
from pipelex.pipe_controllers.sub_pipe_factory import SubPipeFactory
@@ -33,7 +33,10 @@ class PipeParallelFactory(PipeFactoryProtocol[PipeParallelBlueprint, PipeParalle
sub_pipe = SubPipeFactory.make_from_blueprint(sub_pipe_blueprint, concept_codes_from_the_same_domain=concept_codes_from_the_same_domain)
parallel_sub_pipes.append(sub_pipe)
if not blueprint.add_each_output and not blueprint.combined_output:
- msg = "PipeParallel requires either add_each_output or combined_output to be set"
+ msg = (
+ "Unexpected error:PipeParallel requires either add_each_output to be True or combined_output to be set, "
+ "or both, otherwise the pipe won't output anything"
+ )
raise PipeDefinitionError(msg)
if blueprint.combined_output:
@@ -42,7 +45,7 @@ class PipeParallelFactory(PipeFactoryProtocol[PipeParallelBlueprint, PipeParalle
concept_string_or_code=blueprint.output,
concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,
)
- combined_output = get_concept_provider().get_required_concept(
+ combined_output = get_required_concept(
concept_string=ConceptFactory.construct_concept_string_with_domain(
domain=combined_output_domain_and_code.domain,
concept_code=combined_output_domain_and_code.concept_code,
@@ -65,13 +68,13 @@ class PipeParallelFactory(PipeFactoryProtocol[PipeParallelBlueprint, PipeParalle
blueprint=blueprint.inputs or {},
concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,
),
- output=get_concept_provider().get_required_concept(
+ output=get_required_concept(
concept_string=ConceptFactory.construct_concept_string_with_domain(
domain=output_domain_and_code.domain,
concept_code=output_domain_and_code.concept_code,
),
),
parallel_sub_pipes=parallel_sub_pipes,
- add_each_output=blueprint.add_each_output,
+ add_each_output=blueprint.add_each_output or False,
combined_output=combined_output,
)
diff --git a/pipelex/pipe_controllers/sub_pipe_blueprint.py b/pipelex/pipe_controllers/sub_pipe_blueprint.py
index 73ee2d1ac..eb69fbff6 100644
--- a/pipelex/pipe_controllers/sub_pipe_blueprint.py
+++ b/pipelex/pipe_controllers/sub_pipe_blueprint.py
@@ -12,7 +12,7 @@ class SubPipeBlueprint(BaseModel):
result: str | None = None
nb_output: int | None = None
multiple_output: bool | None = None
- batch_over: bool | str = False
+ batch_over: str | None = None
batch_as: str | None = None
@model_validator(mode="after")
@@ -24,14 +24,11 @@ class SubPipeBlueprint(BaseModel):
@model_validator(mode="after")
def validate_batch_params(self) -> Self:
- batch_over_is_specified = self.batch_over is not False and self.batch_over != ""
- batch_as_is_specified = self.batch_as is not None and self.batch_as != ""
-
- if batch_over_is_specified and not batch_as_is_specified:
+ if self.batch_over and not self.batch_as:
msg = f"In pipe '{self.pipe}': When 'batch_over' is specified, 'batch_as' must also be provided"
raise PipeDefinitionError(msg)
- if batch_as_is_specified and not batch_over_is_specified:
+ if self.batch_as and not self.batch_over:
msg = f"In pipe '{self.pipe}': When 'batch_as' is specified, 'batch_over' must also be provided"
raise PipeDefinitionError(msg)
diff --git a/pipelex/tools/plx/plx_utils.py b/pipelex/tools/plx/plx_utils.py
index 0b06accd8..13c82ec5b 100644
--- a/pipelex/tools/plx/plx_utils.py
+++ b/pipelex/tools/plx/plx_utils.py
@@ -1,7 +1,5 @@
from __future__ import annotations
-import re
-from re import Match
from typing import TYPE_CHECKING, Any, Mapping, cast
import tomlkit
diff --git a/tests/unit/pipelex/core/concepts/concept_factory.py/test_concept_factory.py b/tests/unit/pipelex/core/concepts/concept_factory.py/test_concept_factory.py
index 8ea032084..500709d87 100644
--- a/tests/unit/pipelex/core/concepts/concept_factory.py/test_concept_factory.py
+++ b/tests/unit/pipelex/core/concepts/concept_factory.py/test_concept_factory.py
@@ -19,14 +19,16 @@ class TestConceptFactory:
"test_name,blueprint,expected_result",
TestCases.MAKE_REFINES_TEST_CASES,
)
- def test_make_refines(
+ def test_make_refine(
self,
test_name: str,
blueprint: ConceptBlueprint,
expected_result: str,
):
"""Test make_refines method with different blueprint configurations."""
- result = ConceptFactory.make_refines(blueprint=blueprint)
+ if not blueprint.refines:
+ pytest.skip("Test case has no refines")
+ result = ConceptFactory.make_refine(refine=blueprint.refines)
assert result == expected_result, f"Failed for test case: {test_name}"
def test_normalize_structure_blueprint(self):
diff --git a/tests/unit/pipelex/core/concepts/concept_factory/test_concept_factory.py b/tests/unit/pipelex/core/concepts/concept_factory/test_concept_factory.py
index 195a43805..1d3a7a066 100644
--- a/tests/unit/pipelex/core/concepts/concept_factory/test_concept_factory.py
+++ b/tests/unit/pipelex/core/concepts/concept_factory/test_concept_factory.py
@@ -21,14 +21,16 @@ class TestConceptFactory:
"test_name,blueprint,expected_result",
TestCases.MAKE_REFINES_TEST_CASES,
)
- def test_make_refines(
+ def test_make_refine(
self,
test_name: str,
blueprint: ConceptBlueprint,
expected_result: str,
):
"""Test make_refines method with different blueprint configurations."""
- result = ConceptFactory.make_refines(blueprint=blueprint)
+ if not blueprint.refines:
+ pytest.skip("Test case has no refines")
+ result = ConceptFactory.make_refine(refine=blueprint.refines)
assert result == expected_result, f"Failed for test case: {test_name}"
def test_normalize_structure_blueprint(self):
diff --git a/tests/unit/pipelex/core/test_data/__init__.py b/tests/unit/pipelex/core/test_data/__init__.py
index b62667c26..995f06715 100644
--- a/tests/unit/pipelex/core/test_data/__init__.py
+++ b/tests/unit/pipelex/core/test_data/__init__.py
@@ -26,20 +26,20 @@ class InterpreterTestCases:
VALID_TEST_CASES: ClassVar[List[Tuple[str, str, PipelexBundleBlueprint]]] = [ # test_name,plx_content,blueprint
# Domain tests
*DOMAIN_TEST_CASES,
- # # Concept tests
+ # Concept tests
*STRUCTURED_CONCEPT_TEST_CASES,
- # *REFINING_CONCEPT_TEST_CASES,
- # *SIMPLE_CONCEPT_TEST_CASES,
- # # Pipe operator tests
- # *PIPE_LLM_TEST_CASES,
- # *PIPE_OCR_TEST_CASES,
- # *PIPE_FUNC_TEST_CASES,
- # *PIPE_IMG_GEN_TEST_CASES,
- # *PIPE_JINJA2_TEST_CASES,
- # # # Pipe controller tests
- # *PIPE_SEQUENCE_TEST_CASES,
- # *PIPE_CONDITION_TEST_CASES,
- # *PIPE_PARALLEL_TEST_CASES,
+ *REFINING_CONCEPT_TEST_CASES,
+ *SIMPLE_CONCEPT_TEST_CASES,
+ # Pipe operator tests
+ *PIPE_LLM_TEST_CASES,
+ *PIPE_OCR_TEST_CASES,
+ *PIPE_FUNC_TEST_CASES,
+ *PIPE_IMG_GEN_TEST_CASES,
+ *PIPE_JINJA2_TEST_CASES,
+ # Pipe controller tests
+ *PIPE_SEQUENCE_TEST_CASES,
+ *PIPE_CONDITION_TEST_CASES,
+ *PIPE_PARALLEL_TEST_CASES,
# *PIPE_BATCH_TEST_CASES,
# # Complex tests
# *COMPLEX_TEST_CASES,
diff --git a/tests/unit/pipelex/core/test_data/pipes/controllers/condition/pipe_condition.py b/tests/unit/pipelex/core/test_data/pipes/controllers/condition/pipe_condition.py
index d83deab06..d25ae5095 100644
--- a/tests/unit/pipelex/core/test_data/pipes/controllers/condition/pipe_condition.py
+++ b/tests/unit/pipelex/core/test_data/pipes/controllers/condition/pipe_condition.py
@@ -10,8 +10,8 @@ definition = "Domain with conditional pipe"
type = "PipeCondition"
definition = "Process based on condition"
output = "ProcessedData"
-pipe_map = { small = "process_small", large = "process_large" }
expression = "input_data.category"
+pipe_map = { small = "process_small", large = "process_large" }
""",
PipelexBundleBlueprint(
domain="test_pipes",
diff --git a/tests/unit/pipelex/core/test_data/pipes/controllers/parallel/pipe_parallel.py b/tests/unit/pipelex/core/test_data/pipes/controllers/parallel/pipe_parallel.py
index 1056a0abc..f184e46de 100644
--- a/tests/unit/pipelex/core/test_data/pipes/controllers/parallel/pipe_parallel.py
+++ b/tests/unit/pipelex/core/test_data/pipes/controllers/parallel/pipe_parallel.py
@@ -9,12 +9,13 @@ definition = "Domain with parallel pipe"
[pipe.parallel_process]
type = "PipeParallel"
-definition = "Process data in parallel"
+definition = "PipeParallel example in PIPE_PARALLEL_TEST_CASES"
output = "ProcessedData"
parallels = [
{ pipe = "process_a", result = "result_a" },
{ pipe = "process_b", result = "result_b" },
]
+add_each_output = true
""",
PipelexBundleBlueprint(
domain="test_pipes",
@@ -22,12 +23,13 @@ parallels = [
pipe={
"parallel_process": PipeParallelBlueprint(
type="PipeParallel",
- definition="Process data in parallel",
+ definition="PipeParallel example in PIPE_PARALLEL_TEST_CASES",
output="ProcessedData",
parallels=[
SubPipeBlueprint(pipe="process_a", result="result_a"),
SubPipeBlueprint(pipe="process_b", result="result_b"),
],
+ add_each_output=True,
),
},
),
diff --git a/tests/unit/pipelex/core/test_data/pipes/operators/llm/pipe_llm.py b/tests/unit/pipelex/core/test_data/pipes/operators/llm/pipe_llm.py
index 3a1fab21f..e59279f18 100644
--- a/tests/unit/pipelex/core/test_data/pipes/operators/llm/pipe_llm.py
+++ b/tests/unit/pipelex/core/test_data/pipes/operators/llm/pipe_llm.py
@@ -55,7 +55,7 @@ Extract information about $topic from this text:
definition="Extract information from text",
inputs={"text": "Text", "topic": "Text"},
output=NativeConceptEnum.TEXT,
- prompt_template="Extract information about $topic from this text:\n\n@text",
+ prompt_template="Extract information about $topic from this text:\n\n@text\n",
),
},
),
@@ -335,7 +335,7 @@ Documents: @documents
# Export all PipeLLM test cases
PIPE_LLM_TEST_CASES = [
- # PIPE_LLM_BASIC,
+ PIPE_LLM_BASIC,
PIPE_LLM_WITH_INPUTS,
PIPE_LLM_WITH_SYSTEM_PROMPT,
PIPE_LLM_MULTIPLE_OUTPUTS,
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_data.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_data.py
index 25023d6eb..63fbca416 100644
--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_data.py
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_data.py
@@ -9,8 +9,8 @@ from pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprint
class PipeParallelTestCases:
- SIMPLE_PARALLEL = (
- "simple_parallel",
+ PARALLEL_WITH_EACH_OUTPUT = (
+ "parallel_with_each_output",
PipeParallelSpec(
the_pipe_code="parallel_processor",
definition="Run pipes in parallel",
@@ -21,6 +21,7 @@ class PipeParallelTestCases:
SubPipeSpec(the_pipe_code="transform_data", result="transformed"),
SubPipeSpec(the_pipe_code="validate_data", result="validation"),
],
+ add_each_output=True,
),
PipeParallelBlueprint(
definition="Run pipes in parallel",
@@ -35,6 +36,7 @@ class PipeParallelTestCases:
],
add_each_output=True,
combined_output=None,
+ source="tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_data.py PipeParallelTestCases.PARALLEL_WITH_EACH_OUTPUT",
),
)
@@ -64,10 +66,42 @@ class PipeParallelTestCases:
],
add_each_output=False,
combined_output="MergedData",
+ source="tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_data.py PipeParallelTestCases.PARALLEL_WITH_COMBINED",
+ ),
+ )
+
+ PARALLEL_WITH_BOTH_EACH_OUTPUT_AND_COMBINED = (
+ "parallel_with_both_each_output_and_combined",
+ PipeParallelSpec(
+ the_pipe_code="combined_parallel",
+ definition="Parallel with combined output",
+ inputs={"input": InputRequirementSpec(concept="Input")},
+ output="CombinedResult",
+ parallels=[
+ SubPipeSpec(the_pipe_code="pipe1", result="result1"),
+ SubPipeSpec(the_pipe_code="pipe2", result="result2"),
+ ],
+ add_each_output=True,
+ combined_output="MergedData",
+ ),
+ PipeParallelBlueprint(
+ definition="Parallel with combined output",
+ inputs={"input": InputRequirementBlueprint(concept="Input")},
+ output="CombinedResult",
+ type="PipeParallel",
+ category="PipeController",
+ parallels=[
+ SubPipeBlueprint(pipe="pipe1", result="result1"),
+ SubPipeBlueprint(pipe="pipe2", result="result2"),
+ ],
+ add_each_output=True,
+ combined_output="MergedData",
+ source="tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_data.py PipeParallelTestCases.PARALLEL_WITH_BOTH_EACH_OUTPUT_AND_COMBINED",
),
)
TEST_CASES: ClassVar[list[tuple[str, PipeParallelSpec, PipeParallelBlueprint]]] = [
- SIMPLE_PARALLEL,
+ PARALLEL_WITH_EACH_OUTPUT,
PARALLEL_WITH_COMBINED,
+ PARALLEL_WITH_BOTH_EACH_OUTPUT_AND_COMBINED,
]
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_pipe_parallel.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_pipe_parallel.py
index 04d999a7f..e35dd1669 100644
--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_pipe_parallel.py
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_pipe_parallel.py
@@ -4,7 +4,7 @@ from pipelex.libraries.pipelines.builder.pipe.pipe_parallel_spec import PipePara
from pipelex.pipe_controllers.parallel.pipe_parallel_blueprint import PipeParallelBlueprint
from .test_data import PipeParallelTestCases
-
+from pipelex import log
class TestPipeParallelBlueprintConversion:
@pytest.mark.parametrize(
@@ -17,5 +17,7 @@ class TestPipeParallelBlueprintConversion:
pipe_spec: PipeParallelSpec,
expected_blueprint: PipeParallelBlueprint,
):
+ log.verbose(f"Testing {test_name}")
result = pipe_spec.to_blueprint()
+ expected_blueprint.source = None
assert result == expected_blueprint