Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
Make PipeParallel enforce "add each output" or "combined output" or both Stronger error handling in library loading.
Louis Choquelf6a55398baa0
diff --git a/pipelex/cli/commands/build_cmd.py b/pipelex/cli/commands/build_cmd.pyindex c4e0dda4c..66e70c02a 100644--- a/pipelex/cli/commands/build_cmd.py+++ b/pipelex/cli/commands/build_cmd.py@@ -32,7 +32,7 @@ def build_pipe_cmd(output_path: Annotated[str | None,typer.Option("--output", "-o", help="Path to save the generated PLX file (use --output='' to skip saving)"),- ] = "./generated_pipeline.plx",+ ] = "./results/generated_pipeline.plx",) -> None:Pipelex.make(relative_config_folder_path="../../../pipelex/libraries", from_file=True)typer.echo("=" * 70)diff --git a/pipelex/core/bundles/pipelex_bundle_blueprint.py b/pipelex/core/bundles/pipelex_bundle_blueprint.pyindex 66ce37b84..91cb86c39 100644--- a/pipelex/core/bundles/pipelex_bundle_blueprint.py+++ b/pipelex/core/bundles/pipelex_bundle_blueprint.py@@ -31,6 +31,7 @@ PipeBlueprintUnion = Annotated[class PipelexBundleBlueprint(BaseModel):model_config = ConfigDict(extra="forbid")+ source: str | None = Nonedomain: strdefinition: str | None = Nonesystem_prompt: str | None = Nonediff --git a/pipelex/core/concepts/concept_factory.py b/pipelex/core/concepts/concept_factory.pyindex f501974be..b80758831 100644--- a/pipelex/core/concepts/concept_factory.py+++ b/pipelex/core/concepts/concept_factory.py@@ -13,7 +13,7 @@ from pipelex.core.concepts.concept_native import NativeConceptEnumData, NativeCofrom pipelex.core.concepts.structure_generator import StructureGeneratorfrom pipelex.core.domains.domain import SpecialDomainfrom pipelex.core.stuffs.stuff_content import TextContent-from pipelex.exceptions import ConceptFactoryError, StructureClassError+from pipelex.exceptions import ConceptCodeError, ConceptDefinitionError, ConceptRefineError, StructureClassErrorclass DomainAndConceptCode(BaseModel):@@ -106,16 +106,10 @@ class ConceptFactory:@classmethoddef make_refine(cls, refine: str) -> str:- if NativeConceptManager.is_native_concept(concept_string_or_code=refine):- return NativeConceptManager.get_native_concept_string(concept_string_or_code=refine)- msg = f"Refine '{refine}' is not a native concept"- raise ConceptFactoryError(msg)-- @classmethod- def make_refines(cls, blueprint: ConceptBlueprint) -> str | None:- if blueprint.refines:- return cls.make_refine(refine=blueprint.refines)- return None+ if not NativeConceptManager.is_native_concept(concept_string_or_code=refine):+ msg = f"Refine '{refine}' is not a native concept"+ raise ConceptRefineError(msg)+ return NativeConceptManager.get_native_concept_string(concept_string_or_code=refine)@classmethoddef make_from_blueprint(@@ -125,7 +119,11 @@ class ConceptFactory:blueprint: ConceptBlueprint,concept_codes_from_the_same_domain: list[str] | None = None,) -> Concept:- ConceptBlueprint.validate_concept_code(concept_code=concept_code)+ try:+ ConceptBlueprint.validate_concept_code(concept_code=concept_code)+ except ConceptCodeError as exc:+ msg = f"Could not validate concept code '{concept_code}' in domain '{domain}': {exc}"+ raise ConceptDefinitionError(msg) from excstructure_class_name: strcurrent_refine: str | None = None@@ -163,7 +161,7 @@ class ConceptFactory:except Exception as exc:msg = f"Error generating structure class for concept '{concept_code}' in domain '{domain}': {exc}"- raise ConceptFactoryError(msg) from exc+ raise ConceptDefinitionError(msg) from exc# Handle refines definitionelif blueprint.refines:@@ -173,8 +171,12 @@ class ConceptFactory:f"Concept '{concept_code}' in domain '{domain}' has refines but also has a structure class registered. ""A concept cannot have both structure and refines.")- raise ConceptFactoryError(msg)- current_refine = cls.make_refines(blueprint=blueprint)+ raise ConceptDefinitionError(msg)+ try:+ current_refine = cls.make_refine(refine=blueprint.refines)+ except ConceptRefineError as exc:+ msg = f"Could not validate refine '{blueprint.refines}' for concept '{concept_code}' in domain '{domain}': {exc}"+ raise ConceptDefinitionError(msg) from excstructure_class_name = current_refine.split(".")[1] + "Content" if current_refine else TextContent.__name__# Handle neither structure nor refines - check the class registry# If there is a class, use it. structure_class_name is then the concept_codediff --git a/pipelex/core/domains/domain_factory.py b/pipelex/core/domains/domain_factory.pyindex 449349ea2..bc9dfc7f6 100644--- a/pipelex/core/domains/domain_factory.py+++ b/pipelex/core/domains/domain_factory.py@@ -1,14 +1,23 @@+from pydantic import ValidationError+from pipelex.core.domains.domain import Domainfrom pipelex.core.domains.domain_blueprint import DomainBlueprint+from pipelex.exceptions import DomainDefinitionError+from pipelex.tools.typing.pydantic_utils import format_pydantic_validation_errorclass DomainFactory:@classmethoddef make_from_blueprint(cls, blueprint: DomainBlueprint) -> Domain:- return Domain(- code=blueprint.code,- definition=blueprint.definition,- system_prompt=blueprint.system_prompt,- system_prompt_to_structure=blueprint.system_prompt_to_structure,- prompt_template_to_structure=blueprint.prompt_template_to_structure,- )+ try:+ return Domain(+ code=blueprint.code,+ definition=blueprint.definition,+ system_prompt=blueprint.system_prompt,+ system_prompt_to_structure=blueprint.system_prompt_to_structure,+ prompt_template_to_structure=blueprint.prompt_template_to_structure,+ )+ except ValidationError as exc:+ formatted_error_msg = format_pydantic_validation_error(exc)+ msg = f"Could not make domain from blueprint: {formatted_error_msg}"+ raise DomainDefinitionError(msg) from excdiff --git a/pipelex/core/pipes/pipe_blueprint.py b/pipelex/core/pipes/pipe_blueprint.pyindex d684c82ed..07e1c5fe8 100644--- a/pipelex/core/pipes/pipe_blueprint.py+++ b/pipelex/core/pipes/pipe_blueprint.py@@ -37,6 +37,7 @@ class AllowedPipeTypes(StrEnum):class PipeBlueprint(BaseModel):+ source: str | None = Nonecategory: Anytype: Any # TODO: Find a better way to handle this.definition: str | None = Nonediff --git a/pipelex/core/pipes/pipe_run_params.py b/pipelex/core/pipes/pipe_run_params.pyindex 2a71d40d4..037c2b501 100644--- a/pipelex/core/pipes/pipe_run_params.py+++ b/pipelex/core/pipes/pipe_run_params.py@@ -177,19 +177,14 @@ class BatchParams(BaseModel):@classmethoddef make_optional_batch_params(cls,- input_list_name: bool | str,+ input_list_name: str | None,input_item_name: str | None = None,) -> BatchParams | None:the_batch_params: BatchParams | None = None- if input_list_name or input_item_name:- input_list_stuff_name: str- if isinstance(input_list_name, str):- input_list_stuff_name = input_list_name- else:- input_list_stuff_name = MAIN_STUFF_NAME+ if input_list_name:input_item_stuff_name = input_item_name or BATCH_ITEM_STUFF_NAMEthe_batch_params = BatchParams(- input_list_stuff_name=input_list_stuff_name,+ input_list_stuff_name=input_list_name,input_item_stuff_name=input_item_stuff_name,)return the_batch_paramsdiff --git a/pipelex/exceptions.py b/pipelex/exceptions.pyindex 1b83b4d3e..f16a8b59d 100644--- a/pipelex/exceptions.py+++ b/pipelex/exceptions.py@@ -104,19 +104,15 @@ class ClientAuthenticationError(PipelexError):pass-class DomainDefinitionError(PipelexError):- pass--class ConceptLibraryConceptNotFoundError(PipelexError):pass-class ConceptFactoryError(PipelexError):+class LibraryError(PipelexError):pass-class LibraryError(PipelexError):+class LibraryLoadingError(LibraryError):pass@@ -144,10 +140,48 @@ class LibraryParsingError(LibraryError):pass-class PipeDefinitionError(PipelexError):+# TODO: add details from all cases raising this error+class DomainDefinitionError(PipelexError):+ pass+++# TODO: add details from all cases raising this error+class ConceptDefinitionError(PipelexError):pass+# TODO: add details from all cases raising this error+class PipeDefinitionError(PipelexError):+ def __init__(+ self,+ message: str,+ domain: str | None = None,+ pipe_code: str | None = None,+ description: str | None = None,+ source: str | None = None,+ ):+ self.domain = domain+ self.pipe_code = pipe_code+ self.description = description+ self.source = source+ message = message + " • " + self.pipe_details()+ super().__init__(message)++ def pipe_details(self) -> str:+ if not self.domain and not self.pipe_code and not self.description and not self.source:+ return "No pipe details provided"+ details = "Pipe details:"+ if self.domain:+ details += f" • domain='{self.domain}'"+ if self.pipe_code:+ details += f" • pipe='{self.pipe_code}'"+ if self.description:+ details += f" • description='{self.description}'"+ if self.source:+ details += f" • source='{self.source}'"+ return details++class UnexpectedPipeDefinitionError(PipeDefinitionError):pass@@ -239,6 +273,10 @@ class ConceptCodeError(ConceptError):pass+class ConceptRefineError(ConceptError):+ pass++class PipelineManagerNotFoundError(PipelexError):passdiff --git a/pipelex/libraries/library_config.py b/pipelex/libraries/library_config.pyindex bfb5650b4..65e176f9b 100644--- a/pipelex/libraries/library_config.py+++ b/pipelex/libraries/library_config.py@@ -1,3 +1,4 @@+from pathlib import Pathfrom typing import ClassVarfrom pipelex.tools.config.config_model import ConfigModel@@ -27,8 +28,8 @@ class LibraryConfig(ConfigModel):return "tests/test_pipelines"@property- def failing_pipelines_file_paths(self) -> list[str]:- return ["tests/test_pipelines/failing_pipelines.plx"]+ def failing_pipelines_file_paths(self) -> set[Path]:+ return {Path("tests/test_pipelines/failing_pipelines.plx")}def get_templates_paths(self) -> list[str]:return [str(path) for path in find_files_in_dir(dir_path=self.templates_dir_path, pattern="*.toml", is_recursive=True)]diff --git a/pipelex/libraries/library_manager.py b/pipelex/libraries/library_manager.pyindex 164fb76b5..171439ab1 100644--- a/pipelex/libraries/library_manager.py+++ b/pipelex/libraries/library_manager.py@@ -21,9 +21,12 @@ from pipelex.core.pipes.pipe_abstract import PipeAbstractfrom pipelex.core.pipes.pipe_factory import PipeFactoryfrom pipelex.core.pipes.pipe_library import PipeLibraryfrom pipelex.exceptions import (+ ConceptDefinitionError,ConceptLibraryError,+ DomainDefinitionError,LibraryError,- PipelexSetupError,+ LibraryLoadingError,+ PipeDefinitionError,PipeLibraryError,)from pipelex.libraries.library_config import LibraryConfig@@ -140,29 +143,45 @@ class LibraryManager(LibraryManagerAbstract):return all_plx_paths- @override- def load_from_file(self, plx_path: Path) -> None:- """Load a single file - this method is kept for compatibility."""- if not PipelexInterpreter.is_pipelex_file(plx_path):- msg = f"File is not a valid Pipelex PLX file: {plx_path}"- raise LibraryError(msg)+ # @override+ # def load_from_file(self, plx_path: Path) -> None:+ # """Load a single file - this method is kept for compatibility."""+ # if not PipelexInterpreter.is_pipelex_file(plx_path):+ # msg = f"File is not a valid Pipelex PLX file: {plx_path}"+ # raise LibraryError(msg)- blueprint = PipelexInterpreter(file_path=plx_path).make_pipelex_bundle_blueprint()- self.load_from_blueprint(blueprint)+ # blueprint = PipelexInterpreter(file_path=plx_path).make_pipelex_bundle_blueprint()+ # try:+ # self.load_from_blueprint(blueprint)+ # except PipeDefinitionError as exc:+ # msg = f"Could not load PLX blueprint from '{plx_path}': {exc}"+ # raise LibraryError(msg) from exc@overridedef load_from_blueprint(self, blueprint: PipelexBundleBlueprint) -> list[PipeAbstract]:"""Load a blueprint."""# Create and load domain- domain = self._load_domain_from_blueprint(blueprint)+ try:+ domain = self._load_domain_from_blueprint(blueprint)+ except DomainDefinitionError as exc:+ msg = f"Could not load domain from PLX blueprint at '{blueprint.source}', domain code: '{blueprint.domain}': {exc}"+ raise LibraryLoadingError(msg) from excself.domain_library.add_domain(domain=domain)# Create and load concepts- concepts = self._load_concepts_from_blueprint(blueprint)+ try:+ concepts = self._load_concepts_from_blueprint(blueprint)+ except ConceptDefinitionError as exc:+ msg = f"Could not load concepts from PLX blueprint at '{blueprint.source}', domain code: '{blueprint.domain}': {exc}"+ raise LibraryLoadingError(msg) from excself.concept_library.add_concepts(concepts=concepts)# Create and load pipes- pipes = self._load_pipes_from_blueprint(blueprint)+ try:+ pipes = self._load_pipes_from_blueprint(blueprint)+ except PipeDefinitionError as exc:+ msg = f"Could not load pipes from PLX blueprint at '{blueprint.source}', domain code: '{blueprint.domain}': {exc}"+ raise LibraryLoadingError(msg) from excself.pipe_library.add_pipes(pipes=pipes)return pipes@@ -194,19 +213,23 @@ class LibraryManager(LibraryManagerAbstract):)def _load_concepts_from_blueprint(self, blueprint: PipelexBundleBlueprint) -> list[Concept]:- concepts: list[Concept] = []+ if blueprint.concept is None:+ return []- if blueprint.concept is not None:- for concept_code, concept_blueprint_or_str in blueprint.concept.items():- concept = ConceptFactory.make_from_blueprint(- domain=blueprint.domain,- concept_code=concept_code,- concept_codes_from_the_same_domain=list(blueprint.concept.keys()),- blueprint=ConceptBlueprint(definition=concept_blueprint_or_str)- if isinstance(concept_blueprint_or_str, str)- else concept_blueprint_or_str,- )- concepts.append(concept)+ concepts: list[Concept] = []+ for concept_code, concept_blueprint_or_str in blueprint.concept.items():+ concept_blueprint: ConceptBlueprint+ if isinstance(concept_blueprint_or_str, str):+ concept_blueprint = ConceptBlueprint(definition=concept_blueprint_or_str)+ else:+ concept_blueprint = concept_blueprint_or_str+ concept = ConceptFactory.make_from_blueprint(+ domain=blueprint.domain,+ concept_code=concept_code,+ concept_codes_from_the_same_domain=list(blueprint.concept.keys()),+ blueprint=concept_blueprint,+ )+ concepts.append(concept)return conceptsdef _load_pipes_from_blueprint(self, blueprint: PipelexBundleBlueprint) -> list[PipeAbstract]:@@ -228,43 +251,48 @@ class LibraryManager(LibraryManagerAbstract):library_dirs: list[Path] | None = None,library_file_paths: list[Path] | None = None,) -> None:- dirs_to_use: list[Path] = self._get_pipeline_library_dirs()- all_plx_paths: list[Path] = self._get_pipelex_plx_files_from_dirs(dirs_to_use)-- # Remove failing pipelines from the list- failing_pipelines_file_paths = get_config().pipelex.library_config.failing_pipelines_file_paths- failing_paths_set = {Path(fp) for fp in failing_pipelines_file_paths}- all_plx_paths = [path for path in all_plx_paths if path not in failing_paths_set]+ dirs_to_use = library_dirs or self._get_pipeline_library_dirs()- if library_dirs is not None:- dirs_to_use = library_dirs+ valid_plx_paths: list[Path]+ if library_file_paths:+ valid_plx_paths = library_file_paths+ else:+ all_plx_paths: list[Path] = self._get_pipelex_plx_files_from_dirs(dirs_to_use)+ # Remove failing pipelines from the list+ failing_pipelines_file_paths = get_config().pipelex.library_config.failing_pipelines_file_paths+ valid_plx_paths = [path for path in all_plx_paths if path not in failing_pipelines_file_paths]# Register classes in the directoriesfor library_dir in dirs_to_use:ClassRegistryUtils.register_classes_in_folder(folder_path=str(library_dir))FuncRegistryUtils.register_funcs_in_folder(folder_path=str(library_dir))- if library_file_paths is not None:- all_plx_paths = library_file_paths-# Parse all blueprints firstblueprints: list[PipelexBundleBlueprint] = []- for plx_file_path in all_plx_paths:+ for plx_file_path in valid_plx_paths:try:blueprint = PipelexInterpreter(file_path=plx_file_path).make_pipelex_bundle_blueprint()except FileNotFoundError as exc:msg = f"Could not find PLX blueprint at '{plx_file_path}'"- raise PipelexSetupError(msg) from exc+ raise LibraryLoadingError(msg) from excexcept ValidationError as exc:formatted_error_msg = format_pydantic_validation_error(exc)msg = f"Could not load PLX blueprint from '{plx_file_path}' because of: {formatted_error_msg}"- raise PipelexSetupError(msg) from exc+ raise LibraryLoadingError(msg) from exc+ except PipeDefinitionError as exc:+ msg = f"Could not load PLX blueprint from '{plx_file_path}': {exc}"+ raise LibraryLoadingError(msg) from exc+ blueprint.source = str(plx_file_path)blueprints.append(blueprint)# Load all domains firstall_domains: list[Domain] = []for blueprint in blueprints:- domain = self._load_domain_from_blueprint(blueprint)+ try:+ domain = self._load_domain_from_blueprint(blueprint)+ except DomainDefinitionError as exc:+ msg = f"Could not load domain from PLX blueprint at '{blueprint.source}', domain code: '{blueprint.domain}': {exc}"+ raise LibraryLoadingError(msg) from excall_domains.append(domain)for domain in all_domains:self.domain_library.add_domain(domain=domain)@@ -272,13 +300,21 @@ class LibraryManager(LibraryManagerAbstract):# Load all concepts secondall_concepts: list[Concept] = []for blueprint in blueprints:- concepts = self._load_concepts_from_blueprint(blueprint)+ try:+ concepts = self._load_concepts_from_blueprint(blueprint)+ except ConceptDefinitionError as exc:+ msg = f"Could not load concepts from PLX blueprint at '{blueprint.source}', domain code: '{blueprint.domain}': {exc}"+ raise LibraryLoadingError(msg) from excall_concepts.extend(concepts)self.concept_library.add_concepts(concepts=all_concepts)# Load all pipes thirdall_pipes: list[PipeAbstract] = []for blueprint in blueprints:- pipes = self._load_pipes_from_blueprint(blueprint)+ try:+ pipes = self._load_pipes_from_blueprint(blueprint)+ except PipeDefinitionError as exc:+ msg = f"Could not load pipes from PLX blueprint at '{blueprint.source}', domain code: '{blueprint.domain}': {exc}"+ raise LibraryLoadingError(msg) from excall_pipes.extend(pipes)self.pipe_library.add_pipes(pipes=all_pipes)diff --git a/pipelex/libraries/library_manager_abstract.py b/pipelex/libraries/library_manager_abstract.pyindex 3f93d87ed..dfc6840b8 100644--- a/pipelex/libraries/library_manager_abstract.py+++ b/pipelex/libraries/library_manager_abstract.py@@ -26,9 +26,9 @@ class LibraryManagerAbstract(ABC):def load_libraries(self, library_dirs: list[Path] | None = None, library_file_paths: list[Path] | None = None) -> None:pass- @abstractmethod- def load_from_file(self, plx_path: Path) -> None:- pass+ # @abstractmethod+ # def load_from_file(self, plx_path: Path) -> None:+ # pass@abstractmethoddef load_from_blueprint(self, blueprint: PipelexBundleBlueprint) -> list[PipeAbstract]:diff --git a/pipelex/libraries/pipelines/builder/builder.plx b/pipelex/libraries/pipelines/builder/builder.plxindex d610ba2bf..2289a482e 100644--- a/pipelex/libraries/pipelines/builder/builder.plx+++ b/pipelex/libraries/pipelines/builder/builder.plx@@ -102,6 +102,7 @@ parallels = [{ pipe = "draft_to_conceptspecs_text", result = "concept_spec_drafts_text" },{ pipe = "draft_to_pipesignatures_text", result = "pipe_signatures_text" },]+add_each_output = true[pipe.draft_to_conceptspecs_text]type = "PipeLLM"diff --git a/pipelex/libraries/pipelines/builder/pipe/pipe_parallel_spec.py b/pipelex/libraries/pipelines/builder/pipe/pipe_parallel_spec.pyindex 2afdf8b6e..f3e481410 100644--- a/pipelex/libraries/pipelines/builder/pipe/pipe_parallel_spec.py+++ b/pipelex/libraries/pipelines/builder/pipe/pipe_parallel_spec.py@@ -1,12 +1,14 @@from typing import Literal-from pydantic import Field, field_validator+from pydantic import Field, field_validator, model_validatorfrom typing_extensions import override+from pipelex.exceptions import PipeDefinitionErrorfrom pipelex.libraries.pipelines.builder.concept.concept_spec import ConceptBlueprintfrom pipelex.libraries.pipelines.builder.pipe.pipe_signature import PipeSpecfrom pipelex.libraries.pipelines.builder.pipe.sub_pipe_spec import SubPipeSpecfrom pipelex.pipe_controllers.parallel.pipe_parallel_blueprint import PipeParallelBlueprint+from pipelex.types import Selfclass PipeParallelSpec(PipeSpec):@@ -38,7 +40,7 @@ class PipeParallelSpec(PipeSpec):category: Literal["PipeController"] = "PipeController"the_pipe_code: str = Field(description="Pipe code. Must be snake_case.")parallels: list[SubPipeSpec]- add_each_output: bool = True+ add_each_output: bool = Falsecombined_output: str | None = None@field_validator("combined_output", mode="before")@@ -48,6 +50,16 @@ class PipeParallelSpec(PipeSpec):ConceptBlueprint.validate_concept_string_or_code(concept_string_or_code=combined_output)return combined_output+ @model_validator(mode="after")+ def validate_output_options(self) -> Self:+ if not self.add_each_output and not self.combined_output:+ msg = (+ "PipeParallel requires either add_each_output to be True or combined_output to be set, "+ "or both, otherwise the pipe won't output anything"+ )+ raise PipeDefinitionError(msg)+ return self+@overridedef to_blueprint(self) -> PipeParallelBlueprint:base_blueprint = super().to_blueprint()diff --git a/pipelex/libraries/pipelines/builder/pipe/sub_pipe_spec.py b/pipelex/libraries/pipelines/builder/pipe/sub_pipe_spec.pyindex 42423bd3b..ce0cd7ab1 100644--- a/pipelex/libraries/pipelines/builder/pipe/sub_pipe_spec.py+++ b/pipelex/libraries/pipelines/builder/pipe/sub_pipe_spec.py@@ -42,7 +42,7 @@ class SubPipeSpec(StructuredContent):result: str | None = Nonenb_output: int | None = Nonemultiple_output: bool | None = None- batch_over: bool | str = False+ batch_over: str | None = Nonebatch_as: str | None = None@model_validator(mode="after")@@ -54,14 +54,11 @@ class SubPipeSpec(StructuredContent):@model_validator(mode="after")def validate_batch_params(self) -> Self:- batch_over_is_specified = self.batch_over is not False and self.batch_over != ""- batch_as_is_specified = self.batch_as is not None and self.batch_as != ""-- if batch_over_is_specified and not batch_as_is_specified:+ if self.batch_over and not self.batch_as:msg = f"In pipe '{self.the_pipe_code}': When 'batch_over' is specified, 'batch_as' must also be provided"raise PipeDefinitionError(msg)- if batch_as_is_specified and not batch_over_is_specified:+ if self.batch_as and not self.batch_over:msg = f"In pipe '{self.the_pipe_code}': When 'batch_as' is specified, 'batch_over' must also be provided"raise PipeDefinitionError(msg)diff --git a/pipelex/pipe_controllers/batch/pipe_batch_factory.py b/pipelex/pipe_controllers/batch/pipe_batch_factory.pyindex 3eeb04bb2..469bccaea 100644--- a/pipelex/pipe_controllers/batch/pipe_batch_factory.py+++ b/pipelex/pipe_controllers/batch/pipe_batch_factory.py@@ -41,7 +41,7 @@ class PipeBatchFactory(PipeFactoryProtocol[PipeBatchBlueprint, PipeBatch]):),branch_pipe_code=blueprint.branch_pipe_code,batch_params=BatchParams.make_optional_batch_params(- input_list_name=blueprint.input_list_name or False,+ input_list_name=blueprint.input_list_name,input_item_name=blueprint.input_item_name,),)diff --git a/pipelex/pipe_controllers/parallel/pipe_parallel_blueprint.py b/pipelex/pipe_controllers/parallel/pipe_parallel_blueprint.pyindex f9f5599a0..9351f0814 100644--- a/pipelex/pipe_controllers/parallel/pipe_parallel_blueprint.py+++ b/pipelex/pipe_controllers/parallel/pipe_parallel_blueprint.py@@ -1,17 +1,19 @@from typing import Literal-from pydantic import field_validator+from pydantic import field_validator, model_validatorfrom pipelex.core.concepts.concept_blueprint import ConceptBlueprintfrom pipelex.core.pipes.pipe_blueprint import PipeBlueprint+from pipelex.exceptions import PipeDefinitionErrorfrom pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprint+from pipelex.types import Selfclass PipeParallelBlueprint(PipeBlueprint):type: Literal["PipeParallel"] = "PipeParallel"category: Literal["PipeController"] = "PipeController"parallels: list[SubPipeBlueprint]- add_each_output: bool = True+ add_each_output: bool = Falsecombined_output: str | None = None@field_validator("combined_output", mode="before")@@ -20,3 +22,13 @@ class PipeParallelBlueprint(PipeBlueprint):if combined_output:ConceptBlueprint.validate_concept_string_or_code(concept_string_or_code=combined_output)return combined_output++ @model_validator(mode="after")+ def validate_output_options(self) -> Self:+ if not self.add_each_output and not self.combined_output:+ msg = (+ "PipeParallel requires either add_each_output to be True or combined_output to be set, "+ "or both, otherwise the pipe won't output anything"+ )+ raise PipeDefinitionError(message=msg, description=self.definition, source=self.source)+ return selfdiff --git a/pipelex/pipe_controllers/parallel/pipe_parallel_factory.py b/pipelex/pipe_controllers/parallel/pipe_parallel_factory.pyindex af322d650..b43fa0423 100644--- a/pipelex/pipe_controllers/parallel/pipe_parallel_factory.py+++ b/pipelex/pipe_controllers/parallel/pipe_parallel_factory.py@@ -6,7 +6,7 @@ from pipelex.core.concepts.concept_factory import ConceptFactoryfrom pipelex.core.pipes.pipe_factory import PipeFactoryProtocolfrom pipelex.core.pipes.pipe_input_factory import PipeInputSpecFactoryfrom pipelex.exceptions import PipeDefinitionError-from pipelex.hub import get_concept_provider+from pipelex.hub import get_required_conceptfrom pipelex.pipe_controllers.parallel.pipe_parallel import PipeParallelfrom pipelex.pipe_controllers.parallel.pipe_parallel_blueprint import PipeParallelBlueprintfrom pipelex.pipe_controllers.sub_pipe_factory import SubPipeFactory@@ -33,7 +33,10 @@ class PipeParallelFactory(PipeFactoryProtocol[PipeParallelBlueprint, PipeParallesub_pipe = SubPipeFactory.make_from_blueprint(sub_pipe_blueprint, concept_codes_from_the_same_domain=concept_codes_from_the_same_domain)parallel_sub_pipes.append(sub_pipe)if not blueprint.add_each_output and not blueprint.combined_output:- msg = "PipeParallel requires either add_each_output or combined_output to be set"+ msg = (+ "Unexpected error:PipeParallel requires either add_each_output to be True or combined_output to be set, "+ "or both, otherwise the pipe won't output anything"+ )raise PipeDefinitionError(msg)if blueprint.combined_output:@@ -42,7 +45,7 @@ class PipeParallelFactory(PipeFactoryProtocol[PipeParallelBlueprint, PipeParalleconcept_string_or_code=blueprint.output,concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,)- combined_output = get_concept_provider().get_required_concept(+ combined_output = get_required_concept(concept_string=ConceptFactory.construct_concept_string_with_domain(domain=combined_output_domain_and_code.domain,concept_code=combined_output_domain_and_code.concept_code,@@ -65,13 +68,13 @@ class PipeParallelFactory(PipeFactoryProtocol[PipeParallelBlueprint, PipeParalleblueprint=blueprint.inputs or {},concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,),- output=get_concept_provider().get_required_concept(+ output=get_required_concept(concept_string=ConceptFactory.construct_concept_string_with_domain(domain=output_domain_and_code.domain,concept_code=output_domain_and_code.concept_code,),),parallel_sub_pipes=parallel_sub_pipes,- add_each_output=blueprint.add_each_output,+ add_each_output=blueprint.add_each_output or False,combined_output=combined_output,)diff --git a/pipelex/pipe_controllers/sub_pipe_blueprint.py b/pipelex/pipe_controllers/sub_pipe_blueprint.pyindex 73ee2d1ac..eb69fbff6 100644--- a/pipelex/pipe_controllers/sub_pipe_blueprint.py+++ b/pipelex/pipe_controllers/sub_pipe_blueprint.py@@ -12,7 +12,7 @@ class SubPipeBlueprint(BaseModel):result: str | None = Nonenb_output: int | None = Nonemultiple_output: bool | None = None- batch_over: bool | str = False+ batch_over: str | None = Nonebatch_as: str | None = None@model_validator(mode="after")@@ -24,14 +24,11 @@ class SubPipeBlueprint(BaseModel):@model_validator(mode="after")def validate_batch_params(self) -> Self:- batch_over_is_specified = self.batch_over is not False and self.batch_over != ""- batch_as_is_specified = self.batch_as is not None and self.batch_as != ""-- if batch_over_is_specified and not batch_as_is_specified:+ if self.batch_over and not self.batch_as:msg = f"In pipe '{self.pipe}': When 'batch_over' is specified, 'batch_as' must also be provided"raise PipeDefinitionError(msg)- if batch_as_is_specified and not batch_over_is_specified:+ if self.batch_as and not self.batch_over:msg = f"In pipe '{self.pipe}': When 'batch_as' is specified, 'batch_over' must also be provided"raise PipeDefinitionError(msg)diff --git a/pipelex/tools/plx/plx_utils.py b/pipelex/tools/plx/plx_utils.pyindex 0b06accd8..13c82ec5b 100644--- a/pipelex/tools/plx/plx_utils.py+++ b/pipelex/tools/plx/plx_utils.py@@ -1,7 +1,5 @@from __future__ import annotations-import re-from re import Matchfrom typing import TYPE_CHECKING, Any, Mapping, castimport tomlkitdiff --git a/tests/unit/pipelex/core/concepts/concept_factory.py/test_concept_factory.py b/tests/unit/pipelex/core/concepts/concept_factory.py/test_concept_factory.pyindex 8ea032084..500709d87 100644--- a/tests/unit/pipelex/core/concepts/concept_factory.py/test_concept_factory.py+++ b/tests/unit/pipelex/core/concepts/concept_factory.py/test_concept_factory.py@@ -19,14 +19,16 @@ class TestConceptFactory:"test_name,blueprint,expected_result",TestCases.MAKE_REFINES_TEST_CASES,)- def test_make_refines(+ def test_make_refine(self,test_name: str,blueprint: ConceptBlueprint,expected_result: str,):"""Test make_refines method with different blueprint configurations."""- result = ConceptFactory.make_refines(blueprint=blueprint)+ if not blueprint.refines:+ pytest.skip("Test case has no refines")+ result = ConceptFactory.make_refine(refine=blueprint.refines)assert result == expected_result, f"Failed for test case: {test_name}"def test_normalize_structure_blueprint(self):diff --git a/tests/unit/pipelex/core/concepts/concept_factory/test_concept_factory.py b/tests/unit/pipelex/core/concepts/concept_factory/test_concept_factory.pyindex 195a43805..1d3a7a066 100644--- a/tests/unit/pipelex/core/concepts/concept_factory/test_concept_factory.py+++ b/tests/unit/pipelex/core/concepts/concept_factory/test_concept_factory.py@@ -21,14 +21,16 @@ class TestConceptFactory:"test_name,blueprint,expected_result",TestCases.MAKE_REFINES_TEST_CASES,)- def test_make_refines(+ def test_make_refine(self,test_name: str,blueprint: ConceptBlueprint,expected_result: str,):"""Test make_refines method with different blueprint configurations."""- result = ConceptFactory.make_refines(blueprint=blueprint)+ if not blueprint.refines:+ pytest.skip("Test case has no refines")+ result = ConceptFactory.make_refine(refine=blueprint.refines)assert result == expected_result, f"Failed for test case: {test_name}"def test_normalize_structure_blueprint(self):diff --git a/tests/unit/pipelex/core/test_data/__init__.py b/tests/unit/pipelex/core/test_data/__init__.pyindex b62667c26..995f06715 100644--- a/tests/unit/pipelex/core/test_data/__init__.py+++ b/tests/unit/pipelex/core/test_data/__init__.py@@ -26,20 +26,20 @@ class InterpreterTestCases:VALID_TEST_CASES: ClassVar[List[Tuple[str, str, PipelexBundleBlueprint]]] = [ # test_name,plx_content,blueprint# Domain tests*DOMAIN_TEST_CASES,- # # Concept tests+ # Concept tests*STRUCTURED_CONCEPT_TEST_CASES,- # *REFINING_CONCEPT_TEST_CASES,- # *SIMPLE_CONCEPT_TEST_CASES,- # # Pipe operator tests- # *PIPE_LLM_TEST_CASES,- # *PIPE_OCR_TEST_CASES,- # *PIPE_FUNC_TEST_CASES,- # *PIPE_IMG_GEN_TEST_CASES,- # *PIPE_JINJA2_TEST_CASES,- # # # Pipe controller tests- # *PIPE_SEQUENCE_TEST_CASES,- # *PIPE_CONDITION_TEST_CASES,- # *PIPE_PARALLEL_TEST_CASES,+ *REFINING_CONCEPT_TEST_CASES,+ *SIMPLE_CONCEPT_TEST_CASES,+ # Pipe operator tests+ *PIPE_LLM_TEST_CASES,+ *PIPE_OCR_TEST_CASES,+ *PIPE_FUNC_TEST_CASES,+ *PIPE_IMG_GEN_TEST_CASES,+ *PIPE_JINJA2_TEST_CASES,+ # Pipe controller tests+ *PIPE_SEQUENCE_TEST_CASES,+ *PIPE_CONDITION_TEST_CASES,+ *PIPE_PARALLEL_TEST_CASES,# *PIPE_BATCH_TEST_CASES,# # Complex tests# *COMPLEX_TEST_CASES,diff --git a/tests/unit/pipelex/core/test_data/pipes/controllers/condition/pipe_condition.py b/tests/unit/pipelex/core/test_data/pipes/controllers/condition/pipe_condition.pyindex d83deab06..d25ae5095 100644--- a/tests/unit/pipelex/core/test_data/pipes/controllers/condition/pipe_condition.py+++ b/tests/unit/pipelex/core/test_data/pipes/controllers/condition/pipe_condition.py@@ -10,8 +10,8 @@ definition = "Domain with conditional pipe"type = "PipeCondition"definition = "Process based on condition"output = "ProcessedData"-pipe_map = { small = "process_small", large = "process_large" }expression = "input_data.category"+pipe_map = { small = "process_small", large = "process_large" }""",PipelexBundleBlueprint(domain="test_pipes",diff --git a/tests/unit/pipelex/core/test_data/pipes/controllers/parallel/pipe_parallel.py b/tests/unit/pipelex/core/test_data/pipes/controllers/parallel/pipe_parallel.pyindex 1056a0abc..f184e46de 100644--- a/tests/unit/pipelex/core/test_data/pipes/controllers/parallel/pipe_parallel.py+++ b/tests/unit/pipelex/core/test_data/pipes/controllers/parallel/pipe_parallel.py@@ -9,12 +9,13 @@ definition = "Domain with parallel pipe"[pipe.parallel_process]type = "PipeParallel"-definition = "Process data in parallel"+definition = "PipeParallel example in PIPE_PARALLEL_TEST_CASES"output = "ProcessedData"parallels = [{ pipe = "process_a", result = "result_a" },{ pipe = "process_b", result = "result_b" },]+add_each_output = true""",PipelexBundleBlueprint(domain="test_pipes",@@ -22,12 +23,13 @@ parallels = [pipe={"parallel_process": PipeParallelBlueprint(type="PipeParallel",- definition="Process data in parallel",+ definition="PipeParallel example in PIPE_PARALLEL_TEST_CASES",output="ProcessedData",parallels=[SubPipeBlueprint(pipe="process_a", result="result_a"),SubPipeBlueprint(pipe="process_b", result="result_b"),],+ add_each_output=True,),},),diff --git a/tests/unit/pipelex/core/test_data/pipes/operators/llm/pipe_llm.py b/tests/unit/pipelex/core/test_data/pipes/operators/llm/pipe_llm.pyindex 3a1fab21f..e59279f18 100644--- a/tests/unit/pipelex/core/test_data/pipes/operators/llm/pipe_llm.py+++ b/tests/unit/pipelex/core/test_data/pipes/operators/llm/pipe_llm.py@@ -55,7 +55,7 @@ Extract information about $topic from this text:definition="Extract information from text",inputs={"text": "Text", "topic": "Text"},output=NativeConceptEnum.TEXT,- prompt_template="Extract information about $topic from this text:\n\n@text",+ prompt_template="Extract information about $topic from this text:\n\n@text\n",),},),@@ -335,7 +335,7 @@ Documents: @documents# Export all PipeLLM test casesPIPE_LLM_TEST_CASES = [- # PIPE_LLM_BASIC,+ PIPE_LLM_BASIC,PIPE_LLM_WITH_INPUTS,PIPE_LLM_WITH_SYSTEM_PROMPT,PIPE_LLM_MULTIPLE_OUTPUTS,diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_data.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_data.pyindex 25023d6eb..63fbca416 100644--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_data.py+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_data.py@@ -9,8 +9,8 @@ from pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprintclass PipeParallelTestCases:- SIMPLE_PARALLEL = (- "simple_parallel",+ PARALLEL_WITH_EACH_OUTPUT = (+ "parallel_with_each_output",PipeParallelSpec(the_pipe_code="parallel_processor",definition="Run pipes in parallel",@@ -21,6 +21,7 @@ class PipeParallelTestCases:SubPipeSpec(the_pipe_code="transform_data", result="transformed"),SubPipeSpec(the_pipe_code="validate_data", result="validation"),],+ add_each_output=True,),PipeParallelBlueprint(definition="Run pipes in parallel",@@ -35,6 +36,7 @@ class PipeParallelTestCases:],add_each_output=True,combined_output=None,+ source="tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_data.py PipeParallelTestCases.PARALLEL_WITH_EACH_OUTPUT",),)@@ -64,10 +66,42 @@ class PipeParallelTestCases:],add_each_output=False,combined_output="MergedData",+ source="tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_data.py PipeParallelTestCases.PARALLEL_WITH_COMBINED",+ ),+ )++ PARALLEL_WITH_BOTH_EACH_OUTPUT_AND_COMBINED = (+ "parallel_with_both_each_output_and_combined",+ PipeParallelSpec(+ the_pipe_code="combined_parallel",+ definition="Parallel with combined output",+ inputs={"input": InputRequirementSpec(concept="Input")},+ output="CombinedResult",+ parallels=[+ SubPipeSpec(the_pipe_code="pipe1", result="result1"),+ SubPipeSpec(the_pipe_code="pipe2", result="result2"),+ ],+ add_each_output=True,+ combined_output="MergedData",+ ),+ PipeParallelBlueprint(+ definition="Parallel with combined output",+ inputs={"input": InputRequirementBlueprint(concept="Input")},+ output="CombinedResult",+ type="PipeParallel",+ category="PipeController",+ parallels=[+ SubPipeBlueprint(pipe="pipe1", result="result1"),+ SubPipeBlueprint(pipe="pipe2", result="result2"),+ ],+ add_each_output=True,+ combined_output="MergedData",+ source="tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_data.py PipeParallelTestCases.PARALLEL_WITH_BOTH_EACH_OUTPUT_AND_COMBINED",),)TEST_CASES: ClassVar[list[tuple[str, PipeParallelSpec, PipeParallelBlueprint]]] = [- SIMPLE_PARALLEL,+ PARALLEL_WITH_EACH_OUTPUT,PARALLEL_WITH_COMBINED,+ PARALLEL_WITH_BOTH_EACH_OUTPUT_AND_COMBINED,]diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_pipe_parallel.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_pipe_parallel.pyindex 04d999a7f..e35dd1669 100644--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_pipe_parallel.py+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_pipe_parallel.py@@ -4,7 +4,7 @@ from pipelex.libraries.pipelines.builder.pipe.pipe_parallel_spec import PipeParafrom pipelex.pipe_controllers.parallel.pipe_parallel_blueprint import PipeParallelBlueprintfrom .test_data import PipeParallelTestCases-+from pipelex import logclass TestPipeParallelBlueprintConversion:@pytest.mark.parametrize(@@ -17,5 +17,7 @@ class TestPipeParallelBlueprintConversion:pipe_spec: PipeParallelSpec,expected_blueprint: PipeParallelBlueprint,):+ log.verbose(f"Testing {test_name}")result = pipe_spec.to_blueprint()+ expected_blueprint.source = Noneassert result == expected_blueprint