Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
Use tomlkit to generate PLX and improve library error handling (#295)
Louis Choquel844fa8d3db4b
diff --git a/.badges/tests.json b/.badges/tests.jsonindex d43b5371e..056566a85 100644--- a/.badges/tests.json+++ b/.badges/tests.json@@ -1,7 +1,7 @@{- "schemaVersion": 1,- "label": "tests",- "message": "1121",- "color": "blue",- "cacheSeconds": 300- }\ No newline at end of file+ "schemaVersion": 1,+ "label": "tests",+ "message": "1045",+ "color": "blue",+ "cacheSeconds": 300+}\ No newline at end of filediff --git a/Makefile b/Makefileindex d9b38e0e3..ab8a0cd3a 100644--- a/Makefile+++ b/Makefile@@ -455,11 +455,11 @@ docs-deploy: env### SHORTHANDS##########################################################################################-c: format lint pyright pylint mypy+c: format lint pyright mypy pylint@echo "> done: c = check"cc: cleanderived c- @echo "> done: cc = cleanderived format lint pyright pylint mypy"+ @echo "> done: cc = cleanderived format lint pyright mypy pylint"check: cc check-unused-imports@echo "> done: check"diff --git a/pipelex/cli/commands/build_cmd.py b/pipelex/cli/commands/build_cmd.pyindex fb64deadb..567c83224 100644--- a/pipelex/cli/commands/build_cmd.py+++ b/pipelex/cli/commands/build_cmd.py@@ -5,11 +5,12 @@ from typing import Annotatedimport typerfrom pipelex import pretty_print-from pipelex.core.interpreter import PipelexInterpreterfrom pipelex.hub import get_report_delegate+from pipelex.language.plx_factory import PlxFactoryfrom pipelex.libraries.pipelines.builder.builder import PipelexBundleSpecfrom pipelex.pipelex import Pipelexfrom pipelex.pipeline.execute import execute_pipeline+from pipelex.tools.misc.file_utils import ensure_directory_for_file_path, save_text_to_pathbuild_app = typer.Typer(help="Build artifacts like pipelines", no_args_is_help=True)@@ -30,9 +31,13 @@ def build_pipe_cmd(typer.Argument(help="Brief description of what the pipeline should do"),],output_path: Annotated[- str | None,- typer.Option("--output", "-o", help="Path to save the generated PLX file (use --output='' to skip saving)"),- ] = "./generated_pipeline.plx",+ str,+ typer.Option("--output", "-o", help="Path to save the generated PLX file"),+ ] = "./results/generated_pipeline.plx",+ no_output: Annotated[+ bool,+ typer.Option("--no-output", help="Skip saving the pipeline to file"),+ ] = False,) -> None:Pipelex.make(relative_config_folder_path="../../../pipelex/libraries", from_file=True)typer.echo("=" * 70)@@ -40,21 +45,29 @@ def build_pipe_cmd(typer.echo("")async def run_pipeline():+ if not no_output:+ if not output_path:+ typer.echo(typer.style("\n⚠️ Cannot save a pipeline to an empty file name", fg=typer.colors.RED))+ raise typer.Exit(1)++ ensure_directory_for_file_path(file_path=output_path)+ else:+ typer.echo(typer.style("\n⚠️ Pipeline will not be saved to file (--no-output specified)", fg=typer.colors.YELLOW))+pipe_output = await execute_pipeline(pipe_code="pipe_builder",input_memory={"brief": brief},)pretty_print(pipe_output, title="Pipe Output")pipelex_bundle_spec = pipe_output.working_memory.get_stuff_as(name="pipelex_bundle_spec", content_type=PipelexBundleSpec)- plx_content = PipelexInterpreter.make_plx_content(blueprint=pipelex_bundle_spec.to_blueprint())+ plx_content = PlxFactory.make_plx_content(blueprint=pipelex_bundle_spec.to_blueprint())- # Save to file unless explicitly disabled with empty string- if output_path and output_path != "":- with open(output_path, "w") as f:- f.write(plx_content)+ # Save to file unless explicitly disabled with --no-output+ if not no_output:+ save_text_to_path(text=plx_content, path=output_path)typer.echo(typer.style(f"\n✅ Pipeline saved to: {output_path}", fg=typer.colors.GREEN))- elif output_path == "":- typer.echo(typer.style("\n⚠️ Pipeline not saved to file (--output='' specified)", fg=typer.colors.YELLOW))+ else:+ typer.echo(typer.style("\n⚠️ Pipeline not saved to file (--no-output specified)", fg=typer.colors.YELLOW))start_time = time.time()asyncio.run(run_pipeline())diff --git a/pipelex/cli/commands/your-pipe.plx b/pipelex/cli/commands/your-pipe.plxdeleted file mode 100644index e69de29bb..000000000diff --git a/pipelex/config.py b/pipelex/config.pyindex a41a7def0..96d062275 100644--- a/pipelex/config.py+++ b/pipelex/config.py@@ -7,6 +7,7 @@ from pipelex.cogt.config_cogt import Cogtfrom pipelex.cogt.model_backends.prompting_target import PromptingTargetfrom pipelex.exceptions import PipelexConfigError, StaticValidationErrorTypefrom pipelex.hub import get_required_config+from pipelex.language.plx_config import PlxConfigfrom pipelex.libraries.library_config import LibraryConfigfrom pipelex.pipeline.track.tracker_config import TrackerConfigfrom pipelex.tools.aws.aws_config import AwsConfig@@ -110,6 +111,7 @@ class Pipelex(ConfigModel):tracker_config: TrackerConfigstructure_config: StructureConfigprompting_config: PromptingConfig+ plx_config: PlxConfigdry_run_config: DryRunConfigpipe_run_config: PipeRunConfigdiff --git a/pipelex/core/bundles/pipelex_bundle_blueprint.py b/pipelex/core/bundles/pipelex_bundle_blueprint.pyindex 66ce37b84..91cb86c39 100644--- a/pipelex/core/bundles/pipelex_bundle_blueprint.py+++ b/pipelex/core/bundles/pipelex_bundle_blueprint.py@@ -31,6 +31,7 @@ PipeBlueprintUnion = Annotated[class PipelexBundleBlueprint(BaseModel):model_config = ConfigDict(extra="forbid")+ source: str | None = Nonedomain: strdefinition: str | None = Nonesystem_prompt: str | None = Nonediff --git a/pipelex/core/concepts/concept_blueprint.py b/pipelex/core/concepts/concept_blueprint.pyindex 493d4c7b0..eaec48747 100644--- a/pipelex/core/concepts/concept_blueprint.py+++ b/pipelex/core/concepts/concept_blueprint.py@@ -35,7 +35,7 @@ class ConceptStructureBlueprint(BaseModel):item_type: str | None = Nonekey_type: str | None = Nonevalue_type: str | None = None- choices: list[str] | None = Field(default_factory=list)+ choices: list[str] | None = Field(default=None)required: bool | None = Field(default=True)default_value: Any | None = Nonediff --git a/pipelex/core/concepts/concept_factory.py b/pipelex/core/concepts/concept_factory.pyindex f501974be..b80758831 100644--- a/pipelex/core/concepts/concept_factory.py+++ b/pipelex/core/concepts/concept_factory.py@@ -13,7 +13,7 @@ from pipelex.core.concepts.concept_native import NativeConceptEnumData, NativeCofrom pipelex.core.concepts.structure_generator import StructureGeneratorfrom pipelex.core.domains.domain import SpecialDomainfrom pipelex.core.stuffs.stuff_content import TextContent-from pipelex.exceptions import ConceptFactoryError, StructureClassError+from pipelex.exceptions import ConceptCodeError, ConceptDefinitionError, ConceptRefineError, StructureClassErrorclass DomainAndConceptCode(BaseModel):@@ -106,16 +106,10 @@ class ConceptFactory:@classmethoddef make_refine(cls, refine: str) -> str:- if NativeConceptManager.is_native_concept(concept_string_or_code=refine):- return NativeConceptManager.get_native_concept_string(concept_string_or_code=refine)- msg = f"Refine '{refine}' is not a native concept"- raise ConceptFactoryError(msg)-- @classmethod- def make_refines(cls, blueprint: ConceptBlueprint) -> str | None:- if blueprint.refines:- return cls.make_refine(refine=blueprint.refines)- return None+ if not NativeConceptManager.is_native_concept(concept_string_or_code=refine):+ msg = f"Refine '{refine}' is not a native concept"+ raise ConceptRefineError(msg)+ return NativeConceptManager.get_native_concept_string(concept_string_or_code=refine)@classmethoddef make_from_blueprint(@@ -125,7 +119,11 @@ class ConceptFactory:blueprint: ConceptBlueprint,concept_codes_from_the_same_domain: list[str] | None = None,) -> Concept:- ConceptBlueprint.validate_concept_code(concept_code=concept_code)+ try:+ ConceptBlueprint.validate_concept_code(concept_code=concept_code)+ except ConceptCodeError as exc:+ msg = f"Could not validate concept code '{concept_code}' in domain '{domain}': {exc}"+ raise ConceptDefinitionError(msg) from excstructure_class_name: strcurrent_refine: str | None = None@@ -163,7 +161,7 @@ class ConceptFactory:except Exception as exc:msg = f"Error generating structure class for concept '{concept_code}' in domain '{domain}': {exc}"- raise ConceptFactoryError(msg) from exc+ raise ConceptDefinitionError(msg) from exc# Handle refines definitionelif blueprint.refines:@@ -173,8 +171,12 @@ class ConceptFactory:f"Concept '{concept_code}' in domain '{domain}' has refines but also has a structure class registered. ""A concept cannot have both structure and refines.")- raise ConceptFactoryError(msg)- current_refine = cls.make_refines(blueprint=blueprint)+ raise ConceptDefinitionError(msg)+ try:+ current_refine = cls.make_refine(refine=blueprint.refines)+ except ConceptRefineError as exc:+ msg = f"Could not validate refine '{blueprint.refines}' for concept '{concept_code}' in domain '{domain}': {exc}"+ raise ConceptDefinitionError(msg) from excstructure_class_name = current_refine.split(".")[1] + "Content" if current_refine else TextContent.__name__# Handle neither structure nor refines - check the class registry# If there is a class, use it. structure_class_name is then the concept_codediff --git a/pipelex/core/domains/domain_factory.py b/pipelex/core/domains/domain_factory.pyindex 449349ea2..bc9dfc7f6 100644--- a/pipelex/core/domains/domain_factory.py+++ b/pipelex/core/domains/domain_factory.py@@ -1,14 +1,23 @@+from pydantic import ValidationError+from pipelex.core.domains.domain import Domainfrom pipelex.core.domains.domain_blueprint import DomainBlueprint+from pipelex.exceptions import DomainDefinitionError+from pipelex.tools.typing.pydantic_utils import format_pydantic_validation_errorclass DomainFactory:@classmethoddef make_from_blueprint(cls, blueprint: DomainBlueprint) -> Domain:- return Domain(- code=blueprint.code,- definition=blueprint.definition,- system_prompt=blueprint.system_prompt,- system_prompt_to_structure=blueprint.system_prompt_to_structure,- prompt_template_to_structure=blueprint.prompt_template_to_structure,- )+ try:+ return Domain(+ code=blueprint.code,+ definition=blueprint.definition,+ system_prompt=blueprint.system_prompt,+ system_prompt_to_structure=blueprint.system_prompt_to_structure,+ prompt_template_to_structure=blueprint.prompt_template_to_structure,+ )+ except ValidationError as exc:+ formatted_error_msg = format_pydantic_validation_error(exc)+ msg = f"Could not make domain from blueprint: {formatted_error_msg}"+ raise DomainDefinitionError(msg) from excdiff --git a/pipelex/core/domains/domain_library.py b/pipelex/core/domains/domain_library.pyindex f9a0abe14..9574d0026 100644--- a/pipelex/core/domains/domain_library.py+++ b/pipelex/core/domains/domain_library.py@@ -28,6 +28,10 @@ class DomainLibrary(RootModel[DomainLibraryRoot], DomainProviderAbstract):else:self.root[domain_code] = domain+ def add_domains(self, domains: list[Domain]):+ for domain in domains:+ self.add_domain(domain=domain)+def remove_domain_by_code(self, domain_code: str) -> None:if domain_code in self.root:del self.root[domain_code]diff --git a/pipelex/core/interpreter.py b/pipelex/core/interpreter.pyindex 0d6d9509b..383ce55e0 100644--- a/pipelex/core/interpreter.py+++ b/pipelex/core/interpreter.py@@ -1,32 +1,13 @@-from collections.abc import Mappingfrom pathlib import Path-from typing import Any, cast+from typing import Anyimport tomlfrom pydantic import BaseModel, model_validatorfrom pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint-from pipelex.core.concepts.concept_blueprint import ConceptBlueprint, ConceptStructureBlueprintfrom pipelex.core.exceptions import (PipelexConfigurationError,- PipelexUnknownPipeError,)-from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint-from pipelex.pipe_controllers.batch.pipe_batch_blueprint import PipeBatchBlueprint-from pipelex.pipe_controllers.condition.pipe_condition_blueprint import (- PipeConditionBlueprint,-)-from pipelex.pipe_controllers.parallel.pipe_parallel_blueprint import (- PipeParallelBlueprint,-)-from pipelex.pipe_controllers.sequence.pipe_sequence_blueprint import (- PipeSequenceBlueprint,-)-from pipelex.pipe_operators.func.pipe_func_blueprint import PipeFuncBlueprint-from pipelex.pipe_operators.img_gen.pipe_img_gen_blueprint import PipeImgGenBlueprint-from pipelex.pipe_operators.jinja2.pipe_jinja2_blueprint import PipeJinja2Blueprint-from pipelex.pipe_operators.llm.pipe_llm_blueprint import PipeLLMBlueprint-from pipelex.pipe_operators.ocr.pipe_ocr_blueprint import PipeOcrBlueprintfrom pipelex.tools.misc.toml_utils import (clean_trailing_whitespace,validate_toml_content,@@ -142,866 +123,3 @@ class PipelexInterpreter(BaseModel):file_content = self._read_and_reformat()plx_data = self._parse_plx_content(file_content)return PipelexBundleBlueprint.model_validate(plx_data)-- @staticmethod- def make_plx_content(blueprint: PipelexBundleBlueprint) -> str:- """Convert a PipelexBundleBlueprint to properly formatted PLX content."""- plx_parts: list[str] = []-- # Domain-level fields- domain_fields: list[str] = []- domain_fields.append(f'domain = "{PipelexInterpreter.escape_plx_string(blueprint.domain)}"')- if blueprint.definition:- domain_fields.append(f'definition = "{PipelexInterpreter.escape_plx_string(blueprint.definition)}"')- if blueprint.system_prompt:- domain_fields.append(f'system_prompt = "{PipelexInterpreter.escape_plx_string(blueprint.system_prompt)}"')- if blueprint.system_prompt_to_structure:- domain_fields.append(f'system_prompt_to_structure = "{PipelexInterpreter.escape_plx_string(blueprint.system_prompt_to_structure)}"')- if blueprint.prompt_template_to_structure:- domain_fields.append(f'prompt_template_to_structure = "{PipelexInterpreter.escape_plx_string(blueprint.prompt_template_to_structure)}"')-- plx_parts.append("\n".join(domain_fields))-- # Concepts section- if blueprint.concept:- concept_plx = PipelexInterpreter.concepts_to_plx_string(blueprint.concept)- if concept_plx: # Only add if not empty- plx_parts.append(concept_plx)-- # Pipes section- if blueprint.pipe:- pipes_plx = PipelexInterpreter.pipes_to_plx_string(blueprint.pipe)- if pipes_plx: # Only add if not empty- plx_parts.append(pipes_plx)-- result = "\n\n".join(plx_parts)- if result and not result.endswith("\n"):- result += "\n"- return result-- @staticmethod- def concepts_to_plx_string(concepts: dict[str, ConceptBlueprint | str]) -> str:- """Convert concepts dict to PLX string."""- if not concepts:- return ""-- plx_parts: list[str] = []- simple_concepts: list[str] = []- complex_concepts: list[str] = []-- for concept_name, concept_blueprint in concepts.items():- if isinstance(concept_blueprint, str):- simple_concepts.append(f'{concept_name} = "{PipelexInterpreter.escape_plx_string(concept_blueprint)}"')- # Handle ConceptBlueprint objects- elif concept_blueprint.structure is None and concept_blueprint.refines is None:- # Simple concept with just definition- simple_concepts.append(f'{concept_name} = "{PipelexInterpreter.escape_plx_string(concept_blueprint.definition)}"')- else:- # Complex concept needs its own section- complex_concept_plx = PipelexInterpreter.complex_concept_to_plx_string(concept_name, concept_blueprint)- complex_concepts.append(complex_concept_plx)-- # Add simple concepts section if we have any- if simple_concepts:- plx_parts.append("[concept]")- plx_parts.extend(simple_concepts)-- # Add complex concepts with proper spacing- if complex_concepts:- # If we had simple concepts, add an empty line before complex ones- if simple_concepts:- plx_parts.append("")- # Add each complex concept with empty lines between them- for i, complex_concept in enumerate(complex_concepts):- if i > 0: # Add empty line between complex concepts- plx_parts.append("")- plx_parts.append(complex_concept)-- return "\n".join(plx_parts)-- @staticmethod- def complex_concept_to_plx_string(concept_name: str, concept: ConceptBlueprint) -> str:- """Convert a complex ConceptBlueprint to PLX string."""- lines: list[str] = [f"[concept.{concept_name}]"]-- if concept.definition:- lines.append(f'definition = "{PipelexInterpreter.escape_plx_string(concept.definition)}"')-- if concept.refines:- lines.append(f'refines = "{PipelexInterpreter.escape_plx_string(concept.refines)}"')-- if concept.structure:- if isinstance(concept.structure, str):- lines.append(f'structure = "{PipelexInterpreter.escape_plx_string(concept.structure)}"')- else:- # Complex structure with fields- lines.append("")- lines.append(f"[concept.{concept_name}.structure]")- for field_name, field_def in concept.structure.items():- field_plx = PipelexInterpreter.structure_field_to_plx_string(field_name, field_def)- lines.append(field_plx)-- return "\n".join(lines)-- @staticmethod- def structure_field_to_plx_string(field_name: str, field_def: Any) -> str:- """Convert a structure field to PLX string."""- if isinstance(field_def, str):- return f'{field_name} = "{PipelexInterpreter.escape_plx_string(field_def)}"'- # Complex field with type, definition, required, etc.- field_parts: list[str] = []-- if hasattr(field_def, "type") and field_def.type:- type_value = field_def.type.value if hasattr(field_def.type, "value") else field_def.type- field_parts.append(f'type = "{PipelexInterpreter.escape_plx_string(str(type_value))}"')-- if hasattr(field_def, "definition") and field_def.definition:- field_parts.append(f'definition = "{PipelexInterpreter.escape_plx_string(field_def.definition)}"')-- if hasattr(field_def, "required") and field_def.required is not None:- field_parts.append(f"required = {str(field_def.required).lower()}")-- return f"{field_name} = {{ {', '.join(field_parts)} }}"-- @staticmethod- def pipes_to_plx_string(pipes: dict[str, Any]) -> str:- """Convert pipes dict to PLX string."""- plx_parts: list[str] = []- for pipe_name, blueprint in pipes.items():- pipe_plx = PipelexInterpreter.pipe_to_plx_string(pipe_name, blueprint)- plx_parts.append(pipe_plx)- return "\n\n".join(plx_parts)-- @staticmethod- def pipe_to_plx_string(pipe_name: str, blueprint: Any) -> str:- """Convert a single pipe blueprint to PLX string."""- if isinstance(blueprint, PipeLLMBlueprint):- return PipelexInterpreter.llm_pipe_to_plx_string(pipe_name, blueprint)- elif isinstance(blueprint, PipeSequenceBlueprint):- return PipelexInterpreter.sequence_pipe_to_plx_string(pipe_name, blueprint)- elif isinstance(blueprint, PipeOcrBlueprint):- return PipelexInterpreter.ocr_pipe_to_plx_string(pipe_name, blueprint)- elif isinstance(blueprint, PipeFuncBlueprint):- return PipelexInterpreter.func_pipe_to_plx_string(pipe_name, blueprint)- elif isinstance(blueprint, PipeImgGenBlueprint):- return PipelexInterpreter.img_gen_pipe_to_plx_string(pipe_name, blueprint)- elif isinstance(blueprint, PipeJinja2Blueprint):- return PipelexInterpreter.jinja2_pipe_to_plx_string(pipe_name, blueprint)- elif isinstance(blueprint, PipeConditionBlueprint):- return PipelexInterpreter.condition_pipe_to_plx_string(pipe_name, blueprint)- elif isinstance(blueprint, PipeParallelBlueprint):- return PipelexInterpreter.parallel_pipe_to_plx_string(pipe_name, blueprint)- elif isinstance(blueprint, PipeBatchBlueprint):- return PipelexInterpreter.batch_pipe_to_plx_string(pipe_name, blueprint)- else:- # Fallback to old dict approach for unknown pipe types- pipe_dict = PipelexInterpreter.serialize_pipe(blueprint)- return f"[pipe.{pipe_name}]\n" + "\n".join([f'{k} = "{v}"' if isinstance(v, str) else f"{k} = {v}" for k, v in pipe_dict.items()])-- @staticmethod- def llm_pipe_to_plx_string(pipe_name: str, pipe: PipeLLMBlueprint) -> str:- """Convert a PipeLLM blueprint directly to PLX section string."""- lines: list[str] = [- f"[pipe.{pipe_name}]",- f'type = "{PipelexInterpreter.escape_plx_string(pipe.type)}"',- f'definition = "{PipelexInterpreter.escape_plx_string(pipe.definition)}"',- ]-- # Add inputs first if they exist- PipelexInterpreter.add_inputs_to_lines_if_exist(lines, pipe.inputs)-- # Add output after inputs (or immediately if no inputs)- lines.append(f'output = "{PipelexInterpreter.escape_plx_string(pipe.output)}"')-- # Add optional fields- if pipe.nb_output is not None:- lines.append(f"nb_output = {pipe.nb_output}")- if pipe.multiple_output is not None:- lines.append(f"multiple_output = {str(pipe.multiple_output).lower()}")- if pipe.system_prompt_template:- lines.append(f'system_prompt_template = "{PipelexInterpreter.escape_plx_string(pipe.system_prompt_template)}"')- if pipe.system_prompt:- lines.append(f'system_prompt = "{PipelexInterpreter.escape_plx_string(pipe.system_prompt)}"')- if pipe.prompt_template:- lines.append(f'prompt_template = "{PipelexInterpreter.escape_plx_string(pipe.prompt_template)}"')- if pipe.template_name:- lines.append(f'template_name = "{PipelexInterpreter.escape_plx_string(pipe.template_name)}"')- if pipe.prompt:- lines.append(f'prompt = "{PipelexInterpreter.escape_plx_string(pipe.prompt)}"')-- return "\n".join(lines)-- @staticmethod- def ocr_pipe_to_plx_string(pipe_name: str, pipe: PipeOcrBlueprint) -> str:- """Convert a PipeOcr blueprint directly to PLX section string."""- lines: list[str] = [- f"[pipe.{pipe_name}]",- f'type = "{pipe.type}"',- f'definition = "{pipe.definition}"',- ]-- # Add inputs if they exist- PipelexInterpreter.add_inputs_to_lines_if_exist(lines, pipe.inputs)-- lines.append(f'output = "{PipelexInterpreter.escape_plx_string(pipe.output)}"')- lines.append(f'ocr = "{pipe.ocr}"')-- return "\n".join(lines)-- @staticmethod- def func_pipe_to_plx_string(pipe_name: str, pipe: PipeFuncBlueprint) -> str:- """Convert a PipeFunc blueprint directly to PLX section string."""- lines: list[str] = [- f"[pipe.{pipe_name}]",- f'type = "{pipe.type}"',- f'definition = "{pipe.definition}"',- ]-- # Add inputs if they exist- PipelexInterpreter.add_inputs_to_lines_if_exist(lines, pipe.inputs)-- lines.append(f'output = "{PipelexInterpreter.escape_plx_string(pipe.output)}"')- lines.append(f'function_name = "{PipelexInterpreter.escape_plx_string(pipe.function_name)}"')-- return "\n".join(lines)-- @staticmethod- def img_gen_pipe_to_plx_string(pipe_name: str, pipe: PipeImgGenBlueprint) -> str:- """Convert a PipeImgGen blueprint directly to PLX section string."""- lines: list[str] = [- f"[pipe.{pipe_name}]",- f'type = "{pipe.type}"',- f'definition = "{pipe.definition}"',- ]-- # Add inputs if they exist- PipelexInterpreter.add_inputs_to_lines_if_exist(lines, pipe.inputs)-- lines.append(f'output = "{PipelexInterpreter.escape_plx_string(pipe.output)}"')-- # Add optional fields- if pipe.img_gen_prompt:- lines.append(f'img_gen_prompt = "{PipelexInterpreter.escape_plx_string(pipe.img_gen_prompt)}"')- if pipe.aspect_ratio:- lines.append(f'aspect_ratio = "{pipe.aspect_ratio}"')- if pipe.is_raw is not None:- lines.append(f"is_raw = {str(pipe.is_raw).lower()}")- if pipe.seed:- lines.append(f"seed = {pipe.seed}")- if pipe.nb_output:- lines.append(f"nb_output = {pipe.nb_output}")-- return "\n".join(lines)-- @staticmethod- def jinja2_pipe_to_plx_string(pipe_name: str, pipe: PipeJinja2Blueprint) -> str:- """Convert a PipeJinja2 blueprint directly to PLX section string."""- lines: list[str] = [- f"[pipe.{pipe_name}]",- f'type = "{pipe.type}"',- f'definition = "{pipe.definition}"',- ]-- # Add inputs if they exist- PipelexInterpreter.add_inputs_to_lines_if_exist(lines, pipe.inputs)-- lines.append(f'output = "{PipelexInterpreter.escape_plx_string(pipe.output)}"')-- # Add jinja2 template- if pipe.jinja2:- lines.append(f'jinja2 = "{PipelexInterpreter.escape_plx_string(pipe.jinja2)}"')- if pipe.jinja2_name:- lines.append(f'jinja2_name = "{PipelexInterpreter.escape_plx_string(pipe.jinja2_name)}"')-- return "\n".join(lines)-- @staticmethod- def condition_pipe_to_plx_string(pipe_name: str, pipe: PipeConditionBlueprint) -> str:- """Convert a PipeCondition blueprint directly to PLX section string."""- lines: list[str] = [- f"[pipe.{pipe_name}]",- f'type = "{pipe.type}"',- f'definition = "{pipe.definition}"',- ]-- # Add inputs if they exist- PipelexInterpreter.add_inputs_to_lines_if_exist(lines, pipe.inputs)-- lines.append(f'output = "{PipelexInterpreter.escape_plx_string(pipe.output)}"')-- # Add pipe_map- if pipe.pipe_map:- pipe_map_parts: list[str] = []- pipe_map_dict = pipe.pipe_map.root- for key, value in pipe_map_dict.items():- pipe_map_parts.append(f'{key} = "{value}"')- lines.append(f"pipe_map = {{ {', '.join(pipe_map_parts)} }}")-- # Add optional fields- if pipe.expression_template:- lines.append(f'expression_template = "{pipe.expression_template}"')- if pipe.expression:- lines.append(f'expression = "{pipe.expression}"')- if pipe.default_pipe_code:- lines.append(f'default_pipe_code = "{pipe.default_pipe_code}"')- if pipe.add_alias_from_expression_to:- lines.append(f'add_alias_from_expression_to = "{pipe.add_alias_from_expression_to}"')-- return "\n".join(lines)-- @staticmethod- def parallel_pipe_to_plx_string(pipe_name: str, pipe: PipeParallelBlueprint) -> str:- """Convert a PipeParallel blueprint directly to PLX section string."""- lines: list[str] = [- f"[pipe.{pipe_name}]",- f'type = "{pipe.type}"',- f'definition = "{pipe.definition}"',- ]-- # Add inputs if they exist- PipelexInterpreter.add_inputs_to_lines_if_exist(lines, pipe.inputs)-- lines.append(f'output = "{PipelexInterpreter.escape_plx_string(pipe.output)}"')-- # Add parallels array- if pipe.parallels:- lines.append("parallels = [")- for parallel in pipe.parallels:- parallel_string = PipelexInterpreter.sub_pipe_to_plx_string(parallel)- lines.append(f" {parallel_string},")- lines.append("]")-- # Add optional fields- if pipe.add_each_output is not True: # Only include if not default True- lines.append(f"add_each_output = {str(pipe.add_each_output).lower()}")- if pipe.combined_output:- lines.append(f'combined_output = "{pipe.combined_output}"')-- return "\n".join(lines)-- @staticmethod- def batch_pipe_to_plx_string(pipe_name: str, pipe: PipeBatchBlueprint) -> str:- """Convert a PipeBatch blueprint directly to PLX section string."""- lines: list[str] = [- f"[pipe.{pipe_name}]",- f'type = "{pipe.type}"',- f'definition = "{pipe.definition}"',- ]-- # Add inputs if they exist- PipelexInterpreter.add_inputs_to_lines_if_exist(lines, pipe.inputs)-- lines.append(f'output = "{PipelexInterpreter.escape_plx_string(pipe.output)}"')- lines.append(f'branch_pipe_code = "{pipe.branch_pipe_code}"')-- # Add optional fields- if pipe.input_list_name:- lines.append(f'input_list_name = "{pipe.input_list_name}"')- if pipe.input_item_name:- lines.append(f'input_item_name = "{pipe.input_item_name}"')-- return "\n".join(lines)-- @staticmethod- def serialize_concept_structure_field(field_value: Any) -> Any:- """Serialize a single concept structure field."""- if isinstance(field_value, str):- return field_value- if isinstance(field_value, ConceptStructureBlueprint):- field_data: dict[str, Any] = {- "definition": field_value.definition,- "required": field_value.required,- }- if field_value.type is not None:- field_data["type"] = field_value.type- if field_value.item_type is not None:- field_data["item_type"] = field_value.item_type- if field_value.key_type is not None:- field_data["key_type"] = field_value.key_type- if field_value.value_type is not None:- field_data["value_type"] = field_value.value_type- if field_value.choices:- field_data["choices"] = field_value.choices- if field_value.default_value is not None:- field_data["default_value"] = field_value.default_value- return field_data- return field_value-- @staticmethod- def serialize_concept_structure(- structure: str | dict[str, Any],- ) -> str | dict[str, Any]:- """Serialize concept structure field."""- if isinstance(structure, str):- return structure- result: dict[str, Any] = {}- for field_name, field_value in structure.items():- result[field_name] = PipelexInterpreter.serialize_concept_structure_field(field_value)- return result-- @staticmethod- def serialize_single_concept(- concept_blueprint: ConceptBlueprint | str,- ) -> str | dict[str, Any]:- """Serialize a single concept blueprint."""- if isinstance(concept_blueprint, str):- return concept_blueprint-- # Handle ConceptBlueprint object- if concept_blueprint.structure is not None:- # Structured concept- return {- "definition": concept_blueprint.definition,- "structure": PipelexInterpreter.serialize_concept_structure(concept_blueprint.structure),- }- if concept_blueprint.refines is not None:- # Concept with refines- return {- "definition": concept_blueprint.definition,- "refines": concept_blueprint.refines,- }- # Simple concept with just definition- return concept_blueprint.definition-- @staticmethod- def serialize_concepts(concepts: dict[str, ConceptBlueprint | str] | None) -> dict[str, Any]:- """Serialize concepts section with domain context."""- if concepts is None:- return {}-- result: dict[str, Any] = {}- for concept_name, concept_blueprint in concepts.items():- result[concept_name] = PipelexInterpreter.serialize_single_concept(concept_blueprint)- return result-- @staticmethod- def serialize_pipes(pipes: dict[str, Any]) -> dict[str, Any]:- """Serialize pipes section with domain context."""- result: dict[str, Any] = {}- for pipe_name, blueprint in pipes.items():- result[pipe_name] = PipelexInterpreter.serialize_pipe(blueprint)- return result-- @staticmethod- def serialize_pipe(blueprint: Any) -> dict[str, Any]:- if isinstance(blueprint, PipeLLMBlueprint):- return PipelexInterpreter.serialize_llm_pipe(blueprint)- elif isinstance(blueprint, PipeOcrBlueprint):- return PipelexInterpreter.serialize_ocr_pipe(blueprint)- elif isinstance(blueprint, PipeFuncBlueprint):- return PipelexInterpreter._serialize_func_pipe(blueprint)- elif isinstance(blueprint, PipeImgGenBlueprint):- return PipelexInterpreter._serialize_img_gen_pipe(blueprint)- elif isinstance(blueprint, PipeJinja2Blueprint):- return PipelexInterpreter.serialize_jinja2_pipe(blueprint)- elif isinstance(blueprint, PipeSequenceBlueprint):- return PipelexInterpreter.serialize_sequence_pipe(blueprint)- elif isinstance(blueprint, PipeConditionBlueprint):- return PipelexInterpreter._serialize_condition_pipe(blueprint)- elif isinstance(blueprint, PipeParallelBlueprint):- return PipelexInterpreter._serialize_parallel_pipe(blueprint)- elif isinstance(blueprint, PipeBatchBlueprint):- return PipelexInterpreter._serialize_batch_pipe(blueprint)- else:- msg = f"Unknown pipe blueprint type: {type(blueprint)}"- raise PipelexUnknownPipeError(msg)-- @staticmethod- def serialize_llm_pipe(pipe: PipeLLMBlueprint) -> dict[str, Any]:- """Serialize PipeLLM blueprint."""- result: dict[str, Any] = {- "type": pipe.type,- "definition": pipe.definition,- }-- # Add inputs first if they exist- if pipe.inputs:- result["inputs"] = PipelexInterpreter.serialize_inputs(pipe.inputs)-- # Then output- result["output"] = pipe.output-- # Add optional fields in expected order based on test cases- if pipe.nb_output is not None:- result["nb_output"] = pipe.nb_output- if pipe.multiple_output is not None:- result["multiple_output"] = pipe.multiple_output-- if pipe.system_prompt_template:- result["system_prompt_template"] = pipe.system_prompt_template- if pipe.system_prompt:- result["system_prompt"] = pipe.system_prompt-- if pipe.prompt_template:- result["prompt_template"] = pipe.prompt_template- if pipe.template_name:- result["template_name"] = pipe.template_name- if pipe.prompt:- result["prompt"] = pipe.prompt-- # Less common fields- if pipe.system_prompt_template_name:- result["system_prompt_template_name"] = pipe.system_prompt_template_name- if pipe.system_prompt_name:- result["system_prompt_name"] = pipe.system_prompt_name- if pipe.prompt_name:- result["prompt_name"] = pipe.prompt_name- if pipe.llm:- result["llm"] = pipe.llm- if pipe.llm_to_structure:- result["llm_to_structure"] = pipe.llm_to_structure- if pipe.structuring_method:- result["structuring_method"] = pipe.structuring_method- if pipe.prompt_template_to_structure:- result["prompt_template_to_structure"] = pipe.prompt_template_to_structure- if pipe.system_prompt_to_structure:- result["system_prompt_to_structure"] = pipe.system_prompt_to_structure-- return result-- @staticmethod- def add_common_pipe_fields(result: dict[str, Any], pipe: Any) -> None:- """Add common pipe fields (inputs) to result dict if they exist."""- if hasattr(pipe, "inputs") and pipe.inputs:- result["inputs"] = PipelexInterpreter.serialize_inputs(pipe.inputs)-- @staticmethod- def _serialize_ocr_pipe(pipe: PipeOcrBlueprint) -> dict[str, Any]:- """Serialize PipeOcr blueprint."""- result: dict[str, Any] = {- "type": pipe.type,- "definition": pipe.definition,- "output": pipe.output,- }-- PipelexInterpreter.add_common_pipe_fields(result, pipe)- return result-- @staticmethod- def _serialize_func_pipe(pipe: PipeFuncBlueprint) -> dict[str, Any]:- """Serialize PipeFunc blueprint."""- result: dict[str, Any] = {- "type": pipe.type,- "definition": pipe.definition,- "output": pipe.output,- "function_name": pipe.function_name,- }-- PipelexInterpreter.add_common_pipe_fields(result, pipe)- return result-- @staticmethod- def _serialize_img_gen_pipe(pipe: PipeImgGenBlueprint) -> dict[str, Any]:- """Serialize PipeImgGen blueprint."""- result: dict[str, Any] = {- "type": pipe.type,- "definition": pipe.definition,- "output": pipe.output,- }-- PipelexInterpreter.add_common_pipe_fields(result, pipe)-- # Add optional fields only if they have values- if pipe.img_gen_prompt:- result["img_gen_prompt"] = pipe.img_gen_prompt- if pipe.aspect_ratio:- result["aspect_ratio"] = pipe.aspect_ratio- if pipe.is_raw is not None:- result["is_raw"] = pipe.is_raw- if pipe.seed:- result["seed"] = pipe.seed- if pipe.nb_output:- result["nb_output"] = pipe.nb_output-- return result-- @staticmethod- def serialize_ocr_pipe(pipe: PipeOcrBlueprint) -> dict[str, Any]:- """Serialize a PipeOcr blueprint."""- result: dict[str, Any] = {- "type": pipe.type,- "definition": pipe.definition,- }-- # Add common pipe fields- PipelexInterpreter.add_common_pipe_fields(result, pipe)-- # Add output- result["output"] = pipe.output-- # Add required fields- result["ocr"] = pipe.ocr-- # Add optional fields- if pipe.page_images:- result["page_images"] = pipe.page_images- if pipe.page_image_captions:- result["page_image_captions"] = pipe.page_image_captions- if pipe.page_views:- result["page_views"] = pipe.page_views- if pipe.page_views_dpi:- result["page_views_dpi"] = pipe.page_views_dpi-- return result-- @staticmethod- def serialize_sequence_pipe(pipe: PipeSequenceBlueprint) -> dict[str, Any]:- """Serialize a PipeSequence blueprint."""- result: dict[str, Any] = {- "type": pipe.type,- "definition": pipe.definition,- }-- # Add common pipe fields- PipelexInterpreter.add_common_pipe_fields(result, pipe)-- # Add output- result["output"] = pipe.output-- # Add steps- if pipe.steps:- result["steps"] = [PipelexInterpreter.serialize_sub_pipe(step) for step in pipe.steps]-- return result-- @staticmethod- def serialize_jinja2_pipe(pipe: PipeJinja2Blueprint) -> dict[str, Any]:- """Serialize PipeJinja2 blueprint."""- result: dict[str, Any] = {- "type": pipe.type,- "definition": pipe.definition,- "output": pipe.output,- }-- PipelexInterpreter.add_common_pipe_fields(result, pipe)-- # Add optional fields only if they have values- if pipe.jinja2_name:- result["jinja2_name"] = pipe.jinja2_name- if pipe.jinja2:- result["jinja2"] = pipe.jinja2- if pipe.prompting_style:- result["prompting_style"] = pipe.prompting_style- result["template_category"] = pipe.template_category-- return result-- @staticmethod- def serialize_sub_pipe(sub_pipe: Any) -> dict[str, Any]:- """Serialize a sub pipe (step or parallel) blueprint."""- step_data: dict[str, Any] = {- "pipe": sub_pipe.pipe,- "result": sub_pipe.result,- }- # Only include optional fields if they have non-default values- if sub_pipe.nb_output is not None:- step_data["nb_output"] = sub_pipe.nb_output- if sub_pipe.multiple_output is not None:- step_data["multiple_output"] = sub_pipe.multiple_output- if sub_pipe.batch_over is not False: # Only include if not default False- step_data["batch_over"] = sub_pipe.batch_over- if sub_pipe.batch_as is not None:- step_data["batch_as"] = sub_pipe.batch_as- return step_data-- @staticmethod- def sub_pipe_to_plx_string(sub_pipe: Any) -> str:- """Convert a sub pipe blueprint directly to a PLX inline table string."""- parts = [f'pipe = "{sub_pipe.pipe}"', f'result = "{sub_pipe.result}"']-- # Add optional fields if they have non-default values- if sub_pipe.nb_output is not None:- parts.append(f"nb_output = {sub_pipe.nb_output}")- if sub_pipe.multiple_output is not None:- parts.append(f"multiple_output = {str(sub_pipe.multiple_output).lower()}")- if sub_pipe.batch_over is not False: # Only include if not default False- if isinstance(sub_pipe.batch_over, str):- parts.append(f'batch_over = "{sub_pipe.batch_over}"')- else:- parts.append(f"batch_over = {str(sub_pipe.batch_over).lower()}")- if sub_pipe.batch_as is not None:- parts.append(f'batch_as = "{sub_pipe.batch_as}"')-- return "{ " + ", ".join(parts) + " }"-- @staticmethod- def _serialize_sequence_pipe(pipe: PipeSequenceBlueprint) -> dict[str, Any]:- """Serialize PipeSequence blueprint."""- result: dict[str, Any] = {- "type": pipe.type,- "definition": pipe.definition,- "output": pipe.output,- }-- PipelexInterpreter.add_common_pipe_fields(result, pipe)-- if pipe.steps:- result["steps"] = [PipelexInterpreter.serialize_sub_pipe(step) for step in pipe.steps]-- return result-- @staticmethod- def sequence_pipe_to_plx_string(pipe_name: str, pipe: PipeSequenceBlueprint) -> str:- """Convert a PipeSequence blueprint directly to PLX section string."""- lines: list[str] = [- f"[pipe.{pipe_name}]",- f'type = "{pipe.type}"',- f'definition = "{pipe.definition}"',- ]-- # Add inputs first if they exist- if hasattr(pipe, "inputs"):- PipelexInterpreter.add_inputs_to_lines_if_exist(lines, pipe.inputs)-- # Add output after inputs- lines.append(f'output = "{pipe.output}"')-- # Add steps array with proper spacing- if pipe.steps:- lines.append("steps = [")- for step in pipe.steps:- step_string = PipelexInterpreter.sub_pipe_to_plx_string(step)- lines.append(f" {step_string},")- lines.append("]")-- return "\n".join(lines)-- @staticmethod- def _serialize_condition_pipe(pipe: PipeConditionBlueprint) -> dict[str, Any]:- """Serialize PipeCondition blueprint."""- result: dict[str, Any] = {- "type": pipe.type,- "definition": pipe.definition,- "output": pipe.output,- "pipe_map": pipe.pipe_map,- }-- PipelexInterpreter.add_common_pipe_fields(result, pipe)-- # Add optional fields only if they have values- if pipe.expression_template:- result["expression_template"] = pipe.expression_template- if pipe.expression:- result["expression"] = pipe.expression- if pipe.default_pipe_code:- result["default_pipe_code"] = pipe.default_pipe_code- if pipe.add_alias_from_expression_to:- result["add_alias_from_expression_to"] = pipe.add_alias_from_expression_to-- return result-- @staticmethod- def _serialize_parallel_pipe(pipe: PipeParallelBlueprint) -> dict[str, Any]:- """Serialize PipeParallel blueprint."""- result: dict[str, Any] = {- "type": pipe.type,- "definition": pipe.definition,- "output": pipe.output,- }-- PipelexInterpreter.add_common_pipe_fields(result, pipe)-- if pipe.parallels:- result["parallels"] = [PipelexInterpreter.serialize_sub_pipe(parallel) for parallel in pipe.parallels]- if pipe.add_each_output is not True: # Only include if not default True- result["add_each_output"] = pipe.add_each_output- if pipe.combined_output:- result["combined_output"] = pipe.combined_output-- return result-- @staticmethod- def _serialize_batch_pipe(pipe: PipeBatchBlueprint) -> dict[str, Any]:- """Serialize PipeBatch blueprint."""- result: dict[str, Any] = {- "type": pipe.type,- "definition": pipe.definition,- "output": pipe.output,- "branch_pipe_code": pipe.branch_pipe_code,- }-- PipelexInterpreter.add_common_pipe_fields(result, pipe)-- if pipe.input_list_name:- result["input_list_name"] = pipe.input_list_name- if pipe.input_item_name:- result["input_item_name"] = pipe.input_item_name-- return result-- @staticmethod- def serialize_input_requirement(- input_req: InputRequirementBlueprint,- ) -> dict[str, Any]:- """Serialize a single InputRequirementBlueprint to PLX format."""- result: dict[str, Any] = {"concept": input_req.concept}- if input_req.multiplicity is not None:- result["multiplicity"] = input_req.multiplicity- return result-- @staticmethod- def serialize_inputs(- inputs: Mapping[str, str | InputRequirementBlueprint],- ) -> dict[str, Any]:- """Convert InputRequirementBlueprint objects to proper PLX format for serialization."""- result: dict[str, Any] = {}- for key, value in inputs.items():- if isinstance(value, InputRequirementBlueprint):- result[key] = PipelexInterpreter.serialize_input_requirement(value)- else:- # Already a string - simple case- result[key] = str(value)- return result-- @staticmethod- def inputs_to_plx_string(- inputs: Mapping[str, str | InputRequirementBlueprint],- ) -> str:- """Convert inputs dictionary to PLX string format."""- inputs_dict = PipelexInterpreter.serialize_inputs(inputs)- inputs_parts: list[str] = []-- for key, value in inputs_dict.items():- if isinstance(value, dict):- # Nested dict (like InputRequirementBlueprint)- nested_parts: list[str] = []- nested_dict: dict[str, Any] = cast("dict[str, Any]", value)- for nested_key, nested_value in nested_dict.items():- if isinstance(nested_value, str):- nested_parts.append(f'{nested_key} = "{PipelexInterpreter.escape_plx_string(nested_value)}"')- else:- nested_parts.append(f"{nested_key} = {str(nested_value).lower()}")- inputs_parts.append(f"{key} = {{ {', '.join(nested_parts)} }}")- # Simple string value- elif isinstance(value, str):- inputs_parts.append(f'{key} = "{PipelexInterpreter.escape_plx_string(value)}"')- else:- # Fallback for any other type- inputs_parts.append(f'{key} = "{PipelexInterpreter.escape_plx_string(str(value))}"')-- return f"{{ {', '.join(inputs_parts)} }}"-- @staticmethod- def add_inputs_to_lines_if_exist(- lines: list[str],- pipe_inputs: Mapping[str, str | InputRequirementBlueprint] | None,- ) -> None:- """Add inputs line to PLX lines if inputs exist."""- if pipe_inputs:- inputs_dict = PipelexInterpreter.serialize_inputs(pipe_inputs)- inputs_parts: list[str] = []- for key, value in inputs_dict.items():- if isinstance(value, dict):- nested_parts: list[str] = []- nested_dict: dict[str, Any] = cast("dict[str, Any]", value)- for nested_key, nested_value in nested_dict.items():- if isinstance(nested_value, str):- nested_parts.append(f'{nested_key} = "{PipelexInterpreter.escape_plx_string(nested_value)}"')- else:- nested_parts.append(f"{nested_key} = {str(nested_value).lower()}")- inputs_parts.append(f"{key} = {{ {', '.join(nested_parts)} }}")- elif isinstance(value, str):- inputs_parts.append(f'{key} = "{PipelexInterpreter.escape_plx_string(value)}"')- else:- inputs_parts.append(f'{key} = "{PipelexInterpreter.escape_plx_string(str(value))}"')- lines.append(f"inputs = {{ {', '.join(inputs_parts)} }}")diff --git a/pipelex/core/pipes/pipe_blueprint.py b/pipelex/core/pipes/pipe_blueprint.pyindex d684c82ed..07e1c5fe8 100644--- a/pipelex/core/pipes/pipe_blueprint.py+++ b/pipelex/core/pipes/pipe_blueprint.py@@ -37,6 +37,7 @@ class AllowedPipeTypes(StrEnum):class PipeBlueprint(BaseModel):+ source: str | None = Nonecategory: Anytype: Any # TODO: Find a better way to handle this.definition: str | None = Nonediff --git a/pipelex/core/pipes/pipe_run_params.py b/pipelex/core/pipes/pipe_run_params.pyindex 2a71d40d4..34c0ac93e 100644--- a/pipelex/core/pipes/pipe_run_params.py+++ b/pipelex/core/pipes/pipe_run_params.py@@ -6,6 +6,7 @@ from pydantic import BaseModel, Field, field_validatorfrom pipelex import logfrom pipelex.core.memory.working_memory import BATCH_ITEM_STUFF_NAME, MAIN_STUFF_NAME+from pipelex.exceptions import BatchParamsErrorfrom pipelex.types import Self, StrEnum@@ -177,22 +178,20 @@ class BatchParams(BaseModel):@classmethoddef make_optional_batch_params(cls,- input_list_name: bool | str,- input_item_name: str | None = None,+ input_list_name: str | None,+ input_item_name: str | None,) -> BatchParams | None:- the_batch_params: BatchParams | None = None- if input_list_name or input_item_name:- input_list_stuff_name: str- if isinstance(input_list_name, str):- input_list_stuff_name = input_list_name- else:- input_list_stuff_name = MAIN_STUFF_NAME+ if input_list_name:input_item_stuff_name = input_item_name or BATCH_ITEM_STUFF_NAME- the_batch_params = BatchParams(- input_list_stuff_name=input_list_stuff_name,+ return BatchParams(+ input_list_stuff_name=input_list_name,input_item_stuff_name=input_item_stuff_name,)- return the_batch_params+ elif input_item_name:+ msg = "input_item_name is required when input_list_name is not provided"+ raise BatchParamsError(msg)+ else:+ return None@classmethoddef make_default(cls) -> BatchParams:diff --git a/pipelex/exceptions.py b/pipelex/exceptions.pyindex 1b83b4d3e..c8dfbdb45 100644--- a/pipelex/exceptions.py+++ b/pipelex/exceptions.py@@ -104,19 +104,15 @@ class ClientAuthenticationError(PipelexError):pass-class DomainDefinitionError(PipelexError):- pass--class ConceptLibraryConceptNotFoundError(PipelexError):pass-class ConceptFactoryError(PipelexError):+class LibraryError(PipelexError):pass-class LibraryError(PipelexError):+class LibraryLoadingError(LibraryError):pass@@ -144,10 +140,48 @@ class LibraryParsingError(LibraryError):pass-class PipeDefinitionError(PipelexError):+# TODO: add details from all cases raising this error+class DomainDefinitionError(PipelexError):pass+# TODO: add details from all cases raising this error+class ConceptDefinitionError(PipelexError):+ pass+++# TODO: add details from all cases raising this error+class PipeDefinitionError(PipelexError):+ def __init__(+ self,+ message: str,+ domain: str | None = None,+ pipe_code: str | None = None,+ description: str | None = None,+ source: str | None = None,+ ):+ self.domain = domain+ self.pipe_code = pipe_code+ self.description = description+ self.source = source+ message = message + " • " + self.pipe_details()+ super().__init__(message)++ def pipe_details(self) -> str:+ if not self.domain and not self.pipe_code and not self.description and not self.source:+ return "No pipe details provided"+ details = "Pipe details:"+ if self.domain:+ details += f" • domain='{self.domain}'"+ if self.pipe_code:+ details += f" • pipe='{self.pipe_code}'"+ if self.description:+ details += f" • description='{self.description}'"+ if self.source:+ details += f" • source='{self.source}'"+ return details++class UnexpectedPipeDefinitionError(PipeDefinitionError):pass@@ -187,6 +221,10 @@ class DryRunError(PipeExecutionError):super().__init__(message)+class BatchParamsError(PipelexError):+ pass++class PipeConditionError(PipelexError):pass@@ -239,6 +277,10 @@ class ConceptCodeError(ConceptError):pass+class ConceptRefineError(ConceptError):+ pass++class PipelineManagerNotFoundError(PipelexError):passdiff --git a/tests/unit/pipelex/core/concepts/concept_factory.py/__init__.py b/pipelex/language/__init__.pysimilarity index 100%rename from tests/unit/pipelex/core/concepts/concept_factory.py/__init__.pyrename to pipelex/language/__init__.pydiff --git a/pipelex/language/plx_config.py b/pipelex/language/plx_config.pynew file mode 100644index 000000000..f5a8f65ea--- /dev/null+++ b/pipelex/language/plx_config.py@@ -0,0 +1,28 @@+from pipelex.tools.config.config_model import ConfigModel+++class PlxConfigStrings(ConfigModel):+ prefer_literal: bool+ force_multiline: bool+ length_limit_to_multiline: int+ ensure_trailing_newline: bool+ ensure_leading_blank_line: bool+++class PlxConfigInlineTables(ConfigModel):+ spaces_inside_curly_braces: bool+++class PlxConfigForConcepts(ConfigModel):+ structure_field_ordering: list[str]+++class PlxConfigForPipes(ConfigModel):+ field_ordering: list[str]+++class PlxConfig(ConfigModel):+ strings: PlxConfigStrings+ inline_tables: PlxConfigInlineTables+ concepts: PlxConfigForConcepts+ pipes: PlxConfigForPipesdiff --git a/pipelex/language/plx_factory.py b/pipelex/language/plx_factory.pynew file mode 100644index 000000000..294985259--- /dev/null+++ b/pipelex/language/plx_factory.py@@ -0,0 +1,305 @@+from __future__ import annotations++from typing import TYPE_CHECKING, Any, Mapping, cast++import tomlkit+from tomlkit import array, document, inline_table, table+from tomlkit import string as tomlkit_string++from pipelex import log+from pipelex.config import PlxConfig, get_config+from pipelex.tools.misc.json_utils import remove_none_values_from_dict+from pipelex.types import StrEnum++if TYPE_CHECKING:+ from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint+++class SectionKey(StrEnum):+ PIPE = "pipe"+ CONCEPT = "concept"+++CONCEPT_STRUCTURE_FIELD_KEY = "structure"+++class PlxFactory:+ @classmethod+ def _plx_config(cls) -> PlxConfig:+ return get_config().pipelex.plx_config++ @classmethod+ def format_tomlkit_string(cls, text: str) -> Any: # Can't type this because of tomlkit+ r"""Build a tomlkit string node.+ - If `force_multiline` or the text contains '\\n', we emit a triple-quoted multiline string.+ - When multiline, `ensure_trailing_newline` puts the closing quotes on their own line.+ - When multiline, `ensure_leading_blank_line` inserts a real blank line at the start of the string.+ """+ strings_config = cls._plx_config().strings++ needs_multiline = strings_config.force_multiline or ("\n" in text) or len(text) > strings_config.length_limit_to_multiline+ normalized = text++ if needs_multiline:+ if strings_config.ensure_leading_blank_line and not normalized.startswith("\n"):+ normalized = "\n" + normalized+ if strings_config.ensure_trailing_newline and not normalized.endswith("\n"):+ normalized = normalized + "\n"++ use_literal = strings_config.prefer_literal and ("'''" not in normalized)+ return tomlkit_string(normalized, multiline=needs_multiline, literal=use_literal)++ @classmethod+ def convert_dicts_to_inline_tables(cls, value: Any, field_ordering: list[str] | None = None) -> Any: # Can't type this because of tomlkit+ """Recursively convert Python values; dicts -> inline tables; lists kept as arrays."""+ if isinstance(value, Mapping):+ value = cast("Mapping[str, Any]", value)+ inline_table_obj = inline_table()+ if field_ordering:+ for key in field_ordering:+ if key in value:+ inline_table_obj[key] = cls.convert_dicts_to_inline_tables(value=value[key])+ else:+ for key, value_item in value.items():+ inline_table_obj[key] = cls.convert_dicts_to_inline_tables(value=value_item)+ return inline_table_obj++ elif isinstance(value, list):+ value = cast("list[Any]", value)+ array_obj = array()+ array_obj.multiline(True) # set to False for single-line arrays+ for element in value:+ if isinstance(element, Mapping):+ element = cast("Mapping[str, Any]", element)+ inline_element = inline_table()+ for inner_key, inner_value in element.items():+ inline_element[inner_key] = cls.convert_dicts_to_inline_tables(value=inner_value)+ array_obj.append(inline_element) # pyright: ignore[reportUnknownMemberType]+ else:+ array_obj.append(cls.convert_dicts_to_inline_tables(value=element)) # pyright: ignore[reportUnknownMemberType]+ return array_obj++ elif isinstance(value, str):+ return cls.format_tomlkit_string(text=value)+ else:+ return value++ @classmethod+ def convert_mapping_to_table(+ cls, mapping: Mapping[str, Any], field_ordering: list[str] | None = None+ ) -> Any: # Can't type this because of tomlkit+ """Convert a mapping into a TOML Table where any nested mappings (third level+)+ are converted to inline tables.++ This creates a second-level standard table, and only uses inline tables for+ third level and deeper structures.+ """+ tbl = table()++ # If field ordering is provided, add fields in the specified order first+ if field_ordering:+ for field_key in field_ordering:+ if field_key in mapping and field_key != "category": # Skip category field+ field_value = mapping[field_key]+ if isinstance(field_value, Mapping):+ # Third-level mapping -> inline table+ tbl.add(field_key, cls.convert_dicts_to_inline_tables(field_value))+ else:+ tbl.add(field_key, cls.convert_dicts_to_inline_tables(field_value))++ # Add any remaining fields not in the ordering+ for field_key, field_value in mapping.items():+ if field_key not in field_ordering and field_key != "category":+ if isinstance(field_value, Mapping):+ # Third-level mapping -> inline table+ tbl.add(field_key, cls.convert_dicts_to_inline_tables(field_value))+ else:+ tbl.add(field_key, cls.convert_dicts_to_inline_tables(field_value))+ else:+ # No field ordering provided, use original logic+ for field_key, field_value in mapping.items():+ # Skip the category field as it's not needed in PLX output+ if field_key == "category":+ continue++ if isinstance(field_value, Mapping):+ # Third-level mapping -> inline table+ tbl.add(field_key, cls.convert_dicts_to_inline_tables(field_value))+ else:+ tbl.add(field_key, cls.convert_dicts_to_inline_tables(field_value))+ return tbl++ # TODO: replace this by a proper toml formatter utility+ @classmethod+ def add_spaces_to_inline_tables(cls, toml_string: str) -> str:+ """Add spaces inside inline table curly braces.++ Converts {key = value} to { key = value }.+ Only adds spaces if they're not already present.+ Handles nested inline tables properly.+ """++ def find_and_replace_inline_tables(text: str) -> str:+ """Find inline tables using proper brace matching and add spaces."""+ result = ""+ char_index = 0++ while char_index < len(text):+ if text[char_index] == "{":+ # Check if this is a Jinja2 template (double braces)+ if char_index + 1 < len(text) and text[char_index + 1] == "{":+ # This is a Jinja2 template, find the closing }}+ jinja_end = text.find("}}", char_index + 2)+ if jinja_end != -1:+ # Add the entire Jinja2 template as-is+ result += text[char_index : jinja_end + 2]+ char_index = jinja_end + 2+ continue++ # Found start of inline table, find the matching closing brace+ brace_count = 1+ start = char_index+ char_index += 1++ while char_index < len(text) and brace_count > 0:+ if text[char_index] == "{":+ brace_count += 1+ elif text[char_index] == "}":+ brace_count -= 1+ char_index += 1++ if brace_count == 0:+ # Found complete inline table+ content = text[start + 1 : char_index - 1] # Content between braces++ # Recursively process nested inline tables first+ content = find_and_replace_inline_tables(content)++ # Add spaces if not already present+ if content.startswith(" ") and content.endswith(" "):+ result += "{" + content + "}"+ elif content.startswith(" "):+ result += "{" + content + " }"+ elif content.endswith(" "):+ result += "{ " + content + "}"+ else:+ result += "{ " + content + " }"+ else:+ # Unmatched brace, add as-is+ result += text[start:char_index]+ else:+ result += text[char_index]+ char_index += 1++ return result++ return find_and_replace_inline_tables(toml_string)++ @classmethod+ def make_table_obj_for_pipe(cls, section_value: Mapping[str, Any]) -> Any:+ """Make a table object for a pipe section."""+ log.debug("******** Making table object for pipe section ********")+ table_obj = table()+ for field_key, field_value in section_value.items():+ log.debug(f"------ Field {field_key} is a {type(field_value)}")+ if not isinstance(field_value, Mapping):+ log.debug(f"Field is not a mapping: key = {field_key}, value = {field_value}")+ table_obj.add(field_key, cls.convert_dicts_to_inline_tables(field_value))+ continue+ log.debug(f"Field is a mapping: key = {field_key}, value = {field_value}")+ field_value = cast("Mapping[str, Any]", field_value)+ table_obj.add(field_key, cls.convert_mapping_to_table(field_value, field_ordering=cls._plx_config().pipes.field_ordering))+ return table_obj++ @classmethod+ def make_table_obj_for_concept(cls, section_value: Mapping[str, Any]) -> Any:+ """Make a table object for a concept section."""+ log.debug("******** Making table object for concept section ********")+ table_obj = table()+ for concept_key, concept_value in section_value.items():+ if isinstance(concept_value, str):+ log.debug(f"Concept '{concept_key}' is a string: {concept_value}")+ table_obj.add(concept_key, concept_value)+ continue+ if not isinstance(concept_value, Mapping):+ msg = f"Concept field value is not a mapping: key = {concept_key}, value = {concept_value}"+ raise TypeError(msg)+ log.debug(f"Concept '{concept_key}' is a mapping: {concept_value}")+ concept_value = cast("Mapping[str, Any]", concept_value)+ concept_table_obj = table()+ for concept_field_key, concept_field_value in concept_value.items():+ if concept_field_key == CONCEPT_STRUCTURE_FIELD_KEY:+ if isinstance(concept_field_value, str):+ log.debug(f"Structure for concept '{concept_key}' is a string: {concept_field_value}")+ concept_table_obj.add("structure", concept_field_value)+ continue+ if not isinstance(concept_field_value, Mapping):+ msg = f"Structure field value is not a mapping: key = {concept_field_key}, value = {concept_field_value}"+ raise TypeError(msg)+ log.debug(f"Structure for concept '{concept_key}' is a mapping: {concept_field_value}")+ structure_value = cast("Mapping[str, Any]", concept_field_value)+ structure_table_obj = table()+ for structure_field_key, structure_field_value in structure_value.items():+ if isinstance(structure_field_value, str):+ log.debug(f"Structure '{structure_field_key}' is a string: {structure_field_value}")+ structure_table_obj.add(structure_field_key, structure_field_value)+ continue+ if not isinstance(structure_field_value, Mapping):+ msg = (+ f"Structure field value is neither a string nor a mapping: "+ f"key = {structure_field_key}, value = {structure_field_value}"+ )+ raise TypeError(msg)+ log.debug(f"Structure for '{concept_key}' is a mapping: {structure_field_value}")+ structure_table_obj.add(+ structure_field_key,+ cls.convert_dicts_to_inline_tables(+ value=structure_field_value, field_ordering=cls._plx_config().concepts.structure_field_ordering+ ),+ )+ concept_table_obj.add("structure", structure_table_obj)+ else:+ # sub_table = _convert_mapping_to_table(concept_field_value)+ log.debug(f"{concept_key}/'{concept_field_key}' is inline: {concept_field_value}")+ concept_table_obj.add(concept_field_key, cls.convert_dicts_to_inline_tables(concept_field_value))+ table_obj.add(concept_key, concept_table_obj)+ return table_obj++ @classmethod+ def dict_to_plx_styled_toml(cls, data: Mapping[str, Any]) -> str:+ """Top-level keys become tables; second-level mappings become tables; inline tables start at third level."""+ log.debug("=" * 100)+ data = remove_none_values_from_dict(data=data)+ document_root = document()+ for root_key, root_value in data.items():+ if not isinstance(root_value, Mapping):+ log.debug(f"Root root_key is not a mapping: key = {root_key}, value = {root_value}")+ document_root.add(root_key, cls.convert_dicts_to_inline_tables(root_value))+ continue++ # It's a mapping, therefore it's a section+ log.debug(f"Root {root_key} is a section -------------------")++ section_key = SectionKey(root_key)+ section_value = cast("Mapping[str, Any]", root_value)+ # Skip empty mappings (empty concept and pipe sections)+ if not section_value:+ log.debug(f"Section {section_key} is empty, skipping")+ continue+ match section_key:+ case SectionKey.PIPE:+ table_obj_for_pipe = cls.make_table_obj_for_pipe(section_value)+ document_root.add(section_key, table_obj_for_pipe)+ case SectionKey.CONCEPT:+ table_obj_for_concept = cls.make_table_obj_for_concept(section_value)+ document_root.add(section_key, table_obj_for_concept)++ toml_output = tomlkit.dumps(document_root) # pyright: ignore[reportUnknownMemberType]+ if cls._plx_config().inline_tables.spaces_inside_curly_braces:+ return cls.add_spaces_to_inline_tables(toml_output)+ return toml_output++ @classmethod+ def make_plx_content(cls, blueprint: PipelexBundleBlueprint) -> str:+ blueprint_dict = blueprint.model_dump(serialize_as_any=True)+ return cls.dict_to_plx_styled_toml(data=blueprint_dict)diff --git a/pipelex/libraries/library_config.py b/pipelex/libraries/library_config.pyindex bfb5650b4..65e176f9b 100644--- a/pipelex/libraries/library_config.py+++ b/pipelex/libraries/library_config.py@@ -1,3 +1,4 @@+from pathlib import Pathfrom typing import ClassVarfrom pipelex.tools.config.config_model import ConfigModel@@ -27,8 +28,8 @@ class LibraryConfig(ConfigModel):return "tests/test_pipelines"@property- def failing_pipelines_file_paths(self) -> list[str]:- return ["tests/test_pipelines/failing_pipelines.plx"]+ def failing_pipelines_file_paths(self) -> set[Path]:+ return {Path("tests/test_pipelines/failing_pipelines.plx")}def get_templates_paths(self) -> list[str]:return [str(path) for path in find_files_in_dir(dir_path=self.templates_dir_path, pattern="*.toml", is_recursive=True)]diff --git a/pipelex/libraries/library_manager.py b/pipelex/libraries/library_manager.pyindex 164fb76b5..5bdf0a9fa 100644--- a/pipelex/libraries/library_manager.py+++ b/pipelex/libraries/library_manager.py@@ -21,9 +21,12 @@ from pipelex.core.pipes.pipe_abstract import PipeAbstractfrom pipelex.core.pipes.pipe_factory import PipeFactoryfrom pipelex.core.pipes.pipe_library import PipeLibraryfrom pipelex.exceptions import (+ ConceptDefinitionError,ConceptLibraryError,+ DomainDefinitionError,LibraryError,- PipelexSetupError,+ LibraryLoadingError,+ PipeDefinitionError,PipeLibraryError,)from pipelex.libraries.library_config import LibraryConfig@@ -140,29 +143,31 @@ class LibraryManager(LibraryManagerAbstract):return all_plx_paths- @override- def load_from_file(self, plx_path: Path) -> None:- """Load a single file - this method is kept for compatibility."""- if not PipelexInterpreter.is_pipelex_file(plx_path):- msg = f"File is not a valid Pipelex PLX file: {plx_path}"- raise LibraryError(msg)-- blueprint = PipelexInterpreter(file_path=plx_path).make_pipelex_bundle_blueprint()- self.load_from_blueprint(blueprint)-@overridedef load_from_blueprint(self, blueprint: PipelexBundleBlueprint) -> list[PipeAbstract]:"""Load a blueprint."""# Create and load domain- domain = self._load_domain_from_blueprint(blueprint)+ try:+ domain = self._load_domain_from_blueprint(blueprint)+ except DomainDefinitionError as exc:+ msg = f"Could not load domain from PLX blueprint at '{blueprint.source}', domain code: '{blueprint.domain}': {exc}"+ raise LibraryLoadingError(msg) from excself.domain_library.add_domain(domain=domain)# Create and load concepts- concepts = self._load_concepts_from_blueprint(blueprint)+ try:+ concepts = self._load_concepts_from_blueprint(blueprint)+ except ConceptDefinitionError as exc:+ msg = f"Could not load concepts from PLX blueprint at '{blueprint.source}', domain code: '{blueprint.domain}': {exc}"+ raise LibraryLoadingError(msg) from excself.concept_library.add_concepts(concepts=concepts)# Create and load pipes- pipes = self._load_pipes_from_blueprint(blueprint)+ try:+ pipes = self._load_pipes_from_blueprint(blueprint)+ except PipeDefinitionError as exc:+ msg = f"Could not load pipes from PLX blueprint at '{blueprint.source}', domain code: '{blueprint.domain}': {exc}"+ raise LibraryLoadingError(msg) from excself.pipe_library.add_pipes(pipes=pipes)return pipes@@ -194,19 +199,23 @@ class LibraryManager(LibraryManagerAbstract):)def _load_concepts_from_blueprint(self, blueprint: PipelexBundleBlueprint) -> list[Concept]:- concepts: list[Concept] = []+ if blueprint.concept is None:+ return []- if blueprint.concept is not None:- for concept_code, concept_blueprint_or_str in blueprint.concept.items():- concept = ConceptFactory.make_from_blueprint(- domain=blueprint.domain,- concept_code=concept_code,- concept_codes_from_the_same_domain=list(blueprint.concept.keys()),- blueprint=ConceptBlueprint(definition=concept_blueprint_or_str)- if isinstance(concept_blueprint_or_str, str)- else concept_blueprint_or_str,- )- concepts.append(concept)+ concepts: list[Concept] = []+ for concept_code, concept_blueprint_or_str in blueprint.concept.items():+ concept_blueprint: ConceptBlueprint+ if isinstance(concept_blueprint_or_str, str):+ concept_blueprint = ConceptBlueprint(definition=concept_blueprint_or_str)+ else:+ concept_blueprint = concept_blueprint_or_str+ concept = ConceptFactory.make_from_blueprint(+ domain=blueprint.domain,+ concept_code=concept_code,+ concept_codes_from_the_same_domain=list(blueprint.concept.keys()),+ blueprint=concept_blueprint,+ )+ concepts.append(concept)return conceptsdef _load_pipes_from_blueprint(self, blueprint: PipelexBundleBlueprint) -> list[PipeAbstract]:@@ -228,57 +237,69 @@ class LibraryManager(LibraryManagerAbstract):library_dirs: list[Path] | None = None,library_file_paths: list[Path] | None = None,) -> None:- dirs_to_use: list[Path] = self._get_pipeline_library_dirs()- all_plx_paths: list[Path] = self._get_pipelex_plx_files_from_dirs(dirs_to_use)-- # Remove failing pipelines from the list- failing_pipelines_file_paths = get_config().pipelex.library_config.failing_pipelines_file_paths- failing_paths_set = {Path(fp) for fp in failing_pipelines_file_paths}- all_plx_paths = [path for path in all_plx_paths if path not in failing_paths_set]+ dirs_to_use = library_dirs or self._get_pipeline_library_dirs()- if library_dirs is not None:- dirs_to_use = library_dirs+ valid_plx_paths: list[Path]+ if library_file_paths:+ valid_plx_paths = library_file_paths+ else:+ all_plx_paths: list[Path] = self._get_pipelex_plx_files_from_dirs(dirs_to_use)+ # Remove failing pipelines from the list+ failing_pipelines_file_paths = get_config().pipelex.library_config.failing_pipelines_file_paths+ valid_plx_paths = [path for path in all_plx_paths if path not in failing_pipelines_file_paths]# Register classes in the directoriesfor library_dir in dirs_to_use:ClassRegistryUtils.register_classes_in_folder(folder_path=str(library_dir))FuncRegistryUtils.register_funcs_in_folder(folder_path=str(library_dir))- if library_file_paths is not None:- all_plx_paths = library_file_paths-# Parse all blueprints firstblueprints: list[PipelexBundleBlueprint] = []- for plx_file_path in all_plx_paths:+ for plx_file_path in valid_plx_paths:try:blueprint = PipelexInterpreter(file_path=plx_file_path).make_pipelex_bundle_blueprint()except FileNotFoundError as exc:msg = f"Could not find PLX blueprint at '{plx_file_path}'"- raise PipelexSetupError(msg) from exc+ raise LibraryLoadingError(msg) from excexcept ValidationError as exc:formatted_error_msg = format_pydantic_validation_error(exc)msg = f"Could not load PLX blueprint from '{plx_file_path}' because of: {formatted_error_msg}"- raise PipelexSetupError(msg) from exc+ raise LibraryLoadingError(msg) from exc+ except PipeDefinitionError as exc:+ msg = f"Could not load PLX blueprint from '{plx_file_path}': {exc}"+ raise LibraryLoadingError(msg) from exc+ blueprint.source = str(plx_file_path)blueprints.append(blueprint)# Load all domains firstall_domains: list[Domain] = []for blueprint in blueprints:- domain = self._load_domain_from_blueprint(blueprint)+ try:+ domain = self._load_domain_from_blueprint(blueprint)+ except DomainDefinitionError as exc:+ msg = f"Could not load domain from PLX blueprint at '{blueprint.source}', domain code: '{blueprint.domain}': {exc}"+ raise LibraryLoadingError(msg) from excall_domains.append(domain)- for domain in all_domains:- self.domain_library.add_domain(domain=domain)+ self.domain_library.add_domains(domains=all_domains)# Load all concepts secondall_concepts: list[Concept] = []for blueprint in blueprints:- concepts = self._load_concepts_from_blueprint(blueprint)+ try:+ concepts = self._load_concepts_from_blueprint(blueprint)+ except ConceptDefinitionError as exc:+ msg = f"Could not load concepts from PLX blueprint at '{blueprint.source}', domain code: '{blueprint.domain}': {exc}"+ raise LibraryLoadingError(msg) from excall_concepts.extend(concepts)self.concept_library.add_concepts(concepts=all_concepts)# Load all pipes thirdall_pipes: list[PipeAbstract] = []for blueprint in blueprints:- pipes = self._load_pipes_from_blueprint(blueprint)+ try:+ pipes = self._load_pipes_from_blueprint(blueprint)+ except PipeDefinitionError as exc:+ msg = f"Could not load pipes from PLX blueprint at '{blueprint.source}', domain code: '{blueprint.domain}': {exc}"+ raise LibraryLoadingError(msg) from excall_pipes.extend(pipes)self.pipe_library.add_pipes(pipes=all_pipes)diff --git a/pipelex/libraries/library_manager_abstract.py b/pipelex/libraries/library_manager_abstract.pyindex 3f93d87ed..9c23a9f09 100644--- a/pipelex/libraries/library_manager_abstract.py+++ b/pipelex/libraries/library_manager_abstract.py@@ -26,10 +26,6 @@ class LibraryManagerAbstract(ABC):def load_libraries(self, library_dirs: list[Path] | None = None, library_file_paths: list[Path] | None = None) -> None:pass- @abstractmethod- def load_from_file(self, plx_path: Path) -> None:- pass-@abstractmethoddef load_from_blueprint(self, blueprint: PipelexBundleBlueprint) -> list[PipeAbstract]:passdiff --git a/pipelex/libraries/pipelines/builder/builder.plx b/pipelex/libraries/pipelines/builder/builder.plxindex d610ba2bf..2289a482e 100644--- a/pipelex/libraries/pipelines/builder/builder.plx+++ b/pipelex/libraries/pipelines/builder/builder.plx@@ -102,6 +102,7 @@ parallels = [{ pipe = "draft_to_conceptspecs_text", result = "concept_spec_drafts_text" },{ pipe = "draft_to_pipesignatures_text", result = "pipe_signatures_text" },]+add_each_output = true[pipe.draft_to_conceptspecs_text]type = "PipeLLM"diff --git a/pipelex/libraries/pipelines/builder/pipe/pipe_parallel_spec.py b/pipelex/libraries/pipelines/builder/pipe/pipe_parallel_spec.pyindex 2afdf8b6e..f3e481410 100644--- a/pipelex/libraries/pipelines/builder/pipe/pipe_parallel_spec.py+++ b/pipelex/libraries/pipelines/builder/pipe/pipe_parallel_spec.py@@ -1,12 +1,14 @@from typing import Literal-from pydantic import Field, field_validator+from pydantic import Field, field_validator, model_validatorfrom typing_extensions import override+from pipelex.exceptions import PipeDefinitionErrorfrom pipelex.libraries.pipelines.builder.concept.concept_spec import ConceptBlueprintfrom pipelex.libraries.pipelines.builder.pipe.pipe_signature import PipeSpecfrom pipelex.libraries.pipelines.builder.pipe.sub_pipe_spec import SubPipeSpecfrom pipelex.pipe_controllers.parallel.pipe_parallel_blueprint import PipeParallelBlueprint+from pipelex.types import Selfclass PipeParallelSpec(PipeSpec):@@ -38,7 +40,7 @@ class PipeParallelSpec(PipeSpec):category: Literal["PipeController"] = "PipeController"the_pipe_code: str = Field(description="Pipe code. Must be snake_case.")parallels: list[SubPipeSpec]- add_each_output: bool = True+ add_each_output: bool = Falsecombined_output: str | None = None@field_validator("combined_output", mode="before")@@ -48,6 +50,16 @@ class PipeParallelSpec(PipeSpec):ConceptBlueprint.validate_concept_string_or_code(concept_string_or_code=combined_output)return combined_output+ @model_validator(mode="after")+ def validate_output_options(self) -> Self:+ if not self.add_each_output and not self.combined_output:+ msg = (+ "PipeParallel requires either add_each_output to be True or combined_output to be set, "+ "or both, otherwise the pipe won't output anything"+ )+ raise PipeDefinitionError(msg)+ return self+@overridedef to_blueprint(self) -> PipeParallelBlueprint:base_blueprint = super().to_blueprint()diff --git a/pipelex/libraries/pipelines/builder/pipe/sub_pipe_spec.py b/pipelex/libraries/pipelines/builder/pipe/sub_pipe_spec.pyindex 42423bd3b..ce0cd7ab1 100644--- a/pipelex/libraries/pipelines/builder/pipe/sub_pipe_spec.py+++ b/pipelex/libraries/pipelines/builder/pipe/sub_pipe_spec.py@@ -42,7 +42,7 @@ class SubPipeSpec(StructuredContent):result: str | None = Nonenb_output: int | None = Nonemultiple_output: bool | None = None- batch_over: bool | str = False+ batch_over: str | None = Nonebatch_as: str | None = None@model_validator(mode="after")@@ -54,14 +54,11 @@ class SubPipeSpec(StructuredContent):@model_validator(mode="after")def validate_batch_params(self) -> Self:- batch_over_is_specified = self.batch_over is not False and self.batch_over != ""- batch_as_is_specified = self.batch_as is not None and self.batch_as != ""-- if batch_over_is_specified and not batch_as_is_specified:+ if self.batch_over and not self.batch_as:msg = f"In pipe '{self.the_pipe_code}': When 'batch_over' is specified, 'batch_as' must also be provided"raise PipeDefinitionError(msg)- if batch_as_is_specified and not batch_over_is_specified:+ if self.batch_as and not self.batch_over:msg = f"In pipe '{self.the_pipe_code}': When 'batch_as' is specified, 'batch_over' must also be provided"raise PipeDefinitionError(msg)diff --git a/pipelex/pipe_controllers/batch/pipe_batch_factory.py b/pipelex/pipe_controllers/batch/pipe_batch_factory.pyindex 3eeb04bb2..469bccaea 100644--- a/pipelex/pipe_controllers/batch/pipe_batch_factory.py+++ b/pipelex/pipe_controllers/batch/pipe_batch_factory.py@@ -41,7 +41,7 @@ class PipeBatchFactory(PipeFactoryProtocol[PipeBatchBlueprint, PipeBatch]):),branch_pipe_code=blueprint.branch_pipe_code,batch_params=BatchParams.make_optional_batch_params(- input_list_name=blueprint.input_list_name or False,+ input_list_name=blueprint.input_list_name,input_item_name=blueprint.input_item_name,),)diff --git a/pipelex/pipe_controllers/parallel/pipe_parallel_blueprint.py b/pipelex/pipe_controllers/parallel/pipe_parallel_blueprint.pyindex f9f5599a0..9351f0814 100644--- a/pipelex/pipe_controllers/parallel/pipe_parallel_blueprint.py+++ b/pipelex/pipe_controllers/parallel/pipe_parallel_blueprint.py@@ -1,17 +1,19 @@from typing import Literal-from pydantic import field_validator+from pydantic import field_validator, model_validatorfrom pipelex.core.concepts.concept_blueprint import ConceptBlueprintfrom pipelex.core.pipes.pipe_blueprint import PipeBlueprint+from pipelex.exceptions import PipeDefinitionErrorfrom pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprint+from pipelex.types import Selfclass PipeParallelBlueprint(PipeBlueprint):type: Literal["PipeParallel"] = "PipeParallel"category: Literal["PipeController"] = "PipeController"parallels: list[SubPipeBlueprint]- add_each_output: bool = True+ add_each_output: bool = Falsecombined_output: str | None = None@field_validator("combined_output", mode="before")@@ -20,3 +22,13 @@ class PipeParallelBlueprint(PipeBlueprint):if combined_output:ConceptBlueprint.validate_concept_string_or_code(concept_string_or_code=combined_output)return combined_output++ @model_validator(mode="after")+ def validate_output_options(self) -> Self:+ if not self.add_each_output and not self.combined_output:+ msg = (+ "PipeParallel requires either add_each_output to be True or combined_output to be set, "+ "or both, otherwise the pipe won't output anything"+ )+ raise PipeDefinitionError(message=msg, description=self.definition, source=self.source)+ return selfdiff --git a/pipelex/pipe_controllers/parallel/pipe_parallel_factory.py b/pipelex/pipe_controllers/parallel/pipe_parallel_factory.pyindex af322d650..b43fa0423 100644--- a/pipelex/pipe_controllers/parallel/pipe_parallel_factory.py+++ b/pipelex/pipe_controllers/parallel/pipe_parallel_factory.py@@ -6,7 +6,7 @@ from pipelex.core.concepts.concept_factory import ConceptFactoryfrom pipelex.core.pipes.pipe_factory import PipeFactoryProtocolfrom pipelex.core.pipes.pipe_input_factory import PipeInputSpecFactoryfrom pipelex.exceptions import PipeDefinitionError-from pipelex.hub import get_concept_provider+from pipelex.hub import get_required_conceptfrom pipelex.pipe_controllers.parallel.pipe_parallel import PipeParallelfrom pipelex.pipe_controllers.parallel.pipe_parallel_blueprint import PipeParallelBlueprintfrom pipelex.pipe_controllers.sub_pipe_factory import SubPipeFactory@@ -33,7 +33,10 @@ class PipeParallelFactory(PipeFactoryProtocol[PipeParallelBlueprint, PipeParallesub_pipe = SubPipeFactory.make_from_blueprint(sub_pipe_blueprint, concept_codes_from_the_same_domain=concept_codes_from_the_same_domain)parallel_sub_pipes.append(sub_pipe)if not blueprint.add_each_output and not blueprint.combined_output:- msg = "PipeParallel requires either add_each_output or combined_output to be set"+ msg = (+ "Unexpected error:PipeParallel requires either add_each_output to be True or combined_output to be set, "+ "or both, otherwise the pipe won't output anything"+ )raise PipeDefinitionError(msg)if blueprint.combined_output:@@ -42,7 +45,7 @@ class PipeParallelFactory(PipeFactoryProtocol[PipeParallelBlueprint, PipeParalleconcept_string_or_code=blueprint.output,concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,)- combined_output = get_concept_provider().get_required_concept(+ combined_output = get_required_concept(concept_string=ConceptFactory.construct_concept_string_with_domain(domain=combined_output_domain_and_code.domain,concept_code=combined_output_domain_and_code.concept_code,@@ -65,13 +68,13 @@ class PipeParallelFactory(PipeFactoryProtocol[PipeParallelBlueprint, PipeParalleblueprint=blueprint.inputs or {},concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,),- output=get_concept_provider().get_required_concept(+ output=get_required_concept(concept_string=ConceptFactory.construct_concept_string_with_domain(domain=output_domain_and_code.domain,concept_code=output_domain_and_code.concept_code,),),parallel_sub_pipes=parallel_sub_pipes,- add_each_output=blueprint.add_each_output,+ add_each_output=blueprint.add_each_output or False,combined_output=combined_output,)diff --git a/pipelex/pipe_controllers/sub_pipe_blueprint.py b/pipelex/pipe_controllers/sub_pipe_blueprint.pyindex 73ee2d1ac..eb69fbff6 100644--- a/pipelex/pipe_controllers/sub_pipe_blueprint.py+++ b/pipelex/pipe_controllers/sub_pipe_blueprint.py@@ -12,7 +12,7 @@ class SubPipeBlueprint(BaseModel):result: str | None = Nonenb_output: int | None = Nonemultiple_output: bool | None = None- batch_over: bool | str = False+ batch_over: str | None = Nonebatch_as: str | None = None@model_validator(mode="after")@@ -24,14 +24,11 @@ class SubPipeBlueprint(BaseModel):@model_validator(mode="after")def validate_batch_params(self) -> Self:- batch_over_is_specified = self.batch_over is not False and self.batch_over != ""- batch_as_is_specified = self.batch_as is not None and self.batch_as != ""-- if batch_over_is_specified and not batch_as_is_specified:+ if self.batch_over and not self.batch_as:msg = f"In pipe '{self.pipe}': When 'batch_over' is specified, 'batch_as' must also be provided"raise PipeDefinitionError(msg)- if batch_as_is_specified and not batch_over_is_specified:+ if self.batch_as and not self.batch_over:msg = f"In pipe '{self.pipe}': When 'batch_as' is specified, 'batch_over' must also be provided"raise PipeDefinitionError(msg)diff --git a/pipelex/pipelex.toml b/pipelex/pipelex.tomlindex 3f8a28669..352b8b794 100644--- a/pipelex/pipelex.toml+++ b/pipelex/pipelex.toml@@ -204,14 +204,34 @@ text_gen_truncate_length = 256nb_list_items = 3nb_ocr_pages = 4allowed_to_fail_pipes = [- "infinite_loop_1", # Loop but only for testing purposes- "fix_failing_pipes_once", # Loop but its normal+ "infinite_loop_1", # Loop but only for testing purposes+ "fix_failing_pipes_once", # Loop but its normal"validate_pipelex_bundle_spec", # Loop but its normal- "handle_validation_result", # Complex case of a sub pipe needed a ListContent because its batching on, but the dry_run cannot know.- "pipe_builder", # because of the 'fix_failing_pipes_once', 'validate_pipelex_bundle_spec', 'handle_validation_result'+ "handle_validation_result", # Complex case of a sub pipe needed a ListContent because its batching on, but the dry_run cannot know.+ "pipe_builder", # because of the 'fix_failing_pipes_once', 'validate_pipelex_bundle_spec', 'handle_validation_result']image_urls = ["https://storage.googleapis.com/public_test_files_7fa6_4277_9ab/fashion/fashion_photo_1.jpg","https://storage.googleapis.com/public_test_files_7fa6_4277_9ab/fashion/fashion_photo_2.png",]+####################################################################################################+# PLX config+####################################################################################################++[pipelex.plx_config.inline_tables]+spaces_inside_curly_braces = true++[pipelex.plx_config.strings]+prefer_literal = false+force_multiline = false+length_limit_to_multiline = 100+ensure_trailing_newline = true+ensure_leading_blank_line = true++[pipelex.plx_config.concepts]+structure_field_ordering = ["type", "definition", "choices", "required"]++[pipelex.plx_config.pipes]+field_ordering = ["type", "definition", "inputs", "output"]+diff --git a/pipelex/tools/config/manager.py b/pipelex/tools/config/manager.pyindex 15376ec22..97aefde57 100644--- a/pipelex/tools/config/manager.py+++ b/pipelex/tools/config/manager.py@@ -10,7 +10,7 @@ from pipelex.tools.config.config_root import (CONFIG_BASE_OVERRIDES_BEFORE_ENV,)from pipelex.tools.misc.json_utils import deep_update-from pipelex.tools.misc.toml_utils import failable_load_toml_from_path, load_toml_from_path+from pipelex.tools.misc.toml_utils import load_toml_from_path, load_toml_from_path_if_existsfrom pipelex.tools.runtime_manager import runtime_managerCONFIG_DIR_NAME = ".pipelex"@@ -74,8 +74,9 @@ class ConfigManager:Dict[str, Any]: The configuration dictionary from the local pipelex.toml"""- config_path = os.path.join(self.local_root_dir, CONFIG_NAME)- return failable_load_toml_from_path(config_path) or {}+ config_path = os.path.join(self.pipelex_config_dir, CONFIG_NAME)+ print(f"Getting local config from {config_path}")+ return load_toml_from_path_if_exists(config_path) or {}def load_inheritance_config(self, the_pipelex_config: dict[str, Any]):"""Load the config by inheritance in a pyproject.toml file.@@ -105,7 +106,7 @@ class ConfigManager:if package_path:config_path = os.path.join(package_path, "pipelex.toml")if os.path.exists(config_path):- config = failable_load_toml_from_path(config_path)+ config = load_toml_from_path_if_exists(config_path)if config:deep_update(the_pipelex_config, config)@@ -150,12 +151,12 @@ class ConfigManager:override_path = os.path.join(self.local_root_dir, "tests", f"pipelex_{override}.toml")else:override_path = os.path.join(self.local_root_dir, "pipelex" if self.is_in_pipelex_config else "", f"pipelex_{override}.toml")- if override_dict := failable_load_toml_from_path(override_path):+ if override_dict := load_toml_from_path_if_exists(override_path):deep_update(pipelex_config, override_dict)#################### 5. Load specific config ####################if specific_config_path:- config = failable_load_toml_from_path(specific_config_path)+ config = load_toml_from_path_if_exists(specific_config_path)if config:deep_update(pipelex_config, config)else:diff --git a/pipelex/tools/misc/file_utils.py b/pipelex/tools/misc/file_utils.pyindex 2aa7e5a09..d5fe8e29a 100644--- a/pipelex/tools/misc/file_utils.py+++ b/pipelex/tools/misc/file_utils.py@@ -251,6 +251,15 @@ def ensure_path(path: str) -> bool:return True+def ensure_directory_for_file_path(file_path: str) -> None:+ """Ensures a directory exists for the specified file path.++ Args:+ file_path (str): The path to the file.+ """+ ensure_directory_exists(os.path.dirname(file_path))++def path_exists(path_str: str) -> bool:"""Checks if a file or directory exists at the specified path.diff --git a/pipelex/tools/misc/toml_utils.py b/pipelex/tools/misc/toml_utils.pyindex d20b5bec8..0fc407277 100644--- a/pipelex/tools/misc/toml_utils.py+++ b/pipelex/tools/misc/toml_utils.py@@ -1,17 +1,10 @@from __future__ import annotations-import re-from collections.abc import Mapping-from typing import Any, cast+from typing import Anyimport toml-import tomlkit-from pydantic import BaseModel-from tomlkit import array, document, inline_table, table-from tomlkit import string as tk_stringfrom pipelex.tools.misc.file_utils import path_exists-from pipelex.tools.misc.json_utils import remove_none_values_from_dictclass TOMLValidationError(Exception):@@ -122,267 +115,8 @@ def load_toml_from_path(path: str) -> dict[str, Any]:raise toml.TomlDecodeError(msg, exc.doc, exc.pos) from exc-def failable_load_toml_from_path(path: str) -> dict[str, Any] | None:- """Load TOML from path with failure handling."""+def load_toml_from_path_if_exists(path: str) -> dict[str, Any] | None:+ """Load TOML from path if it exists."""if not path_exists(path):return None- try:- return load_toml_from_path(path)- except (toml.TomlDecodeError, TOMLValidationError) as exc:- print(f"Failed to parse TOML file '{path}': {exc}")- return None---def make_toml_string(- text: str,- prefer_literal: bool = False,- force_multiline: bool = False,- ensure_trailing_newline: bool = True,- ensure_leading_blank_line_in_value: bool = False,-):- r"""Build a tomlkit string node.- - If `force_multiline` or the text contains '\\n', we emit a triple-quoted multiline string.- - When multiline, `ensure_trailing_newline` puts the closing quotes on their own line.- - When multiline, `ensure_leading_blank_line_in_value` inserts a real blank line at the start of the value.- """- needs_multiline = force_multiline or ("\n" in text)- normalized = text-- if needs_multiline:- if ensure_leading_blank_line_in_value and not normalized.startswith("\n"):- normalized = "\n" + normalized- if ensure_trailing_newline and not normalized.endswith("\n"):- normalized = normalized + "\n"-- use_literal = prefer_literal and ("'''" not in normalized)- return tk_string(normalized, multiline=needs_multiline, literal=use_literal)---def _convert_to_inline(value: Any) -> Any:- """Recursively convert Python values; dicts -> inline tables; lists kept as arrays."""- # Handle Pydantic models by converting them to dict first- if isinstance(value, BaseModel):- # For RootModel, use the root attribute; for regular models, use model_dump()- value = value.root if hasattr(value, "root") else value.model_dump() # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownVariableType]-- if isinstance(value, Mapping):- value = cast("Mapping[str, Any]", value)- inline_table_obj = inline_table()- for key, value_item in value.items():- inline_table_obj[key] = _convert_to_inline(value_item)-- return inline_table_obj-- if isinstance(value, list):- value = cast("list[Any]", value)- array_obj = array()- array_obj.multiline(True) # set to False for single-line arrays- for element in value:- if isinstance(element, Mapping):- element = cast("Mapping[str, Any]", element)- inline_element = inline_table()- for inner_key, inner_value in element.items():- inline_element[inner_key] = _convert_to_inline(inner_value)- array_obj.append(inline_element) # pyright: ignore[reportUnknownMemberType]- else:- array_obj.append(_convert_to_inline(element)) # pyright: ignore[reportUnknownMemberType]- return array_obj-- if isinstance(value, str):- # For prompt templates and similar fields, escape newlines instead of using multiline format- if any(keyword in str(value) for keyword in ["@", "$"]) and "\n" in value:- # This looks like a prompt template - escape newlines for single-line format- # Return the string directly and let tomlkit handle the escaping properly- return value- # Triple quotes if needed (or forced); closing quotes on their own line.- return make_toml_string(- value,- prefer_literal=False, # flip to True for '''...'''- force_multiline=False, # flip to True to force """...""" even without \n- ensure_trailing_newline=True, # keep closing """ on its own line- ensure_leading_blank_line_in_value=False, # flip to True to keep a blank first line- )- return value # pyright: ignore[reportUnknownVariableType]---def _filter_empty_values(value: Any) -> Any:- """Filter out empty lists and None values from data structures."""- if isinstance(value, dict):- filtered: dict[str, Any] = {}- for k, v in cast("dict[str, Any]", value).items():- filtered_v = _filter_empty_values(v)- # Keep empty dicts but skip empty lists and None values- if filtered_v is not None and (not isinstance(filtered_v, list) or filtered_v):- filtered[k] = filtered_v- return filtered- if isinstance(value, list):- return [_filter_empty_values(item) for item in cast("list[Any]", value) if _filter_empty_values(item) is not None]- return value---def _create_ordered_inline_table(data: Mapping[str, Any]) -> Any:- """Create an inline table with fields in the expected order."""- inline_table_obj = inline_table()-- # Define the preferred order for concept structure fields- field_order = ["type", "definition", "required"]-- # Add fields in preferred order first- for field in field_order:- if field in data:- value = data[field]- # Skip empty lists- if isinstance(value, list) and not value:- continue- inline_table_obj[field] = _convert_to_inline(value)-- # Add any remaining fields not in the preferred order- for key, value in data.items():- if key not in field_order:- # Skip empty lists- if isinstance(value, list) and not value:- continue- inline_table_obj[key] = _convert_to_inline(value)-- return inline_table_obj---def dict_to_toml(data: Mapping[str, Any]) -> str:- """Convert dictionary to TOML format matching Pipelex expectations."""- data = remove_none_values_from_dict(data=data)- data = _filter_empty_values(data)- document_root = document()-- # Handle top-level fields first (domain, definition, system_prompt, etc.)- for key, value in data.items():- if not isinstance(value, Mapping):- document_root.add(key, _convert_to_inline(value))-- # Handle sections (concepts, pipes)- for section_key, section_value in data.items():- if isinstance(section_value, Mapping):- section_value = cast("Mapping[str, Any]", section_value)-- # Skip empty sections- if not section_value:- continue-- # Create the section table- section_table = table()-- # Process each item in the section- for item_key, item_value in section_value.items():- if isinstance(item_value, str):- # Simple string value (e.g., "SimpleData = 'Simple data concept'")- section_table.add(item_key, _convert_to_inline(item_value))- elif isinstance(item_value, Mapping):- # Complex object that needs its own table- item_value = cast("Mapping[str, Any]", item_value)- item_table = table()-- # Handle the structure field specially- for field_key, field_value in item_value.items():- # THIS IS CUSTOM CODE FOR THE CONCEPT STRUCUTRE FIELD. It should be generic- if field_key == "structure" and isinstance(field_value, Mapping):- # Structure should be its own table [section.item.structure]- structure_table = table()- for struct_key, struct_value in cast("Mapping[str, Any]", field_value).items():- if isinstance(struct_value, Mapping):- structure_table.add(struct_key, _create_ordered_inline_table(cast("Mapping[str, Any]", struct_value)))- else:- structure_table.add(struct_key, _convert_to_inline(struct_value))- item_table.add(field_key, structure_table)- else:- # Skip empty lists like refines = []- if isinstance(field_value, list) and not field_value:- continue- item_table.add(field_key, _convert_to_inline(field_value))-- section_table.add(item_key, item_table)-- document_root.add(section_key, section_table)-- dumped_content = tomlkit.dumps(document_root) # pyright: ignore[reportUnknownMemberType]-- # Post-process to fix inline table spacing: {key = "value"} -> { key = "value" }-- # Fix spacing around inline table contents - ensure single space after { and before }- # This regex matches TOML inline tables (containing = signs) but not Jinja2 templates- # We need to handle nested braces properly by working from innermost to outermost-- def fix_inline_table_spacing(content: str) -> str:- """Fix spacing in inline tables, handling nested structures, but avoid modifying braces inside strings."""- # Split content into lines and process each line- lines = content.split("\n")- processed_lines: list[str] = []-- for line in lines:- # Only process lines that contain TOML inline tables (have '=' and braces outside quotes)- # Skip lines that are just string values with braces (like Jinja2 templates)- if "=" in line and "{" in line and "}" in line:- # Check if this line has a TOML key-value assignment with inline table- # Pattern: key = { ... } or key = { nested = { ... } }- if re.match(r"^\s*\w+\s*=\s*\{", line):- # This is a TOML inline table assignment, apply spacing fixes- processed_line = line-- # Apply multiple passes to handle nested structures properly- for _ in range(10): # Prevent infinite loops- old_line = processed_line-- # First: fix simple inline tables (no nested braces)- processed_line = re.sub(r"\{\s*([^{}]*=[^{}]*?)\s*\}", r"{ \1 }", processed_line)-- # Second: fix nested inline tables - outer tables containing inner ones- # Match pattern like { key = { inner = "value" } }- processed_line = re.sub(r"\{\s*([^={}]*=\s*\{[^}]*\}[^{}]*?)\s*\}", r"{ \1 }", processed_line)-- # Third: fix complex nested cases with multiple inner tables- # Handle cases like {key1 = {inner = "value"}, key2 = {inner2 = "value2"}}- processed_line = re.sub(r"\{\s*([^{}]*=\s*\{[^}]*\}.*?)\s*\}", r"{ \1 }", processed_line)-- # Fourth: clean up any excessive whitespace that might remain- # Remove multiple consecutive spaces inside tables- processed_line = re.sub(r"\{\s*([^{}]*?)\s{2,}\}", r"{ \1 }", processed_line)-- # Fifth: ensure consistent single space formatting- processed_line = re.sub(r"\{\s{2,}([^{}]*?)\s*\}", r"{ \1 }", processed_line)-- if old_line == processed_line:- break-- # Final cleanup: fix any remaining spacing issues in inline tables- # First: ensure there's a space between adjacent closing braces "}}"- processed_line = re.sub(r"\}\}", "} }", processed_line)-- # Then: remove extra spaces (2 or more) before closing braces, but keep single spaces- # This pattern matches cases like "} }" (2+ spaces) and replaces with "} }"- processed_line = re.sub(r"\}\s{2,}\}", "} }", processed_line)-- processed_lines.append(processed_line)- else:- # This line has braces but is not a TOML inline table (e.g., string with Jinja2)- # Leave it unchanged- processed_lines.append(line)- else:- # No braces or no assignment, leave unchanged- processed_lines.append(line)-- return "\n".join(processed_lines)-- return fix_inline_table_spacing(dumped_content)---def save_toml_to_path(data: dict[str, Any], path: str) -> None:- """Save dictionary as TOML to file path.-- Args:- data: Dictionary to save as TOML- path: Path where to save the TOML file-- """- data_cleaned = data- with open(path, "w", encoding="utf-8") as file:- toml_content: str = dict_to_toml(data=data_cleaned)- cleaned_content = clean_trailing_whitespace(toml_content)- file.write(cleaned_content)+ return load_toml_from_path(path)diff --git a/pipelex/types.py b/pipelex/types.pyindex 1f1c2116c..d1127172d 100644--- a/pipelex/types.py+++ b/pipelex/types.py@@ -1,5 +1,3 @@-# ruff: noqa: UP-try:from typing import Self # Python 3.11+except ImportError: # Python 3.10diff --git a/pyproject.toml b/pyproject.tomlindex 0cad96831..f4d38d4c5 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -115,7 +115,7 @@ module = ["filetype", "json2html", "pypdfium2", "pypdfium2.raw"][tool.pyright]pythonVersion = "3.11"include = ["pipelex", "tests"]-exclude = ["**/__pycache__", ".venv", ".git", "build", "dist", "tests"]+exclude = ["**/__pycache__", ".venv", ".git", "build", "dist"]analyzeUnannotatedFunctions = truedeprecateTypingAliases = falsedisableBytesTypePromotions = truediff --git a/tests/integration/pipelex/language/test_plx_factory.py b/tests/integration/pipelex/language/test_plx_factory.pynew file mode 100644index 000000000..a7cbbed86--- /dev/null+++ b/tests/integration/pipelex/language/test_plx_factory.py@@ -0,0 +1,22 @@+from pathlib import Path+from typing import Type++import pytest++from pipelex import log, pretty_print+from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint+from pipelex.language.plx_factory import PlxFactory++from tests.unit.pipelex.core.test_data import InterpreterTestCases+++class TestPlxFactoryIntegration:++ @pytest.mark.parametrize("test_name,expected_plx_content,blueprint", InterpreterTestCases.VALID_TEST_CASES)+ def test_make_plx_content(self, test_name: str, expected_plx_content: str, blueprint: PipelexBundleBlueprint):+ """Test making blueprint from various valid PLX content."""+ plx_content = PlxFactory.make_plx_content(blueprint=blueprint)+ pretty_print(plx_content, title=f"Plx content {test_name}")+ pretty_print(expected_plx_content, title=f"Expected PLX content {test_name}")+ assert plx_content == expected_plx_content+diff --git a/tests/test_pipelines/discord_newsletter.py b/tests/test_pipelines/discord_newsletter.pyindex 6de3cb373..6bd5799a7 100644--- a/tests/test_pipelines/discord_newsletter.py+++ b/tests/test_pipelines/discord_newsletter.py@@ -2,6 +2,7 @@from pydantic import Fieldfrom pipelex.core.stuffs.stuff_content import StructuredContent+from pipelex.tools.typing.pydantic_utils import CustomBaseModel, empty_list_factory_ofclass Attachment(StructuredContent):@@ -24,8 +25,8 @@ class DiscordMessage(StructuredContent):author: str = Field(..., description="Author of the message")content: str = Field(..., description="Content of the message")- attachments: list[Attachment] = Field(default_factory=list, description="List of message attachments")- embeds: list[Embed] = Field(default_factory=list, description="List of message embeds")+ attachments: list[Attachment] = Field(default_factory=empty_list_factory_of(Attachment), description="List of message attachments")+ embeds: list[Embed] = Field(default_factory=empty_list_factory_of(Embed), description="List of message embeds")link: str = Field(..., description="Link to the message")@@ -34,7 +35,7 @@ class DiscordChannelUpdate(StructuredContent):name: str = Field(..., description="Name of the Discord channel")position: int = Field(..., description="Position of the channel")- messages: list[DiscordMessage] = Field(default_factory=list, description="List of messages in the channel")+ messages: list[DiscordMessage] = Field(default_factory=empty_list_factory_of(DiscordMessage), description="List of messages in the channel")class ChannelSummary(StructuredContent):@@ -48,6 +49,6 @@ class Newsletter(StructuredContent):"""Represents the final newsletter content"""weekly_summary: str = Field(..., description="200 character summary of weekly Share channel content")- new_members: list[str] = Field(default_factory=list, description="New member introductions in bullet points")- channel_sections: list[ChannelSummary] = Field(default_factory=list, description="Ordered channel summaries")- geographic_hubs: list[ChannelSummary] = Field(default_factory=list, description="Geographic hub channels grouped at end")+ new_members: list[str] = Field(default_factory=empty_list_factory_of(str), description="New member introductions in bullet points")+ channel_sections: list[ChannelSummary] = Field(default_factory=empty_list_factory_of(ChannelSummary), description="Ordered channel summaries")+ geographic_hubs: list[ChannelSummary] = Field(default_factory=empty_list_factory_of(ChannelSummary), description="Geographic hub channels grouped at end")diff --git a/tests/test_pipelines/test_structures_basic.py b/tests/test_pipelines/test_structures_basic.pyindex cfc6fddfe..b7228f507 100644--- a/tests/test_pipelines/test_structures_basic.py+++ b/tests/test_pipelines/test_structures_basic.py@@ -5,6 +5,7 @@ from datetime import datetimefrom pydantic import Fieldfrom pipelex.core.stuffs.stuff_content import StructuredContent+from pipelex.tools.typing.pydantic_utils import empty_list_factory_ofclass ConceptWithSimpleStructure(StructuredContent):@@ -28,7 +29,7 @@ class ConceptWithLists(StructuredContent):"""A structure with list fields."""string_list: list[str] = Field(default_factory=list, description="A list of strings")- number_list: list[int] = Field(default_factory=list, description="A list of numbers")+ number_list: list[int] = Field(default_factory=empty_list_factory_of(int), description="A list of numbers")optional_list: list[str] | None = Field(None, description="An optional list")@@ -37,4 +38,4 @@ class ConceptWithNestedStructures(StructuredContent):simple_nested: ConceptWithSimpleStructure = Field(..., description="A nested simple structure")optional_nested: ConceptWithOptionals | None = Field(None, description="An optional nested structure")- list_of_nested: list[ConceptWithSimpleStructure] = Field(default_factory=list, description="A list of nested structures")+ list_of_nested: list[ConceptWithSimpleStructure] = Field(default_factory=empty_list_factory_of(ConceptWithSimpleStructure), description="A list of nested structures")diff --git a/tests/test_pipelines/test_structures_complex.py b/tests/test_pipelines/test_structures_complex.pyindex 7ba682a6d..0ecc88e1d 100644--- a/tests/test_pipelines/test_structures_complex.py+++ b/tests/test_pipelines/test_structures_complex.py@@ -5,6 +5,7 @@ from pydantic import Fieldfrom pipelex.core.stuffs.stuff_content import StructuredContentfrom tests.test_pipelines.test_structures_basic import ConceptWithOptionals, ConceptWithSimpleStructure+from pipelex.tools.typing.pydantic_utils import empty_list_factory_ofclass ConceptWithDicts(StructuredContent):@@ -20,7 +21,7 @@ class ConceptWithUnions(StructuredContent):string_or_int: str | int = Field(..., description="A field that can be string or int")optional_union: str | bool | None = Field(None, description="An optional union field")- list_of_unions: list[str | int] = Field(default_factory=list, description="A list of union types")+ list_of_unions: list[str | int] = Field(default_factory=lambda: [], description="A list of union types")class ConceptWithComplexUnions(StructuredContent):@@ -37,7 +38,7 @@ class ConceptWithNestedUnions(StructuredContent):simple_or_complex: ConceptWithSimpleStructure | ConceptWithOptionals = Field(..., description="Simple or complex structure")list_of_union_structures: list[ConceptWithSimpleStructure | ConceptWithUnions] = Field(- default_factory=list, description="List of different structure types",+ default_factory=lambda: [], description="List of different structure types",)optional_nested_union: ConceptWithUnions | ConceptWithComplexUnions | None = Field(None, description="Optional union of union structures",diff --git a/tests/unit/pipelex/core/concepts/concept_factory.py/data.py b/tests/unit/pipelex/core/concepts/concept_factory.py/data.pydeleted file mode 100644index 5f1c22cc8..000000000--- a/tests/unit/pipelex/core/concepts/concept_factory.py/data.py+++ /dev/null@@ -1,260 +0,0 @@-from typing import ClassVar, Optional, List, Tuple--from pipelex.core.concepts.concept import Concept-from pipelex.core.concepts.concept_blueprint import (- ConceptBlueprint,- ConceptStructureBlueprint,- ConceptStructureBlueprintFieldType,-)-from pipelex.core.concepts.concept_factory import DomainAndConceptCode-from pipelex.core.concepts.concept_native import NATIVE_CONCEPTS_DATA, NativeConceptEnum-from pipelex.core.domains.domain import SpecialDomain---class TestCases:- # Test cases for make_refines method - only native concepts can be refined- MAKE_REFINES_TEST_CASES: ClassVar[list[tuple[str, ConceptBlueprint, str]]] = [- (- "native_concept_string",- ConceptBlueprint(definition="A concept that refines a native text concept", refines=NativeConceptEnum.TEXT),- f"{SpecialDomain.NATIVE}.{NativeConceptEnum.TEXT}",- ),- (- "fully_qualified_native_string",- ConceptBlueprint(- definition="A concept that refines a fully qualified native concept",- refines=f"{SpecialDomain.NATIVE}.{NativeConceptEnum.TEXT}",- ),- f"{SpecialDomain.NATIVE}.{NativeConceptEnum.TEXT}",- ),- ]-- # Test cases for make_domain_and_concept_code_from_concept_string_or_code method- MAKE_DOMAIN_AND_CONCEPT_CODE_TEST_CASES: ClassVar[List[Tuple[str, str, List[str] | None, DomainAndConceptCode]]] = [- # Test case 1: Concept string with dot notation- ("my_domain", "other_domain.ConceptName", None, DomainAndConceptCode(domain="other_domain", concept_code="ConceptName")),- # Test case 2: Concept string with dot notation (ignores same domain codes)- ("my_domain", "other_domain.ConceptName", ["ConceptName"], DomainAndConceptCode(domain="other_domain", concept_code="ConceptName")),- # Test case 3: Native concept code (Text)- ("my_domain", "Text", None, DomainAndConceptCode(domain=SpecialDomain.NATIVE, concept_code="Text")),- # Test case 4: Native concept code (Image)- ("my_domain", "Image", None, DomainAndConceptCode(domain=SpecialDomain.NATIVE, concept_code="Image")),- # Test case 5: Native concept code (PDF)- ("my_domain", "PDF", None, DomainAndConceptCode(domain=SpecialDomain.NATIVE, concept_code="PDF")),- # Test case 6: Native concept code with same domain codes provided (native takes precedence)- ("my_domain", "Text", ["Text", "OtherConcept"], DomainAndConceptCode(domain=SpecialDomain.NATIVE, concept_code="Text")),- # Test case 7: Concept code from same domain- ("my_domain", "MyConcept", ["MyConcept", "OtherConcept"], DomainAndConceptCode(domain="my_domain", concept_code="MyConcept")),- # Test case 8: Concept code from same domain (case sensitive)- ("my_domain", "MyConcept", ["MyCon", "OtherConcept"], DomainAndConceptCode(domain=SpecialDomain.IMPLICIT, concept_code="MyConcept")),- # Test case 9: Unknown concept code (no same domain codes)- ("my_domain", "UnknownConcept", None, DomainAndConceptCode(domain=SpecialDomain.IMPLICIT, concept_code="UnknownConcept")),- # Test case 10: Unknown concept code (not in same domain codes)- (- "my_domain",- "UnknownConcept",- ["KnownConcept", "OtherConcept"],- DomainAndConceptCode(domain=SpecialDomain.IMPLICIT, concept_code="UnknownConcept"),- ),- # Test case 11: Empty same domain codes list- ("my_domain", "SomeConcept", [], DomainAndConceptCode(domain=SpecialDomain.IMPLICIT, concept_code="SomeConcept")),- # Test case 12: Different domain in concept string- ("my_domain", "another_domain.SomeConcept", ["SomeConcept"], DomainAndConceptCode(domain="another_domain", concept_code="SomeConcept")),- # Test case 13: All native concept codes- ("my_domain", "Dynamic", None, DomainAndConceptCode(domain=SpecialDomain.NATIVE, concept_code="Dynamic")),- ("my_domain", "TextAndImages", None, DomainAndConceptCode(domain=SpecialDomain.NATIVE, concept_code="TextAndImages")),- ("my_domain", "Number", None, DomainAndConceptCode(domain=SpecialDomain.NATIVE, concept_code="Number")),- ("my_domain", "LlmPrompt", None, DomainAndConceptCode(domain=SpecialDomain.NATIVE, concept_code="LlmPrompt")),- ("my_domain", "Page", None, DomainAndConceptCode(domain=SpecialDomain.NATIVE, concept_code="Page")),- ("my_domain", "Anything", None, DomainAndConceptCode(domain=SpecialDomain.NATIVE, concept_code="Anything")),- ]-- # Test cases for make_from_blueprint method- MAKE_FROM_BLUEPRINT_TEST_CASES: ClassVar[list[tuple[str, str, str, ConceptBlueprint, list[str] | None, Concept]]] = [- # Test case 1: Simple blueprint with no structure or refines (goes to implicit)- (- "simple_blueprint",- "my_domain",- "SimpleConcept",- ConceptBlueprint(definition="A simple concept"),- None,- Concept(- domain=SpecialDomain.IMPLICIT,- code="SimpleConcept",- definition="A simple concept",- structure_class_name=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT].content_class_name,- refines=None,- ),- ),- # Test case 2: Blueprint with string structure (existing class, goes to implicit)- (- "string_structure_existing_class",- "my_domain",- "ConceptWithStructure",- ConceptBlueprint(definition="A concept with string structure", structure="TextContent"),- None,- Concept(- domain=SpecialDomain.IMPLICIT,- code="ConceptWithStructure",- definition="A concept with string structure",- structure_class_name=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT].content_class_name,- refines=None,- ),- ),- # Test case 3: Blueprint with refines (native concept, goes to implicit)- (- "blueprint_with_refines_text",- "my_domain",- "MyTextConcept",- ConceptBlueprint(definition="A concept that refines Text", refines="Text"),- None,- Concept(- domain=SpecialDomain.IMPLICIT,- code="MyTextConcept",- definition="A concept that refines Text",- structure_class_name=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT].content_class_name,- refines="native.Text",- ),- ),- # Test case 4: Blueprint with refines (fully qualified native concept, goes to implicit)- (- "blueprint_with_refines_native_image",- "my_domain",- "MyImageConcept",- ConceptBlueprint(definition="A concept that refines native.Image", refines="native.Image"),- None,- Concept(- domain=SpecialDomain.IMPLICIT,- code="MyImageConcept",- definition="A concept that refines native.Image",- structure_class_name=NATIVE_CONCEPTS_DATA[NativeConceptEnum.IMAGE].content_class_name,- refines="native.Image",- ),- ),- # Test case 5: Native concept code (should go to native domain)- (- "native_concept_code",- "my_domain",- "Text",- ConceptBlueprint(definition="Native text concept"),- None,- Concept(- domain=SpecialDomain.NATIVE,- code="Text",- definition="Native text concept",- structure_class_name=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT].content_class_name,- refines=None,- ),- ),- # Test case 6: Concept code from same domain- (- "same_domain_concept",- "my_domain",- "DomainConcept",- ConceptBlueprint(definition="A concept from same domain"),- ["DomainConcept", "OtherConcept"],- Concept(- domain="my_domain",- code="DomainConcept",- definition="A concept from same domain",- structure_class_name=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT].content_class_name,- refines=None,- ),- ),- # Test case 7: Unknown concept code (should go to implicit domain)- (- "implicit_concept",- "my_domain",- "UnknownConcept",- ConceptBlueprint(definition="An unknown concept"),- ["KnownConcept"],- Concept(- domain=SpecialDomain.IMPLICIT,- code="UnknownConcept",- definition="An unknown concept",- structure_class_name=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT].content_class_name,- refines=None,- ),- ),- # Test case 8: Blueprint with dict structure (implicit domain)- (- "dict_structure_implicit",- SpecialDomain.IMPLICIT,- "PersonConcept",- ConceptBlueprint(- definition="A person with structured data",- structure={- "name": "The person's name",- "age": ConceptStructureBlueprint(definition="The person's age", type=ConceptStructureBlueprintFieldType.NUMBER, required=True),- "active": ConceptStructureBlueprint(- definition="Whether the person is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True,- ),- },- ),- None,- Concept(- domain=SpecialDomain.IMPLICIT,- code="PersonConcept",- definition="A person with structured data",- structure_class_name="PersonConcept",- refines=None,- ),- ),- # Test case 9: Blueprint with dict structure (same domain)- (- "dict_structure_same_domain",- "my_domain",- "PersonConcept",- ConceptBlueprint(- definition="A person with structured data",- structure={- "name": ConceptStructureBlueprint(definition="The person's name", type=ConceptStructureBlueprintFieldType.TEXT, required=True),- "age": ConceptStructureBlueprint(definition="The person's age", type=ConceptStructureBlueprintFieldType.NUMBER, required=True),- "active": ConceptStructureBlueprint(- definition="Whether the person is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True,- ),- },- ),- ["PersonConcept", "OtherConcept"],- Concept(- domain="my_domain",- code="PersonConcept",- definition="A person with structured data",- structure_class_name="PersonConcept",- refines=None,- ),- ),- # Test case 10: Blueprint with refines (implicit domain)- (- "refines_implicit_domain",- "my_domain",- "ConceptWithRefines",- ConceptBlueprint(definition="A concept with refines", refines="Text"),- None,- Concept(- domain=SpecialDomain.IMPLICIT,- code="ConceptWithRefines",- definition="A concept with refines",- structure_class_name=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT].content_class_name,- refines="native.Text",- ),- ),- # Test case 11: Blueprint with refines (same domain)- (- "refines_same_domain",- "my_domain",- "ConceptWithRefines",- ConceptBlueprint(definition="A concept with refines", refines="native.Image"),- ["ConceptWithRefines", "OtherConcept"],- Concept(- domain="my_domain",- code="ConceptWithRefines",- definition="A concept with refines",- structure_class_name=NATIVE_CONCEPTS_DATA[NativeConceptEnum.IMAGE].content_class_name,- refines="native.Image",- ),- ),- ]-- # Legacy alias for backward compatibility- TEST_CASES = MAKE_REFINES_TEST_CASESdiff --git a/tests/unit/pipelex/core/concepts/concept_factory.py/test_concept_factory.py b/tests/unit/pipelex/core/concepts/concept_factory.py/test_concept_factory.pydeleted file mode 100644index 8ea032084..000000000--- a/tests/unit/pipelex/core/concepts/concept_factory.py/test_concept_factory.py+++ /dev/null@@ -1,123 +0,0 @@-import pytest-from typing import Optional, List--from pipelex.core.concepts.concept import Concept-from pipelex.core.concepts.concept_blueprint import (- ConceptBlueprint,- ConceptStructureBlueprint,- ConceptStructureBlueprintFieldType,-)-from pipelex.core.concepts.concept_factory import ConceptFactory--from .data import TestCases---class TestConceptFactory:- """Test ConceptFactory methods with various configurations."""-- @pytest.mark.parametrize(- "test_name,blueprint,expected_result",- TestCases.MAKE_REFINES_TEST_CASES,- )- def test_make_refines(- self,- test_name: str,- blueprint: ConceptBlueprint,- expected_result: str,- ):- """Test make_refines method with different blueprint configurations."""- result = ConceptFactory.make_refines(blueprint=blueprint)- assert result == expected_result, f"Failed for test case: {test_name}"-- def test_normalize_structure_blueprint(self):- """Test that mixed structure blueprints are properly normalized."""- mixed_structure_blueprint: dict[str, str | ConceptStructureBlueprint] = {- "name": "The name of the person",- "age": ConceptStructureBlueprint(definition="The age of the person", type=ConceptStructureBlueprintFieldType.NUMBER, required=True),- "active": ConceptStructureBlueprint(- definition="Whether the person is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True,- ),- }-- expected_structure: dict[str, ConceptStructureBlueprint] = {- "name": ConceptStructureBlueprint(definition="The name of the person", type=ConceptStructureBlueprintFieldType.TEXT, required=True),- "age": ConceptStructureBlueprint(definition="The age of the person", type=ConceptStructureBlueprintFieldType.NUMBER, required=True),- "active": ConceptStructureBlueprint(- definition="Whether the person is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True,- ),- }-- assert ConceptFactory.normalize_structure_blueprint(mixed_structure_blueprint) == expected_structure-- mixed_structure_blueprint2: dict[str, str | ConceptStructureBlueprint] = {- "name": ConceptStructureBlueprint(definition="The name of the person", type=ConceptStructureBlueprintFieldType.TEXT, required=True),- "age": ConceptStructureBlueprint(definition="The age of the person", type=ConceptStructureBlueprintFieldType.NUMBER, required=True),- "active": ConceptStructureBlueprint(- definition="Whether the person is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True,- ),- }-- expected_structure2: dict[str, ConceptStructureBlueprint] = {- "name": ConceptStructureBlueprint(definition="The name of the person", type=ConceptStructureBlueprintFieldType.TEXT, required=True),- "age": ConceptStructureBlueprint(definition="The age of the person", type=ConceptStructureBlueprintFieldType.NUMBER, required=True),- "active": ConceptStructureBlueprint(- definition="Whether the person is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True,- ),- }-- assert ConceptFactory.normalize_structure_blueprint(mixed_structure_blueprint2) == expected_structure2-- @pytest.mark.parametrize(- "domain,concept_string_or_code,concept_codes_from_the_same_domain,expected_result",- TestCases.MAKE_DOMAIN_AND_CONCEPT_CODE_TEST_CASES,- )- def test_make_domain_and_concept_code_from_concept_string_or_code(- self,- domain: str,- concept_string_or_code: str,- concept_codes_from_the_same_domain: List[str] | None,- expected_result: List[str],- ):- """Test make_domain_and_concept_code_from_concept_string_or_code method with various inputs."""- result = ConceptFactory.make_domain_and_concept_code_from_concept_string_or_code(- domain=domain,- concept_string_or_code=concept_string_or_code,- concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,- )- assert result == expected_result-- @pytest.mark.parametrize(- "test_name,domain,concept_code,blueprint,concept_codes_from_the_same_domain,expected_concept",- TestCases.MAKE_FROM_BLUEPRINT_TEST_CASES,- )- def test_make_from_blueprint(- self,- test_name: str,- domain: str,- concept_code: str,- blueprint: ConceptBlueprint,- concept_codes_from_the_same_domain: list[str] | None,- expected_concept: Concept,- ):- """Test make_from_blueprint method with various blueprint configurations."""- result = ConceptFactory.make_from_blueprint(- domain=domain,- concept_code=concept_code,- blueprint=blueprint,- concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,- )-- assert result == expected_concept, f"Concept mismatch for {test_name}"-- def test_make_from_blueprint_with_invalid_structure_class(self):- """Test that make_from_blueprint raises StructureClassError for invalid structure class."""- blueprint = ConceptBlueprint(definition="A concept with invalid structure", structure="NonExistentStructureClass")-- from pipelex.exceptions import StructureClassError-- with pytest.raises(StructureClassError, match="is not a registered subclass of StuffContent"):- ConceptFactory.make_from_blueprint(- domain="my_domain",- concept_code="TestConcept",- blueprint=blueprint,- )diff --git a/tests/unit/pipelex/core/concepts/concept_factory/data.py b/tests/unit/pipelex/core/concepts/concept_factory/data.pyindex 930b8696a..2a635dc09 100644--- a/tests/unit/pipelex/core/concepts/concept_factory/data.py+++ b/tests/unit/pipelex/core/concepts/concept_factory/data.py@@ -1,5 +1,3 @@-"""Test data for ConceptFactory tests."""-from typing import ClassVarfrom pipelex.core.concepts.concept import Concept@@ -14,22 +12,20 @@ from pipelex.core.domains.domain import SpecialDomainclass TestCases:- """Test cases for ConceptFactory methods."""-# Test cases for make_refines method - only native concepts can be refinedMAKE_REFINES_TEST_CASES: ClassVar[list[tuple[str, ConceptBlueprint, str]]] = [("native_concept_string",- ConceptBlueprint(definition="A concept that refines a native text concept", refines=NativeConceptEnum.TEXT.value),- f"{SpecialDomain.NATIVE.value}.{NativeConceptEnum.TEXT.value}",+ ConceptBlueprint(definition="A concept that refines a native text concept", refines=NativeConceptEnum.TEXT),+ f"{SpecialDomain.NATIVE}.{NativeConceptEnum.TEXT}",),("fully_qualified_native_string",ConceptBlueprint(definition="A concept that refines a fully qualified native concept",- refines=f"{SpecialDomain.NATIVE.value}.{NativeConceptEnum.TEXT.value}",+ refines=f"{SpecialDomain.NATIVE}.{NativeConceptEnum.TEXT}",),- f"{SpecialDomain.NATIVE.value}.{NativeConceptEnum.TEXT.value}",+ f"{SpecialDomain.NATIVE}.{NativeConceptEnum.TEXT}",),]diff --git a/tests/unit/pipelex/core/concepts/concept_factory/test_concept_factory.py b/tests/unit/pipelex/core/concepts/concept_factory/test_concept_factory.pyindex 195a43805..b8a47a9d1 100644--- a/tests/unit/pipelex/core/concepts/concept_factory/test_concept_factory.py+++ b/tests/unit/pipelex/core/concepts/concept_factory/test_concept_factory.py@@ -1,6 +1,3 @@-"""Tests for ConceptFactory methods."""--import pytestfrom pipelex.core.concepts.concept import Concept@@ -21,14 +18,16 @@ class TestConceptFactory:"test_name,blueprint,expected_result",TestCases.MAKE_REFINES_TEST_CASES,)- def test_make_refines(+ def test_make_refine(self,test_name: str,blueprint: ConceptBlueprint,expected_result: str,):"""Test make_refines method with different blueprint configurations."""- result = ConceptFactory.make_refines(blueprint=blueprint)+ if not blueprint.refines:+ pytest.skip("Test case has no refines")+ result = ConceptFactory.make_refine(refine=blueprint.refines)assert result == expected_result, f"Failed for test case: {test_name}"def test_normalize_structure_blueprint(self):@@ -73,7 +72,7 @@ class TestConceptFactory:"domain,concept_string_or_code,concept_codes_from_the_same_domain,expected_result",TestCases.MAKE_DOMAIN_AND_CONCEPT_CODE_TEST_CASES,)- def test_make_domain_and_concept_code_from_concept_string_orcode(+ def test_make_domain_and_concept_code_from_concept_string_or_code(self,domain: str,concept_string_or_code: str,diff --git a/tests/unit/pipelex/create/test_structured_output_generator.py b/tests/unit/pipelex/core/concepts/test_structured_output_generator.pysimilarity index 100%rename from tests/unit/pipelex/create/test_structured_output_generator.pyrename to tests/unit/pipelex/core/concepts/test_structured_output_generator.pydiff --git a/tests/unit/pipelex/core/interpreter/test_interpreter.py b/tests/unit/pipelex/core/interpreter/test_interpreter.pyindex 12bf546de..74bcade72 100644--- a/tests/unit/pipelex/core/interpreter/test_interpreter.py+++ b/tests/unit/pipelex/core/interpreter/test_interpreter.py@@ -1,8 +1,9 @@from pathlib import Path+from typing import Typeimport pytest-from pipelex import pretty_print+from pipelex import log, pretty_printfrom pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprintfrom pipelex.core.interpreter import PipelexInterpreterfrom tests.unit.pipelex.core.test_data import InterpreterTestCases@@ -10,6 +11,7 @@ from tests.unit.pipelex.core.test_data import InterpreterTestCasesclass TestPipelexInterpreter:def test_init_with_both_file_path_and_content(self, tmp_path: Path):+ """Test initialization with both file_path and file_content."""test_file = tmp_path / "test.plx"test_file.write_text("domain = 'test'\n[concept]\n")content = "domain = 'other'\n[concept]\n"@@ -20,25 +22,19 @@ class TestPipelexInterpreter:@pytest.mark.parametrize("test_name,plx_content,expected_blueprint", InterpreterTestCases.VALID_TEST_CASES)def test_make_pipelex_bundle_blueprint(self, test_name: str, plx_content: str, expected_blueprint: PipelexBundleBlueprint):+ """Test making blueprint from various valid PLX content."""converter = PipelexInterpreter(file_content=plx_content)-blueprint = converter.make_pipelex_bundle_blueprint()pretty_print(blueprint, title=f"Blueprint {test_name}")pretty_print(expected_blueprint, title=f"Expected blueprint {test_name}")assert blueprint == expected_blueprint- @pytest.mark.parametrize("test_name,plx_content,expected_blueprint", InterpreterTestCases.VALID_TEST_CASES)- def test_make_plx_content(self, test_name: str, plx_content: str, expected_blueprint: PipelexBundleBlueprint):- get_plx_content = PipelexInterpreter.make_plx_content(blueprint=expected_blueprint)-- pretty_print(get_plx_content, title=f"PLx content {test_name}")- pretty_print(plx_content, title=f"Expected PLX content {test_name}")- assert get_plx_content == plx_content-@pytest.mark.parametrize("test_name,invalid_plx_content,expected_exception", InterpreterTestCases.ERROR_TEST_CASES)- def test_invalid_plx_should_raise_exception(self, test_name: str, invalid_plx_content: str, expected_exception: type[Exception]):+ def test_invalid_plx_should_raise_exception(self, test_name: str, invalid_plx_content: str, expected_exception: Type[Exception]):+ """Test that invalid PLX content raises appropriate exceptions."""converter = PipelexInterpreter(file_content=invalid_plx_content)+ log.verbose(f"Testing invalid PLX content: {test_name}")with pytest.raises(expected_exception):converter.make_pipelex_bundle_blueprint()diff --git a/tests/unit/pipelex/core/interpreter/test_interpreter_serialize.py b/tests/unit/pipelex/core/interpreter/test_interpreter_serialize.pydeleted file mode 100644index 97145efd8..000000000--- a/tests/unit/pipelex/core/interpreter/test_interpreter_serialize.py+++ /dev/null@@ -1,318 +0,0 @@-from typing import Any--from pipelex.core.concepts.concept_blueprint import ConceptBlueprint, ConceptStructureBlueprint, ConceptStructureBlueprintFieldType-from pipelex.core.interpreter import PipelexInterpreter-from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint-from pipelex.pipe_controllers.sequence.pipe_sequence_blueprint import PipeSequenceBlueprint-from pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprint-from pipelex.pipe_operators.llm.pipe_llm_blueprint import PipeLLMBlueprint-from pipelex.pipe_operators.ocr.pipe_ocr_blueprint import PipeOcrBlueprint---class TestSerializeInputRequirement:- def test_serialize_input_requirement_basic(self):- input_req = InputRequirementBlueprint(concept="Text")- result = PipelexInterpreter.serialize_input_requirement(input_req)-- expected = {"concept": "Text"}- assert result == expected-- def test_serialize_input_requirement_with_multiplicity_int(self):- input_req = InputRequirementBlueprint(concept="Text", multiplicity=3)- result = PipelexInterpreter.serialize_input_requirement(input_req)-- expected = {"concept": "Text", "multiplicity": 3}- assert result == expected-- def test_serialize_input_requirement_with_multiplicity_bool(self):- input_req = InputRequirementBlueprint(concept="Text", multiplicity=True)- result = PipelexInterpreter.serialize_input_requirement(input_req)-- expected = {"concept": "Text", "multiplicity": True}- assert result == expected---class TestSerializeInputs:- def test_serialize_inputs_mixed_types(self):- inputs: dict[str, str | InputRequirementBlueprint] = {- "text": "Text",- "data": InputRequirementBlueprint(concept="Data", multiplicity=2),- "query": InputRequirementBlueprint(concept="Text"),- }- result = PipelexInterpreter.serialize_inputs(inputs)-- expected = {"text": "Text", "data": {"concept": "Data", "multiplicity": 2}, "query": {"concept": "Text"}}- assert result == expected-- def test_serialize_inputs_empty(self):- inputs: dict[str, Any] = {}- result = PipelexInterpreter.serialize_inputs(inputs)-- assert result == {}---class TestSerializeConceptStructure:-- def test_serialize_concept_structure_field_string(self):- field_value = "Simple description"- result = PipelexInterpreter.serialize_concept_structure_field(field_value)-- assert result == "Simple description"-- def test_serialize_concept_structure_field_blueprint(self):- field_value = ConceptStructureBlueprint(definition="A text field", type=ConceptStructureBlueprintFieldType.TEXT, required=True)- result = PipelexInterpreter.serialize_concept_structure_field(field_value)-- expected = {"definition": "A text field", "required": True, "type": "text"}- assert result == expected-- def test_serialize_concept_structure_field_blueprint_with_all_fields(self):- field_value = ConceptStructureBlueprint(- definition="A complex field",- type=ConceptStructureBlueprintFieldType.DICT,- key_type="string",- value_type="integer",- choices=["option1", "option2"],- required=False,- default_value={"default": 42},- )- result = PipelexInterpreter.serialize_concept_structure_field(field_value)-- expected = {- "definition": "A complex field",- "required": False,- "type": "dict",- "key_type": "string",- "value_type": "integer",- "choices": ["option1", "option2"],- "default_value": {"default": 42},- }- assert result == expected-- def test_serialize_concept_structure_string(self):- structure = "Simple structure"- result = PipelexInterpreter.serialize_concept_structure(structure)-- assert result == "Simple structure"-- def test_serialize_concept_structure_dict(self):- structure = {- "name": "Person's name",- "age": ConceptStructureBlueprint(definition="Person's age", type=ConceptStructureBlueprintFieldType.INTEGER, required=True),- }- result = PipelexInterpreter.serialize_concept_structure(structure)-- expected = {- "name": "Person's name",- "age": {"definition": "Person's age", "required": True, "type": "integer"},- }- assert result == expected---class TestSerializeSingleConcept:-- def test_serialize_single_concept_string(self):- concept = "Simple concept definition"- result = PipelexInterpreter.serialize_single_concept(concept)-- assert result == "Simple concept definition"-- def test_serialize_single_concept_simple_definition(self):- concept = ConceptBlueprint(definition="A simple concept")- result = PipelexInterpreter.serialize_single_concept(concept)-- assert result == "A simple concept"-- def test_serialize_single_concept_with_refines(self):- concept = ConceptBlueprint(definition="A specialized text", refines="Text")- result = PipelexInterpreter.serialize_single_concept(concept)-- expected = {"definition": "A specialized text", "refines": "Text"}- assert result == expected-- def test_serialize_single_concept_with_structure(self):- concept = ConceptBlueprint(- definition="A person",- structure={- "name": "Person's name",- "age": ConceptStructureBlueprint(definition="Person's age", type=ConceptStructureBlueprintFieldType.INTEGER, required=True),- },- )- result = PipelexInterpreter.serialize_single_concept(concept)-- expected = {- "definition": "A person",- "structure": {- "name": "Person's name",- "age": {"definition": "Person's age", "required": True, "type": "integer"},- },- }- assert result == expected---class TestSerializeConcepts:- def test_serialize_concepts_empty(self):- result = PipelexInterpreter.serialize_concepts(None)- assert result == {}-- def test_serialize_concepts_mixed(self):- concepts: dict[str, ConceptBlueprint | str] = {- "SimpleText": "A simple text concept",- "PersonInfo": ConceptBlueprint(definition="Information about a person", refines="Text"),- "StructuredData": ConceptBlueprint(- definition="Structured data",- structure={- "title": "Data title",- "value": ConceptStructureBlueprint(definition="Data value", type=ConceptStructureBlueprintFieldType.NUMBER, required=True),- },- ),- }- result = PipelexInterpreter.serialize_concepts(concepts)-- expected = {- "SimpleText": "A simple text concept",- "PersonInfo": {"definition": "Information about a person", "refines": "Text"},- "StructuredData": {- "definition": "Structured data",- "structure": {- "title": "Data title",- "value": {"definition": "Data value", "required": True, "type": "number"},- },- },- }- assert result == expected---class TestSerializeSubPipe:-- def test_serialize_sub_pipe_basic(self):- sub_pipe = SubPipeBlueprint(pipe="test_pipe", result="test_result")- result = PipelexInterpreter.serialize_sub_pipe(sub_pipe)-- expected = {"pipe": "test_pipe", "result": "test_result"}- assert result == expected-- def test_serialize_sub_pipe_with_nb_output_and_batch(self):- sub_pipe = SubPipeBlueprint(pipe="test_pipe", result="test_result", nb_output=3, batch_over="input_list", batch_as="current_item")- result = PipelexInterpreter.serialize_sub_pipe(sub_pipe)-- expected = {"pipe": "test_pipe", "result": "test_result", "nb_output": 3, "batch_over": "input_list", "batch_as": "current_item"}- assert result == expected-- def test_serialize_sub_pipe_with_multiple_output_and_batch(self):- sub_pipe = SubPipeBlueprint(pipe="test_pipe", result="test_result", multiple_output=True, batch_over="input_list", batch_as="current_item")- result = PipelexInterpreter.serialize_sub_pipe(sub_pipe)-- expected = {"pipe": "test_pipe", "result": "test_result", "multiple_output": True, "batch_over": "input_list", "batch_as": "current_item"}- assert result == expected-- def test_serialize_sub_pipe_skip_defaults(self):- sub_pipe = SubPipeBlueprint(- pipe="test_pipe",- result="test_result",- batch_over=False, # Default value- batch_as=None, # Default value- )- result = PipelexInterpreter.serialize_sub_pipe(sub_pipe)-- expected = {"pipe": "test_pipe", "result": "test_result"}- assert result == expected---class TestSerializePipeBlueprints:- def test_serialize_llm_pipe_basic(self):- pipe = PipeLLMBlueprint(type="PipeLLM", definition="Generate text using LLM", output="Text", prompt_template="Generate a story")- result = PipelexInterpreter.serialize_llm_pipe(pipe)-- expected = {"type": "PipeLLM", "definition": "Generate text using LLM", "output": "Text", "prompt_template": "Generate a story"}- assert result == expected-- def test_serialize_llm_pipe_with_inputs_and_options(self):- pipe = PipeLLMBlueprint(- type="PipeLLM",- definition="Extract information",- inputs={"text": "Text", "topic": InputRequirementBlueprint(concept="Text", multiplicity=1)},- output="ExtractedInfo",- system_prompt="You are an expert extractor",- prompt_template="Extract info about $topic from @text",- nb_output=2,- )- result = PipelexInterpreter.serialize_llm_pipe(pipe)-- expected = {- "type": "PipeLLM",- "definition": "Extract information",- "inputs": {"text": "Text", "topic": {"concept": "Text", "multiplicity": 1}},- "output": "ExtractedInfo",- "nb_output": 2,- "system_prompt": "You are an expert extractor",- "prompt_template": "Extract info about $topic from @text",- }- assert result == expected-- def test_serialize_ocr_pipe_basic(self):- pipe = PipeOcrBlueprint(type="PipeOcr", definition="Extract text from PDF", output="Page", ocr="base_ocr_pypdfium2")- result = PipelexInterpreter.serialize_ocr_pipe(pipe)-- expected = {"type": "PipeOcr", "definition": "Extract text from PDF", "output": "Page", "ocr": "base_ocr_pypdfium2"}- assert result == expected-- def test_serialize_sequence_pipe_basic(self):- pipe = PipeSequenceBlueprint(- type="PipeSequence",- definition="Process in sequence",- output="Result",- steps=[SubPipeBlueprint(pipe="step1", result="result1"), SubPipeBlueprint(pipe="step2", result="result2")],- )- result = PipelexInterpreter.serialize_sequence_pipe(pipe)-- expected = {- "type": "PipeSequence",- "definition": "Process in sequence",- "output": "Result",- "steps": [{"pipe": "step1", "result": "result1"}, {"pipe": "step2", "result": "result2"}],- }- assert result == expected---class TestAddCommonPipeFields:- def test_add_common_pipe_fields_with_inputs(self):- result: dict[str, Any] = {"type": "TestPipe"}-- # Mock pipe with inputs- class MockPipe:- def __init__(self):- self.inputs = {"text": "Text", "data": InputRequirementBlueprint(concept="Data")}-- pipe = MockPipe()- PipelexInterpreter.add_common_pipe_fields(result, pipe)-- expected = {"type": "TestPipe", "inputs": {"text": "Text", "data": {"concept": "Data"}}}- assert result == expected-- def test_add_common_pipe_fields_without_inputs(self):- result: dict[str, Any] = {"type": "TestPipe"}-- # Mock pipe without inputs- class MockPipe:- def __init__(self):- self.inputs = None-- pipe = MockPipe()- PipelexInterpreter.add_common_pipe_fields(result, pipe)-- expected = {"type": "TestPipe"}- assert result == expected-- def test_add_common_pipe_fields_no_inputs_attribute(self):- result: dict[str, Any] = {"type": "TestPipe"}-- # Mock pipe without inputs attribute- class MockPipe:- pass-- pipe = MockPipe()- PipelexInterpreter.add_common_pipe_fields(result, pipe)-- expected = {"type": "TestPipe"}- assert result == expecteddiff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_batch_plx.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_batch_plx.pydeleted file mode 100644index 7e55a2efd..000000000--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_batch_plx.py+++ /dev/null@@ -1,52 +0,0 @@-import pytest--from pipelex.core.interpreter import PipelexInterpreter-from pipelex.pipe_controllers.batch.pipe_batch_blueprint import PipeBatchBlueprint---class TestPipelexInterpreterBatchPLX:- @pytest.mark.parametrize(- "pipe_name,blueprint,expected_plx",- [- # Basic Batch pipe- (- "batch_process",- PipeBatchBlueprint(- type="PipeBatch",- definition="Process items in batch",- output="ProcessedData",- branch_pipe_code="process_item",- input_list_name="items",- input_item_name="current_item",- ),- """[pipe.batch_process]-type = "PipeBatch"-definition = "Process items in batch"-output = "ProcessedData"-branch_pipe_code = "process_item"-input_list_name = "items"-input_item_name = "current_item\"""",- ),- # Batch pipe with inputs- (- "batch_with_inputs",- PipeBatchBlueprint(- type="PipeBatch",- definition="Batch processing with inputs",- inputs={"data_list": "Text", "config": "Text"},- output="ProcessedData",- branch_pipe_code="transform_item",- ),- """[pipe.batch_with_inputs]-type = "PipeBatch"-definition = "Batch processing with inputs"-inputs = { data_list = "Text", config = "Text" }-output = "ProcessedData"-branch_pipe_code = "transform_item\"""",- ),- ],- )- def test_batch_pipe_to_plx_string(self, pipe_name: str, blueprint: PipeBatchBlueprint, expected_plx: str):- """Test converting Batch pipe blueprint to PLX string."""- result = PipelexInterpreter.batch_pipe_to_plx_string(pipe_name, blueprint)- assert result == expected_plxdiff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_concept_plx.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_concept_plx.pydeleted file mode 100644index 3e319da41..000000000--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_concept_plx.py+++ /dev/null@@ -1,95 +0,0 @@-import pytest--from pipelex.core.concepts.concept_blueprint import ConceptBlueprint, ConceptStructureBlueprint, ConceptStructureBlueprintFieldType-from pipelex.core.interpreter import PipelexInterpreter---class TestPipelexInterpreterConceptPLX:- @pytest.mark.parametrize(- "concepts,expected_plx",- [- # Simple string concepts- (- {"ProcessedData": "Data that has been processed", "UserInfo": "Information about a user"},- """[concept]-ProcessedData = "Data that has been processed"-UserInfo = "Information about a user\"""",- ),- # Single concept- (- {"Document": "A text document"},- """[concept]-Document = "A text document\"""",- ),- # ConceptBlueprint objects- (- {- "ProcessedData": ConceptBlueprint(definition="Data that has been processed"),- "UserInfo": ConceptBlueprint(definition="Information about a user"),- },- """[concept]-ProcessedData = "Data that has been processed"-UserInfo = "Information about a user\"""",- ),- # Complex ConceptBlueprint with structure- (- {- "FeatureAnalysis": ConceptBlueprint(- definition="Analysis of a photo's visual content and key features",- structure={- "dominant_feature": ConceptStructureBlueprint(- definition="The most important or dominant feature in the image",- type=ConceptStructureBlueprintFieldType.TEXT,- required=True,- ),- "visual_elements": ConceptStructureBlueprint(- definition="Description of key visual elements present",- type=ConceptStructureBlueprintFieldType.TEXT,- required=True,- ),- "composition": "Analysis of the image composition",- "color_palette": "Description of the main colors in the image",- "mood_atmosphere": "The overall mood or atmosphere of the image",- },- ),- },- """[concept.FeatureAnalysis]-definition = "Analysis of a photo's visual content and key features"--[concept.FeatureAnalysis.structure]-dominant_feature = { type = "text", definition = "The most important or dominant feature in the image", required = true }-visual_elements = { type = "text", definition = "Description of key visual elements present", required = true }-composition = "Analysis of the image composition"-color_palette = "Description of the main colors in the image"-mood_atmosphere = "The overall mood or atmosphere of the image\"""",- ),- # ConceptBlueprint with refines- (- {- "EnhancedText": ConceptBlueprint(- definition="Enhanced text content with additional metadata",- refines="Text",- ),- },- """[concept.EnhancedText]-definition = "Enhanced text content with additional metadata"-refines = "Text\"""",- ),- # ConceptBlueprint with structure name- (- {- "ProductInfo": ConceptBlueprint(- definition="Detailed information about a product",- structure="ProductStructure",- ),- },- """[concept.ProductInfo]-definition = "Detailed information about a product"-structure = "ProductStructure\"""",- ),- ],- )- def test_concepts_to_plx_string(self, concepts: dict[str, ConceptBlueprint | str], expected_plx: str):- """Test converting concepts dict to PLX string."""- result = PipelexInterpreter.concepts_to_plx_string(concepts)- assert result == expected_plxdiff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_condition_plx.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_condition_plx.pydeleted file mode 100644index fd7098489..000000000--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_condition_plx.py+++ /dev/null@@ -1,52 +0,0 @@-import pytest--from pipelex.core.interpreter import PipelexInterpreter-from pipelex.pipe_controllers.condition.pipe_condition_blueprint import PipeConditionBlueprint, PipeConditionPipeMapBlueprint---class TestPipelexInterpreterConditionPLX:- @pytest.mark.parametrize(- "pipe_name,blueprint,expected_plx",- [- # Basic Condition pipe with expression- (- "conditional_process",- PipeConditionBlueprint(- type="PipeCondition",- definition="Process based on condition",- output="ProcessedData",- pipe_map=PipeConditionPipeMapBlueprint({"small": "process_small", "large": "process_large"}),- expression="input_data.category",- ),- """[pipe.conditional_process]-type = "PipeCondition"-definition = "Process based on condition"-output = "ProcessedData"-pipe_map = { small = "process_small", large = "process_large" }-expression = "input_data.category\"""",- ),- # Condition pipe with expression template and inputs- (- "complex_condition",- PipeConditionBlueprint(- type="PipeCondition",- definition="Complex conditional processing",- inputs={"data": "Text", "threshold": "Text"},- output="ProcessedData",- pipe_map=PipeConditionPipeMapBlueprint({"low": "process_low", "medium": "process_medium", "high": "process_high"}),- expression_template="{{ data.size if data.size > threshold else 'low' }}",- ),- """[pipe.complex_condition]-type = "PipeCondition"-definition = "Complex conditional processing"-inputs = { data = "Text", threshold = "Text" }-output = "ProcessedData"-pipe_map = { low = "process_low", medium = "process_medium", high = "process_high" }-expression_template = "{{ data.size if data.size > threshold else 'low' }}\"""",- ),- ],- )- def test_condition_pipe_to_plx_string(self, pipe_name: str, blueprint: PipeConditionBlueprint, expected_plx: str):- """Test converting Condition pipe blueprint to PLX string."""- result = PipelexInterpreter.condition_pipe_to_plx_string(pipe_name, blueprint)- assert result == expected_plxdiff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_func_plx.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_func_plx.pydeleted file mode 100644index 80d6d53fb..000000000--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_func_plx.py+++ /dev/null@@ -1,47 +0,0 @@-import pytest--from pipelex.core.interpreter import PipelexInterpreter-from pipelex.pipe_operators.func.pipe_func_blueprint import PipeFuncBlueprint---class TestPipelexInterpreterFuncPLX:- @pytest.mark.parametrize(- "pipe_name,blueprint,expected_plx",- [- # Basic Func pipe- (- "process_data",- PipeFuncBlueprint(- type="PipeFunc",- definition="Process data with function",- output="ProcessedData",- function_name="process_data_function",- ),- """[pipe.process_data]-type = "PipeFunc"-definition = "Process data with function"-output = "ProcessedData"-function_name = "process_data_function\"""",- ),- # Func pipe with inputs- (- "transform_data",- PipeFuncBlueprint(- type="PipeFunc",- definition="Transform input data",- inputs={"data": "Text", "config": "Text"},- output="TransformedData",- function_name="transform_function",- ),- """[pipe.transform_data]-type = "PipeFunc"-definition = "Transform input data"-inputs = { data = "Text", config = "Text" }-output = "TransformedData"-function_name = "transform_function\"""",- ),- ],- )- def test_func_pipe_to_plx_string(self, pipe_name: str, blueprint: PipeFuncBlueprint, expected_plx: str):- result = PipelexInterpreter.func_pipe_to_plx_string(pipe_name, blueprint)- assert result == expected_plxdiff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_img_gen_plx.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_img_gen_plx.pydeleted file mode 100644index 5b02034f4..000000000--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_img_gen_plx.py+++ /dev/null@@ -1,50 +0,0 @@-import pytest--from pipelex.cogt.img_gen.img_gen_job_components import AspectRatio-from pipelex.core.interpreter import PipelexInterpreter-from pipelex.pipe_operators.img_gen.pipe_img_gen_blueprint import PipeImgGenBlueprint---class TestPipelexInterpreterImgGenPLX:- @pytest.mark.parametrize(- "pipe_name,blueprint,expected_plx",- [- # Basic ImgGen pipe- (- "generate_image",- PipeImgGenBlueprint(- type="PipeImgGen",- definition="Generate an image from a prompt",- output="Image",- img_gen_prompt="A beautiful landscape",- ),- """[pipe.generate_image]-type = "PipeImgGen"-definition = "Generate an image from a prompt"-output = "Image"-img_gen_prompt = "A beautiful landscape\"""",- ),- # ImgGen pipe with some options- (- "generate_with_options",- PipeImgGenBlueprint(- type="PipeImgGen",- definition="Generate image with options",- output="Image",- img_gen_prompt="A futuristic city",- aspect_ratio=AspectRatio.LANDSCAPE_16_9,- seed=12345,- ),- """[pipe.generate_with_options]-type = "PipeImgGen"-definition = "Generate image with options"-output = "Image"-img_gen_prompt = "A futuristic city"-aspect_ratio = "landscape_16_9"-seed = 12345""",- ),- ],- )- def test_img_gen_pipe_to_plx_string(self, pipe_name: str, blueprint: PipeImgGenBlueprint, expected_plx: str):- result = PipelexInterpreter.img_gen_pipe_to_plx_string(pipe_name, blueprint)- assert result == expected_plxdiff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_input_spec.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_input_spec.pydeleted file mode 100644index ff135bbe1..000000000--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_input_spec.py+++ /dev/null@@ -1,53 +0,0 @@-from typing import Any--import pytest--from pipelex.core.interpreter import PipelexInterpreter-from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint---class TestPipelexInterpreterInputSpec:- def test_inputs_to_plx_string_simple_strings(self):- # Simple string inputs (like "Text")- simple_inputs = {"text": "Text", "topic": "Text"}-- result = PipelexInterpreter.inputs_to_plx_string(simple_inputs)-- expected = '{ text = "Text", topic = "Text" }'- assert result == expected-- def test_inputs_to_plx_string_input_requirement_blueprints(self):- # InputRequirementBlueprint inputs- complex_inputs = {- "text": InputRequirementBlueprint(concept="Text", multiplicity=1),- "problem": InputRequirementBlueprint(concept="Text"), # multiplicity=None- "documents": InputRequirementBlueprint(concept="Text", multiplicity=True),- "query": InputRequirementBlueprint(concept="Text", multiplicity=False),- }-- result = PipelexInterpreter.inputs_to_plx_string(complex_inputs)-- expected = (- '{ text = { concept = "Text", multiplicity = 1 }, problem = { concept = "Text" }, '- 'documents = { concept = "Text", multiplicity = true }, query = { concept = "Text", multiplicity = false } }'- )- assert result == expected-- @pytest.mark.parametrize(- "input_req,expected",- [- # Test with just concept (multiplicity=None should not be included)- (InputRequirementBlueprint(concept="Text"), {"concept": "Text"}),- # Test with concept and multiplicity=1- (InputRequirementBlueprint(concept="Text", multiplicity=1), {"concept": "Text", "multiplicity": 1}),- # Test with concept and multiplicity=True- (InputRequirementBlueprint(concept="Text", multiplicity=True), {"concept": "Text", "multiplicity": True}),- # Test with concept and multiplicity=False- (InputRequirementBlueprint(concept="Text", multiplicity=False), {"concept": "Text", "multiplicity": False}),- # Test with different concept- (InputRequirementBlueprint(concept="Image", multiplicity=5), {"concept": "Image", "multiplicity": 5}),- ],- )- def test_serialize_input_requirement(self, input_req: InputRequirementBlueprint, expected: dict[str, Any]):- result = PipelexInterpreter.serialize_input_requirement(input_req)- assert result == expecteddiff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_jinja2_plx.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_jinja2_plx.pydeleted file mode 100644index 525a17a9c..000000000--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_jinja2_plx.py+++ /dev/null@@ -1,47 +0,0 @@-import pytest--from pipelex.core.interpreter import PipelexInterpreter-from pipelex.pipe_operators.jinja2.pipe_jinja2_blueprint import PipeJinja2Blueprint---class TestPipelexInterpreterJinja2PLX:- @pytest.mark.parametrize(- "pipe_name,blueprint,expected_plx",- [- # Basic Jinja2 pipe- (- "process_template",- PipeJinja2Blueprint(- type="PipeJinja2",- definition="Process a Jinja2 template",- output="Text",- jinja2="Hello {{ name }}!",- ),- """[pipe.process_template]-type = "PipeJinja2"-definition = "Process a Jinja2 template"-output = "Text"-jinja2 = "Hello {{ name }}!\"""",- ),- # Jinja2 pipe with inputs and named template- (- "format_data",- PipeJinja2Blueprint(- type="PipeJinja2",- definition="Format data with template",- inputs={"data": "Text", "format": "Text"},- output="Text",- jinja2_name="data_formatter",- ),- """[pipe.format_data]-type = "PipeJinja2"-definition = "Format data with template"-inputs = { data = "Text", format = "Text" }-output = "Text"-jinja2_name = "data_formatter\"""",- ),- ],- )- def test_jinja2_pipe_to_plx_string(self, pipe_name: str, blueprint: PipeJinja2Blueprint, expected_plx: str):- result = PipelexInterpreter.jinja2_pipe_to_plx_string(pipe_name, blueprint)- assert result == expected_plxdiff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_jinja2_spacing.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_jinja2_spacing.pydeleted file mode 100644index 41b8ec9f7..000000000--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_jinja2_spacing.py+++ /dev/null@@ -1,77 +0,0 @@-import pytest--from pipelex.core.interpreter import PipelexInterpreter-from pipelex.pipe_operators.jinja2.pipe_jinja2_blueprint import PipeJinja2Blueprint---class TestPipelexInterpreterJinja2PipePLX:- @pytest.mark.parametrize(- "pipe_name,blueprint,expected_plx",- [- # Basic Jinja2 pipe- (- "greeting_pipe",- PipeJinja2Blueprint(- type="PipeJinja2",- definition="Process a Jinja2 template",- output="Text",- jinja2="Hello {{ name }}!",- ),- """[pipe.greeting_pipe]-type = "PipeJinja2"-definition = "Process a Jinja2 template"-output = "Text"-jinja2 = "Hello {{ name }}!\"""",- ),- # Jinja2 pipe with inputs- (- "template_pipe",- PipeJinja2Blueprint(- type="PipeJinja2",- definition="Process template with inputs",- inputs={"name": "Text", "greeting": "Text"},- output="Text",- jinja2="{{ greeting }} {{ name }}!",- ),- """[pipe.template_pipe]-type = "PipeJinja2"-definition = "Process template with inputs"-inputs = { name = "Text", greeting = "Text" }-output = "Text"-jinja2 = "{{ greeting }} {{ name }}!\"""",- ),- # Jinja2 pipe with complex template- (- "complex_pipe",- PipeJinja2Blueprint(- type="PipeJinja2",- definition="Process complex template",- output="Text",- jinja2="{% for item in items %}{{ item.name }}{% endfor %}",- ),- """[pipe.complex_pipe]-type = "PipeJinja2"-definition = "Process complex template"-output = "Text"-jinja2 = "{% for item in items %}{{ item.name }}{% endfor %}\"""",- ),- # Jinja2 pipe with jinja2_name instead of inline jinja2- (- "named_template_pipe",- PipeJinja2Blueprint(- type="PipeJinja2",- definition="Use named template",- output="Text",- jinja2_name="greeting_template",- ),- """[pipe.named_template_pipe]-type = "PipeJinja2"-definition = "Use named template"-output = "Text"-jinja2_name = "greeting_template\"""",- ),- ],- )- def test_jinja2_pipe_to_plx_string(self, pipe_name: str, blueprint: PipeJinja2Blueprint, expected_plx: str):- result = PipelexInterpreter.jinja2_pipe_to_plx_string(pipe_name, blueprint)- assert result == expected_plxdiff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_llm_plx.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_llm_plx.pydeleted file mode 100644index 5b9b0e6f4..000000000--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_llm_plx.py+++ /dev/null@@ -1,91 +0,0 @@-import pytest--from pipelex.core.interpreter import PipelexInterpreter-from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint-from pipelex.pipe_operators.llm.pipe_llm_blueprint import PipeLLMBlueprint---class TestPipelexInterpreterLLMPLX:- @pytest.mark.parametrize(- "pipe_name,blueprint,expected_plx",- [- # Simple LLM pipe with string inputs- (- "simple_llm",- PipeLLMBlueprint(- type="PipeLLM",- definition="Simple text generation",- inputs={"text": "Text", "topic": "Text"},- output="Text",- prompt_template="Generate text about $topic based on: @text",- ),- """[pipe.simple_llm]-type = "PipeLLM"-definition = "Simple text generation"-inputs = { text = "Text", topic = "Text" }-output = "Text"-prompt_template = "Generate text about $topic based on: @text\"""",- ),- # LLM pipe with complex inputs (InputRequirementBlueprint)- (- "complex_llm",- PipeLLMBlueprint(- type="PipeLLM",- definition="Extract structured information",- inputs={- "documents": InputRequirementBlueprint(concept="Text", multiplicity=True),- "query": InputRequirementBlueprint(concept="Text", multiplicity=False),- },- output="DocumentSummary",- prompt_template="Extract information from documents: @documents based on query: $query",- ),- """[pipe.complex_llm]-type = "PipeLLM"-definition = "Extract structured information"-inputs = { documents = { concept = "Text", multiplicity = true }, query = { concept = "Text", multiplicity = false } }-output = "DocumentSummary"-prompt_template = "Extract information from documents: @documents based on query: $query\"""",- ),- # LLM pipe with system prompt- (- "system_prompt_llm",- PipeLLMBlueprint(- type="PipeLLM",- definition="Analysis with system prompt",- inputs={"content": "Text"},- output="Text",- system_prompt_template="You are an expert analyst with $expertise_level knowledge",- prompt_template="Analyze this content: @content",- ),- """[pipe.system_prompt_llm]-type = "PipeLLM"-definition = "Analysis with system prompt"-inputs = { content = "Text" }-output = "Text"-system_prompt_template = "You are an expert analyst with $expertise_level knowledge"-prompt_template = "Analyze this content: @content\"""",- ),- # LLM pipe with nb_output- (- "multiple_output_llm",- PipeLLMBlueprint(- type="PipeLLM",- definition="Generate multiple outputs",- inputs={"topic": "Text"},- output="Text",- nb_output=3,- prompt_template="Generate ideas about: $topic",- ),- """[pipe.multiple_output_llm]-type = "PipeLLM"-definition = "Generate multiple outputs"-inputs = { topic = "Text" }-output = "Text"-nb_output = 3-prompt_template = "Generate ideas about: $topic\"""",- ),- ],- )- def test_llm_pipe_to_plx_string(self, pipe_name: str, blueprint: PipeLLMBlueprint, expected_plx: str):- result = PipelexInterpreter.llm_pipe_to_plx_string(pipe_name, blueprint)- assert result == expected_plxdiff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_ocr_plx.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_ocr_plx.pydeleted file mode 100644index c8f6b3fc9..000000000--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_ocr_plx.py+++ /dev/null@@ -1,47 +0,0 @@-import pytest--from pipelex.core.interpreter import PipelexInterpreter-from pipelex.pipe_operators.ocr.pipe_ocr_blueprint import PipeOcrBlueprint---class TestPipelexInterpreterOcrPLX:- @pytest.mark.parametrize(- "pipe_name,blueprint,expected_plx",- [- # Basic OCR pipe- (- "extract_text",- PipeOcrBlueprint(- type="PipeOcr",- definition="Extract text from document",- output="Page",- ocr="base_ocr_pypdfium2",- ),- """[pipe.extract_text]-type = "PipeOcr"-definition = "Extract text from document"-output = "Page"-ocr = "base_ocr_pypdfium2\"""",- ),- # OCR pipe with inputs- (- "extract_with_input",- PipeOcrBlueprint(- type="PipeOcr",- definition="Extract text from PDF",- inputs={"ocr_input": "PDF"},- output="Page",- ocr="base_ocr_pypdfium2",- ),- """[pipe.extract_with_input]-type = "PipeOcr"-definition = "Extract text from PDF"-inputs = { ocr_input = "PDF" }-output = "Page"-ocr = "base_ocr_pypdfium2\"""",- ),- ],- )- def test_ocr_pipe_to_plx_string(self, pipe_name: str, blueprint: PipeOcrBlueprint, expected_plx: str):- result = PipelexInterpreter.ocr_pipe_to_plx_string(pipe_name, blueprint)- assert result == expected_plxdiff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_parallel_plx.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_parallel_plx.pydeleted file mode 100644index 4ddbac40d..000000000--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_parallel_plx.py+++ /dev/null@@ -1,66 +0,0 @@-import pytest--from pipelex.core.interpreter import PipelexInterpreter-from pipelex.pipe_controllers.parallel.pipe_parallel_blueprint import PipeParallelBlueprint-from pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprint---class TestPipelexInterpreterParallelPLX:- @pytest.mark.parametrize(- "pipe_name,blueprint,expected_plx",- [- # Basic Parallel pipe- (- "parallel_process",- PipeParallelBlueprint(- type="PipeParallel",- definition="Process data in parallel",- output="ProcessedData",- parallels=[- SubPipeBlueprint(pipe="process_a", result="result_a"),- SubPipeBlueprint(pipe="process_b", result="result_b"),- ],- ),- """[pipe.parallel_process]-type = "PipeParallel"-definition = "Process data in parallel"-output = "ProcessedData"-parallels = [- { pipe = "process_a", result = "result_a" },- { pipe = "process_b", result = "result_b" },-]""",- ),- # Parallel pipe with inputs and options- (- "complex_parallel",- PipeParallelBlueprint(- type="PipeParallel",- definition="Complex parallel processing",- inputs={"data": "Text"},- output="ProcessedData",- parallels=[- SubPipeBlueprint(pipe="analyze", result="analysis"),- SubPipeBlueprint(pipe="summarize", result="summary"),- SubPipeBlueprint(pipe="extract", result="extraction"),- ],- add_each_output=False,- combined_output="MergedResults",- ),- """[pipe.complex_parallel]-type = "PipeParallel"-definition = "Complex parallel processing"-inputs = { data = "Text" }-output = "ProcessedData"-parallels = [- { pipe = "analyze", result = "analysis" },- { pipe = "summarize", result = "summary" },- { pipe = "extract", result = "extraction" },-]-add_each_output = false-combined_output = "MergedResults\"""",- ),- ],- )- def test_parallel_pipe_to_plx_string(self, pipe_name: str, blueprint: PipeParallelBlueprint, expected_plx: str):- result = PipelexInterpreter.parallel_pipe_to_plx_string(pipe_name, blueprint)- assert result == expected_plxdiff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_sequence_plx.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_sequence_plx.pydeleted file mode 100644index 8c2e15b65..000000000--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_sequence_plx.py+++ /dev/null@@ -1,84 +0,0 @@-import pytest--from pipelex.core.interpreter import PipelexInterpreter-from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint-from pipelex.pipe_controllers.sequence.pipe_sequence_blueprint import PipeSequenceBlueprint-from pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprint---class TestPipelexInterpreterSequencePLX:- @pytest.mark.parametrize(- "pipe_name,blueprint,expected_plx",- [- # Simple sequence pipe with 2 steps- (- "simple_sequence",- PipeSequenceBlueprint(- type="PipeSequence",- definition="Process data in sequence",- output="ProcessedData",- steps=[- SubPipeBlueprint(pipe="extract_info", result="extracted"),- SubPipeBlueprint(pipe="summarize", result="final_summary"),- ],- ),- """[pipe.simple_sequence]-type = "PipeSequence"-definition = "Process data in sequence"-output = "ProcessedData"-steps = [- { pipe = "extract_info", result = "extracted" },- { pipe = "summarize", result = "final_summary" },-]""",- ),- # Sequence pipe with inputs and steps- (- "sequence_with_inputs",- PipeSequenceBlueprint(- type="PipeSequence",- definition="Process data with inputs",- inputs={"text": InputRequirementBlueprint(concept="Text", multiplicity=1), "topic": "Text"},- output="ProcessedData",- steps=[- SubPipeBlueprint(pipe="extract_info", result="extracted"),- SubPipeBlueprint(pipe="summarize", result="final_summary"),- ],- ),- """[pipe.sequence_with_inputs]-type = "PipeSequence"-definition = "Process data with inputs"-inputs = { text = { concept = "Text", multiplicity = 1 }, topic = "Text" }-output = "ProcessedData"-steps = [- { pipe = "extract_info", result = "extracted" },- { pipe = "summarize", result = "final_summary" },-]""",- ),- # Sequence pipe with complex sub pipe options- (- "complex_sequence",- PipeSequenceBlueprint(- type="PipeSequence",- definition="Complex sequence processing",- output="ProcessedData",- steps=[- SubPipeBlueprint(pipe="step1", result="result1", nb_output=2),- SubPipeBlueprint(pipe="step2", result="result2", multiple_output=True),- SubPipeBlueprint(pipe="step3", result="result3", batch_over="items", batch_as="item"),- ],- ),- """[pipe.complex_sequence]-type = "PipeSequence"-definition = "Complex sequence processing"-output = "ProcessedData"-steps = [- { pipe = "step1", result = "result1", nb_output = 2 },- { pipe = "step2", result = "result2", multiple_output = true },- { pipe = "step3", result = "result3", batch_over = "items", batch_as = "item" },-]""",- ),- ],- )- def test_sequence_pipe_to_plx_string(self, pipe_name: str, blueprint: PipeSequenceBlueprint, expected_plx: str):- result = PipelexInterpreter.sequence_pipe_to_plx_string(pipe_name, blueprint)- assert result == expected_plxdiff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_serialize_sub_pipe.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_serialize_sub_pipe.pydeleted file mode 100644index 95d72fa7c..000000000--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_serialize_sub_pipe.py+++ /dev/null@@ -1,153 +0,0 @@-from typing import Any--import pytest--from pipelex.core.interpreter import PipelexInterpreter-from pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprint---class TestPipelexInterpreterSerializeSubPipe:- @pytest.mark.parametrize(- "sub_pipe_blueprint,expected_result",- [- # Basic sub pipe (step in sequence)- (- SubPipeBlueprint(- pipe="step1",- result="intermediate1",- ),- {- "pipe": "step1",- "result": "intermediate1",- },- ),- # Sub pipe with nb_output- (- SubPipeBlueprint(- pipe="generate_ideas",- result="ideas",- nb_output=3,- ),- {- "pipe": "generate_ideas",- "result": "ideas",- "nb_output": 3,- },- ),- # Sub pipe with multiple_output- (- SubPipeBlueprint(- pipe="brainstorm",- result="solutions",- multiple_output=True,- ),- {- "pipe": "brainstorm",- "result": "solutions",- "multiple_output": True,- },- ),- # Sub pipe with batch_over and batch_as- (- SubPipeBlueprint(- pipe="process_items",- result="processed_items",- batch_over="input_list",- batch_as="current_item",- ),- {- "pipe": "process_items",- "result": "processed_items",- "batch_over": "input_list",- "batch_as": "current_item",- },- ),- # Sub pipe with nb_output and batch fields (no multiple_output)- (- SubPipeBlueprint(- pipe="complex_step",- result="complex_result",- nb_output=2,- batch_over="items",- batch_as="item",- ),- {- "pipe": "complex_step",- "result": "complex_result",- "nb_output": 2,- "batch_over": "items",- "batch_as": "item",- },- ),- # Sub pipe with batch_over=False (default, should not be included)- (- SubPipeBlueprint(- pipe="simple_step",- result="simple_result",- batch_over=False, # This is the default and should not appear in output- ),- {- "pipe": "simple_step",- "result": "simple_result",- # batch_over=False should not be included since it's the default- },- ),- ],- )- def test_serialize_sub_pipe(self, sub_pipe_blueprint: SubPipeBlueprint, expected_result: dict[str, Any]):- result = PipelexInterpreter.serialize_sub_pipe(sub_pipe_blueprint)- assert result == expected_result-- @pytest.mark.parametrize(- "sub_pipe_blueprint,expected_plx_string",- [- # Basic sub pipe (step in sequence)- (- SubPipeBlueprint(- pipe="step1",- result="intermediate1",- ),- '{ pipe = "step1", result = "intermediate1" }',- ),- # Sub pipe with nb_output- (- SubPipeBlueprint(- pipe="generate_ideas",- result="ideas",- nb_output=3,- ),- '{ pipe = "generate_ideas", result = "ideas", nb_output = 3 }',- ),- # Sub pipe with multiple_output- (- SubPipeBlueprint(- pipe="brainstorm",- result="solutions",- multiple_output=True,- ),- '{ pipe = "brainstorm", result = "solutions", multiple_output = true }',- ),- # Sub pipe with batch_over and batch_as- (- SubPipeBlueprint(- pipe="process_items",- result="processed_items",- batch_over="input_list",- batch_as="current_item",- ),- '{ pipe = "process_items", result = "processed_items", batch_over = "input_list", batch_as = "current_item" }',- ),- # Sub pipe with batch_over=False (default, should not be included)- (- SubPipeBlueprint(- pipe="simple_step",- result="simple_result",- batch_over=False, # This is the default and should not appear in output- ),- '{ pipe = "simple_step", result = "simple_result" }',- ),- ],- )- def test_sub_pipe_to_plx_string(self, sub_pipe_blueprint: SubPipeBlueprint, expected_plx_string: str):- result = PipelexInterpreter.sub_pipe_to_plx_string(sub_pipe_blueprint)- assert result == expected_plx_stringdiff --git a/tests/unit/pipelex/core/test_data/__init__.py b/tests/unit/pipelex/core/test_data/__init__.pyindex 6c1d15983..1df3bcca0 100644--- a/tests/unit/pipelex/core/test_data/__init__.py+++ b/tests/unit/pipelex/core/test_data/__init__.py@@ -1,4 +1,4 @@-from typing import ClassVar, List, Tuple, Type, Union+from typing import ClassVar, Type, Unionfrom pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint@@ -23,7 +23,7 @@ class InterpreterTestCases:"""Test cases for PipelexInterpreter with PLX content and expected blueprints."""# Aggregate all valid test cases from organized modules- VALID_TEST_CASES: ClassVar[list[tuple[str, str, PipelexBundleBlueprint]]] = [+ VALID_TEST_CASES: ClassVar[list[tuple[str, str, PipelexBundleBlueprint]]] = [ # test_name,plx_content,blueprint# Domain tests*DOMAIN_TEST_CASES,# Concept tests@@ -36,7 +36,7 @@ class InterpreterTestCases:*PIPE_FUNC_TEST_CASES,*PIPE_IMG_GEN_TEST_CASES,*PIPE_JINJA2_TEST_CASES,- # # Pipe controller tests+ # Pipe controller tests*PIPE_SEQUENCE_TEST_CASES,*PIPE_CONDITION_TEST_CASES,*PIPE_PARALLEL_TEST_CASES,diff --git a/tests/unit/pipelex/core/test_data/pipes/controllers/condition/pipe_condition.py b/tests/unit/pipelex/core/test_data/pipes/controllers/condition/pipe_condition.pyindex d83deab06..d25ae5095 100644--- a/tests/unit/pipelex/core/test_data/pipes/controllers/condition/pipe_condition.py+++ b/tests/unit/pipelex/core/test_data/pipes/controllers/condition/pipe_condition.py@@ -10,8 +10,8 @@ definition = "Domain with conditional pipe"type = "PipeCondition"definition = "Process based on condition"output = "ProcessedData"-pipe_map = { small = "process_small", large = "process_large" }expression = "input_data.category"+pipe_map = { small = "process_small", large = "process_large" }""",PipelexBundleBlueprint(domain="test_pipes",diff --git a/tests/unit/pipelex/core/test_data/pipes/controllers/parallel/pipe_parallel.py b/tests/unit/pipelex/core/test_data/pipes/controllers/parallel/pipe_parallel.pyindex 1056a0abc..f184e46de 100644--- a/tests/unit/pipelex/core/test_data/pipes/controllers/parallel/pipe_parallel.py+++ b/tests/unit/pipelex/core/test_data/pipes/controllers/parallel/pipe_parallel.py@@ -9,12 +9,13 @@ definition = "Domain with parallel pipe"[pipe.parallel_process]type = "PipeParallel"-definition = "Process data in parallel"+definition = "PipeParallel example in PIPE_PARALLEL_TEST_CASES"output = "ProcessedData"parallels = [{ pipe = "process_a", result = "result_a" },{ pipe = "process_b", result = "result_b" },]+add_each_output = true""",PipelexBundleBlueprint(domain="test_pipes",@@ -22,12 +23,13 @@ parallels = [pipe={"parallel_process": PipeParallelBlueprint(type="PipeParallel",- definition="Process data in parallel",+ definition="PipeParallel example in PIPE_PARALLEL_TEST_CASES",output="ProcessedData",parallels=[SubPipeBlueprint(pipe="process_a", result="result_a"),SubPipeBlueprint(pipe="process_b", result="result_b"),],+ add_each_output=True,),},),diff --git a/tests/unit/pipelex/core/test_data/pipes/operators/jinja2/pipe_jinja2.py b/tests/unit/pipelex/core/test_data/pipes/operators/jinja2/pipe_jinja2.pyindex 4781fe0e8..b020bdcd3 100644--- a/tests/unit/pipelex/core/test_data/pipes/operators/jinja2/pipe_jinja2.py+++ b/tests/unit/pipelex/core/test_data/pipes/operators/jinja2/pipe_jinja2.py@@ -1,6 +1,7 @@from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprintfrom pipelex.core.concepts.concept_native import NativeConceptEnumfrom pipelex.pipe_operators.jinja2.pipe_jinja2_blueprint import PipeJinja2Blueprint+from pipelex.tools.templating.jinja2_template_category import Jinja2TemplateCategoryPIPE_JINJA2 = ("pipe_jinja2",@@ -12,6 +13,7 @@ type = "PipeJinja2"definition = "Process a Jinja2 template"output = "Text"jinja2 = "Hello {{ name }}!"+template_category = "markdown"""",PipelexBundleBlueprint(domain="test_pipes",@@ -22,6 +24,7 @@ jinja2 = "Hello {{ name }}!"definition="Process a Jinja2 template",output=NativeConceptEnum.TEXT,jinja2="Hello {{ name }}!",+ template_category=Jinja2TemplateCategory.MARKDOWN,),},),diff --git a/tests/unit/pipelex/core/test_data/pipes/operators/llm/pipe_llm.py b/tests/unit/pipelex/core/test_data/pipes/operators/llm/pipe_llm.pyindex 112a341ac..6fcedd3d3 100644--- a/tests/unit/pipelex/core/test_data/pipes/operators/llm/pipe_llm.py+++ b/tests/unit/pipelex/core/test_data/pipes/operators/llm/pipe_llm.py@@ -32,7 +32,7 @@ prompt_template = "Generate a story about a programmer"# PipeLLM with inputs and template variablesPIPE_LLM_WITH_INPUTS = ("pipe_llm_with_inputs",- """domain = "test_pipes"+ '''domain = "test_pipes"definition = "Domain with pipe definitions"[pipe.extract_info]@@ -40,8 +40,12 @@ type = "PipeLLM"definition = "Extract information from text"inputs = { text = "Text", topic = "Text" }output = "Text"-prompt_template = "Extract information about $topic from this text:\\n\\n@text"-""",+prompt_template = """+Extract information about $topic from this text:++@text+"""+''',PipelexBundleBlueprint(domain="test_pipes",definition="Domain with pipe definitions",@@ -51,7 +55,7 @@ prompt_template = "Extract information about $topic from this text:\\n\\n@text"definition="Extract information from text",inputs={"text": "Text", "topic": "Text"},output=NativeConceptEnum.TEXT,- prompt_template="Extract information about $topic from this text:\n\n@text",+ prompt_template="Extract information about $topic from this text:\n\n@text\n",),},),@@ -85,6 +89,34 @@ prompt_template = "Analyze the following data and provide insights"),)+# PipeLLM with long prompt+PIPE_LLM_WITH_LONG_PROMPT = (+ "pipe_llm_with_long_prompt",+ '''domain = "test_pipes"+definition = "Domain with pipe definitions"++[pipe.expert_analysis]+type = "PipeLLM"+definition = "Expert analysis with system prompt"+output = "Text"+prompt_template = """+Extract all articles/items from this invoice text: $extracted_text. For each item find: item name, quantity, unit price, total price, description, and product code if available. Return each article as separate structured data.+"""+''',+ PipelexBundleBlueprint(+ domain="test_pipes",+ definition="Domain with pipe definitions",+ pipe={+ "expert_analysis": PipeLLMBlueprint(+ type="PipeLLM",+ definition="Expert analysis with system prompt",+ output=NativeConceptEnum.TEXT,+ prompt_template="Extract all articles/items from this invoice text: $extracted_text. For each item find: item name, quantity, unit price, total price, description, and product code if available. Return each article as separate structured data.\n",+ ),+ },+ ),+)+# PipeLLM with multiple outputs (nb_output)PIPE_LLM_MULTIPLE_OUTPUTS = ("pipe_llm_multiple_outputs",@@ -95,8 +127,8 @@ definition = "Domain with pipe definitions"type = "PipeLLM"definition = "Generate multiple ideas"output = "Text"-nb_output = 3prompt_template = "Generate creative ideas for a mobile app"+nb_output = 3""",PipelexBundleBlueprint(domain="test_pipes",@@ -116,7 +148,7 @@ prompt_template = "Generate creative ideas for a mobile app"# PipeLLM with dynamic multiple outputsPIPE_LLM_DYNAMIC_MULTIPLE = ("pipe_llm_dynamic_multiple",- """domain = "test_pipes"+ '''domain = "test_pipes"definition = "Domain with pipe definitions"[pipe.brainstorm_solutions]@@ -124,9 +156,9 @@ type = "PipeLLM"definition = "Brainstorm multiple solutions"inputs = { problem = { concept = "Text" } }output = "Text"+prompt_template = "Brainstorm solutions for this problem: $problem"multiple_output = true-prompt_template = "Brainstorm solutions for this problem: @problem"-""",+''',PipelexBundleBlueprint(domain="test_pipes",definition="Domain with pipe definitions",@@ -137,7 +169,7 @@ prompt_template = "Brainstorm solutions for this problem: @problem"inputs={"problem": InputRequirementBlueprint(concept="Text")},output=NativeConceptEnum.TEXT,multiple_output=True,- prompt_template="Brainstorm solutions for this problem: @problem",+ prompt_template="Brainstorm solutions for this problem: $problem",),},),@@ -290,7 +322,7 @@ prompt_template = "Extract person information from this text: @text"# PipeLLM with boolean multiplicity valuesPIPE_LLM_BOOLEAN_MULTIPLICITY = ("pipe_llm_boolean_multiplicity",- """domain = "test_pipes"+ '''domain = "test_pipes"definition = "Domain with pipe definitions"[concept]@@ -301,8 +333,12 @@ type = "PipeLLM"definition = "Analyze multiple documents and single query"inputs = { documents = { concept = "Text", multiplicity = true }, query = { concept = "Text", multiplicity = false } }output = "DocumentSummary"-prompt_template = "Analyze these documents based on the query: $query\\n\\nDocuments: @documents"-""",+prompt_template = """+Analyze these documents based on the query: $query++Documents: @documents+"""+''',PipelexBundleBlueprint(domain="test_pipes",definition="Domain with pipe definitions",@@ -316,7 +352,10 @@ prompt_template = "Analyze these documents based on the query: $query\\n\\nDocum"query": InputRequirementBlueprint(concept="Text", multiplicity=False),},output="DocumentSummary",- prompt_template="Analyze these documents based on the query: $query\n\nDocuments: @documents",+ prompt_template="""Analyze these documents based on the query: $query++Documents: @documents+""",),},),@@ -327,6 +366,7 @@ PIPE_LLM_TEST_CASES = [PIPE_LLM_BASIC,PIPE_LLM_WITH_INPUTS,PIPE_LLM_WITH_SYSTEM_PROMPT,+ PIPE_LLM_WITH_LONG_PROMPT,PIPE_LLM_MULTIPLE_OUTPUTS,PIPE_LLM_DYNAMIC_MULTIPLE,PIPE_LLM_VISION,diff --git a/tests/unit/pipelex/create/__init__.py b/tests/unit/pipelex/language/__init__.pysimilarity index 100%rename from tests/unit/pipelex/create/__init__.pyrename to tests/unit/pipelex/language/__init__.pydiff --git a/tests/unit/pipelex/language/test_plx_factory.py b/tests/unit/pipelex/language/test_plx_factory.pynew file mode 100644index 000000000..4b71c2687--- /dev/null+++ b/tests/unit/pipelex/language/test_plx_factory.py@@ -0,0 +1,454 @@+from pathlib import Path+from typing import Any, Dict, List, Mapping++import pytest+import tomlkit+from tomlkit import array, inline_table, table+from pytest_mock import MockerFixture++from pipelex import log, pretty_print+from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint+from pipelex.language.plx_config import PlxConfig, PlxConfigStrings, PlxConfigInlineTables, PlxConfigForConcepts, PlxConfigForPipes+from pipelex.language.plx_factory import PlxFactory, SectionKey++from tests.unit.pipelex.core.test_data import InterpreterTestCases+++class TestPlxFactoryUnit:+ """Unit tests for PlxFactory methods."""++ @pytest.fixture+ def mock_plx_config(self) -> PlxConfig:+ """Create a mock PLX configuration for testing."""+ return PlxConfig(+ strings=PlxConfigStrings(+ prefer_literal=True,+ force_multiline=False,+ length_limit_to_multiline=50,+ ensure_trailing_newline=True,+ ensure_leading_blank_line=False,+ ),+ inline_tables=PlxConfigInlineTables(+ spaces_inside_curly_braces=True,+ ),+ concepts=PlxConfigForConcepts(+ structure_field_ordering=["type", "definition", "inputs", "output"],+ ),+ pipes=PlxConfigForPipes(+ field_ordering=["type", "definition", "inputs", "output"],+ ),+ )++ @pytest.fixture+ def sample_mapping_data(self) -> Dict[str, Any]:+ """Sample mapping data for testing."""+ return {+ "simple_field": "simple_value",+ "nested_mapping": {+ "inner_key": "inner_value",+ "inner_number": 42,+ },+ "list_field": ["item1", "item2", "item3"],+ "complex_list": [+ {"name": "first", "value": 1},+ {"name": "second", "value": 2},+ ],+ }++ def test_format_tomlkit_string_simple(self, mocker: MockerFixture, mock_plx_config: PlxConfig):+ """Test formatting simple strings."""+ _mock_config = mocker.patch.object(PlxFactory, '_plx_config', return_value=mock_plx_config)++ # Test simple string+ result = PlxFactory.format_tomlkit_string("simple text")+ assert isinstance(result, tomlkit.items.String) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]+ # The actual string value without quotes+ assert result.value == "simple text"++ def test_format_tomlkit_string_multiline(self, mocker: MockerFixture, mock_plx_config: PlxConfig):+ """Test formatting multiline strings."""+ _mock_config = mocker.patch.object(PlxFactory, '_plx_config', return_value=mock_plx_config)++ # Test string with newlines+ multiline_text = "line1\nline2\nline3"+ result = PlxFactory.format_tomlkit_string(multiline_text)+ assert isinstance(result, tomlkit.items.String) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]+ # Should be multiline with trailing newline+ assert result.value == "line1\nline2\nline3\n"+ # Check if it's a multiline string by checking if it has newlines in the value+ assert "\n" in result.value++ def test_format_tomlkit_string_force_multiline(self, mocker: MockerFixture, mock_plx_config: PlxConfig):+ """Test force multiline configuration."""+ mock_plx_config.strings.force_multiline = True+ _mock_config = mocker.patch.object(PlxFactory, '_plx_config', return_value=mock_plx_config)++ result = PlxFactory.format_tomlkit_string("short")+ assert isinstance(result, tomlkit.items.String) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]+ # Should be multiline even for short text+ assert result.value == "short\n"+ # Check if it's a multiline string by checking if it has newlines in the value+ assert "\n" in result.value++ def test_format_tomlkit_string_length_limit(self, mocker: MockerFixture, mock_plx_config: PlxConfig):+ """Test length limit for multiline conversion."""+ mock_plx_config.strings.length_limit_to_multiline = 10+ _mock_config = mocker.patch.object(PlxFactory, '_plx_config', return_value=mock_plx_config)++ long_text = "this is a very long text that exceeds the limit"+ result = PlxFactory.format_tomlkit_string(long_text)+ assert isinstance(result, tomlkit.items.String) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]+ # Should be multiline due to length+ assert result.value == "this is a very long text that exceeds the limit\n"+ # Check if it's a multiline string by checking if it has newlines in the value+ assert "\n" in result.value++ def test_format_tomlkit_string_leading_blank_line(self, mocker: MockerFixture, mock_plx_config: PlxConfig):+ """Test leading blank line configuration."""+ mock_plx_config.strings.ensure_leading_blank_line = True+ mock_plx_config.strings.force_multiline = True+ _mock_config = mocker.patch.object(PlxFactory, '_plx_config', return_value=mock_plx_config)++ result = PlxFactory.format_tomlkit_string("content")+ assert isinstance(result, tomlkit.items.String) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]+ # Should have leading blank line+ assert result.value == "\ncontent\n"+ # Check if it's a multiline string by checking if it has newlines in the value+ assert "\n" in result.value++ def test_convert_dicts_to_inline_tables_simple_dict(self, mocker: MockerFixture, mock_plx_config: PlxConfig):+ """Test converting simple dictionary to inline table."""+ _mock_config = mocker.patch.object(PlxFactory, '_plx_config', return_value=mock_plx_config)++ input_dict = {"key1": "value1", "key2": "value2"}+ result = PlxFactory.convert_dicts_to_inline_tables(input_dict)++ assert isinstance(result, tomlkit.items.InlineTable) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]+ assert result["key1"].value == "value1"+ assert result["key2"].value == "value2"++ def test_convert_dicts_to_inline_tables_with_field_ordering(self, mocker: MockerFixture, mock_plx_config: PlxConfig):+ """Test converting dictionary with field ordering."""+ _mock_config = mocker.patch.object(PlxFactory, '_plx_config', return_value=mock_plx_config)++ input_dict = {"key2": "value2", "key1": "value1", "key3": "value3"}+ field_ordering = ["key1", "key3"]+ result = PlxFactory.convert_dicts_to_inline_tables(input_dict, field_ordering)++ assert isinstance(result, tomlkit.items.InlineTable) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]+ # Check that ordered fields come first+ keys = list(result.keys())+ assert keys[0] == "key1"+ assert keys[1] == "key3"+ # Note: the implementation might not include all keys if not in ordering+ if len(keys) > 2:+ assert keys[2] == "key2"++ def test_convert_dicts_to_inline_tables_nested_dict(self, mocker: MockerFixture, mock_plx_config: PlxConfig):+ """Test converting nested dictionary."""+ _mock_config = mocker.patch.object(PlxFactory, '_plx_config', return_value=mock_plx_config)++ input_dict = {+ "outer": {+ "inner": "value"+ }+ }+ result = PlxFactory.convert_dicts_to_inline_tables(input_dict)++ assert isinstance(result, tomlkit.items.InlineTable) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]+ assert isinstance(result["outer"], tomlkit.items.InlineTable) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]+ assert result["outer"]["inner"].value == "value"++ def test_convert_dicts_to_inline_tables_list_with_dicts(self, mocker: MockerFixture, mock_plx_config: PlxConfig):+ """Test converting list containing dictionaries."""+ _mock_config = mocker.patch.object(PlxFactory, '_plx_config', return_value=mock_plx_config)++ input_list = [+ {"name": "first", "value": 1},+ {"name": "second", "value": 2}+ ]+ result = PlxFactory.convert_dicts_to_inline_tables(input_list)++ assert isinstance(result, tomlkit.items.Array) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]+ assert len(result) == 2+ assert isinstance(result[0], tomlkit.items.InlineTable) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]+ assert result[0]["name"].value == "first"+ assert result[0]["value"] == 1++ def test_convert_dicts_to_inline_tables_string_handling(self, mocker: MockerFixture, mock_plx_config: PlxConfig):+ """Test string handling in conversion."""+ _mock_config = mocker.patch.object(PlxFactory, '_plx_config', return_value=mock_plx_config)++ # Test simple string+ result = PlxFactory.convert_dicts_to_inline_tables("simple string")+ assert isinstance(result, tomlkit.items.String) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]++ # Test other types pass through+ assert PlxFactory.convert_dicts_to_inline_tables(42) == 42+ assert PlxFactory.convert_dicts_to_inline_tables(True) is True++ def test_convert_mapping_to_table(self, mocker: MockerFixture, mock_plx_config: PlxConfig, sample_mapping_data: Dict[str, Any]):+ """Test converting mapping to table."""+ _mock_config = mocker.patch.object(PlxFactory, '_plx_config', return_value=mock_plx_config)++ result = PlxFactory.convert_mapping_to_table(sample_mapping_data)++ assert isinstance(result, tomlkit.items.Table) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]+ assert "simple_field" in result+ assert "nested_mapping" in result+ assert "list_field" in result+ assert "complex_list" in result++ def test_convert_mapping_to_table_with_field_ordering(self, mocker: MockerFixture, mock_plx_config: PlxConfig):+ """Test converting mapping with field ordering."""+ _mock_config = mocker.patch.object(PlxFactory, '_plx_config', return_value=mock_plx_config)++ mapping = {"field3": "value3", "field1": "value1", "field2": "value2"}+ field_ordering = ["field1", "field2"]++ result = PlxFactory.convert_mapping_to_table(mapping, field_ordering)++ assert isinstance(result, tomlkit.items.Table) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]+ # Check ordering (note: tomlkit preserves insertion order)+ keys = list(result.keys())+ assert keys[0] == "field1"+ assert keys[1] == "field2"+ assert keys[2] == "field3"++ def test_convert_mapping_to_table_skips_category(self, mocker: MockerFixture, mock_plx_config: PlxConfig):+ """Test that category field is skipped."""+ _mock_config = mocker.patch.object(PlxFactory, '_plx_config', return_value=mock_plx_config)++ mapping = {"field1": "value1", "category": "should_be_skipped", "field2": "value2"}+ result = PlxFactory.convert_mapping_to_table(mapping)++ assert isinstance(result, tomlkit.items.Table) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]+ assert "field1" in result+ assert "field2" in result+ assert "category" not in result++ def test_add_spaces_to_inline_tables_simple(self):+ """Test adding spaces to simple inline tables."""+ input_toml = "{key = value}"+ result = PlxFactory.add_spaces_to_inline_tables(input_toml)+ assert result == "{ key = value }"++ def test_add_spaces_to_inline_tables_already_spaced(self):+ """Test that already spaced tables are preserved."""+ input_toml = "{ key = value }"+ result = PlxFactory.add_spaces_to_inline_tables(input_toml)+ assert result == "{ key = value }"++ def test_add_spaces_to_inline_tables_nested(self):+ """Test adding spaces to nested inline tables."""+ input_toml = "{outer = {inner = value}}"+ result = PlxFactory.add_spaces_to_inline_tables(input_toml)+ assert result == "{ outer = { inner = value } }"++ def test_add_spaces_to_inline_tables_with_jinja2(self):+ """Test that Jinja2 templates are preserved."""+ input_toml = "template = '{{ variable }}' and {key = value}"+ result = PlxFactory.add_spaces_to_inline_tables(input_toml)+ assert result == "template = '{{ variable }}' and { key = value }"++ def test_add_spaces_to_inline_tables_complex(self):+ """Test complex inline table spacing."""+ input_toml = "config = {db = {host = 'localhost', port = 5432}, cache = {enabled = true}}"+ result = PlxFactory.add_spaces_to_inline_tables(input_toml)+ expected = "config = { db = { host = 'localhost', port = 5432 }, cache = { enabled = true } }"+ assert result == expected++ def test_add_spaces_to_inline_tables_partial_spacing(self):+ """Test partial spacing scenarios."""+ # Left space only+ input_toml = "{ key = value}"+ result = PlxFactory.add_spaces_to_inline_tables(input_toml)+ assert result == "{ key = value }"++ # Right space only+ input_toml = "{key = value }"+ result = PlxFactory.add_spaces_to_inline_tables(input_toml)+ assert result == "{ key = value }"++ def test_make_table_obj_for_pipe(self, mocker: MockerFixture, mock_plx_config: PlxConfig):+ """Test making table object for pipe section."""+ _mock_config = mocker.patch.object(PlxFactory, '_plx_config', return_value=mock_plx_config)++ pipe_data = {+ "type": "PipeLLM",+ "definition": "Test pipe",+ "inputs": {"input1": "Text"},+ "output": "Text",+ "nested_config": {+ "param1": "value1",+ "param2": 42+ }+ }++ result = PlxFactory.make_table_obj_for_pipe(pipe_data)++ assert isinstance(result, tomlkit.items.Table) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]+ assert "type" in result+ assert "definition" in result+ assert "inputs" in result+ assert "output" in result+ assert "nested_config" in result++ def test_make_table_obj_for_concept_simple_string(self, mocker: MockerFixture, mock_plx_config: PlxConfig):+ """Test making table object for concept with simple string definition."""+ _mock_config = mocker.patch.object(PlxFactory, '_plx_config', return_value=mock_plx_config)++ concept_data = {+ "SimpleConcept": "A simple concept definition"+ }++ result = PlxFactory.make_table_obj_for_concept(concept_data)++ assert isinstance(result, tomlkit.items.Table) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]+ assert "SimpleConcept" in result+ assert result["SimpleConcept"] == "A simple concept definition"++ def test_make_table_obj_for_concept_with_structure(self, mocker: MockerFixture, mock_plx_config: PlxConfig):+ """Test making table object for concept with structure."""+ _mock_config = mocker.patch.object(PlxFactory, '_plx_config', return_value=mock_plx_config)++ concept_data = {+ "ComplexConcept": {+ "definition": "A complex concept",+ "structure": {+ "field1": "string",+ "field2": "int"+ }+ }+ }++ result = PlxFactory.make_table_obj_for_concept(concept_data)++ assert isinstance(result, tomlkit.items.Table) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]+ assert "ComplexConcept" in result+ assert isinstance(result["ComplexConcept"], tomlkit.items.Table) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]+ assert "definition" in result["ComplexConcept"]+ assert "structure" in result["ComplexConcept"]++ def test_make_table_obj_for_concept_structure_string(self, mocker: MockerFixture, mock_plx_config: PlxConfig):+ """Test concept with structure as string."""+ _mock_config = mocker.patch.object(PlxFactory, '_plx_config', return_value=mock_plx_config)++ concept_data = {+ "ConceptWithStringStructure": {+ "structure": "SomeClass"+ }+ }++ result = PlxFactory.make_table_obj_for_concept(concept_data)++ assert isinstance(result, tomlkit.items.Table) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]+ concept_table = result["ConceptWithStringStructure"]+ assert isinstance(concept_table, tomlkit.items.Table) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]+ assert concept_table["structure"] == "SomeClass"++ def test_make_table_obj_for_concept_invalid_structure(self, mocker: MockerFixture, mock_plx_config: PlxConfig):+ """Test error handling for invalid structure types."""+ _mock_config = mocker.patch.object(PlxFactory, '_plx_config', return_value=mock_plx_config)++ concept_data = {+ "InvalidConcept": {+ "structure": 123 # Invalid type+ }+ }++ with pytest.raises(TypeError, match="Structure field value is not a mapping"):+ PlxFactory.make_table_obj_for_concept(concept_data)++ def test_make_table_obj_for_concept_invalid_concept_value(self, mocker: MockerFixture, mock_plx_config: PlxConfig):+ """Test error handling for invalid concept value types."""+ _mock_config = mocker.patch.object(PlxFactory, '_plx_config', return_value=mock_plx_config)++ concept_data = {+ "InvalidConcept": 123 # Invalid type+ }++ with pytest.raises(TypeError, match="Concept field value is not a mapping"):+ PlxFactory.make_table_obj_for_concept(concept_data)++ def test_dict_to_plx_styled_toml_with_spacing(self, mocker: MockerFixture, mock_plx_config: PlxConfig):+ """Test dict to PLX styled TOML with spacing enabled."""+ mock_plx_config.inline_tables.spaces_inside_curly_braces = True+ _mock_config = mocker.patch.object(PlxFactory, '_plx_config', return_value=mock_plx_config)+ mock_add_spaces = mocker.patch.object(PlxFactory, 'add_spaces_to_inline_tables', return_value="spaced_output")++ data = {"domain": "test", "definition": "test domain"}++ result = PlxFactory.dict_to_plx_styled_toml(data)++ assert result == "spaced_output"+ mock_add_spaces.assert_called_once()++ def test_dict_to_plx_styled_toml_without_spacing(self, mocker: MockerFixture, mock_plx_config: PlxConfig):+ """Test dict to PLX styled TOML without spacing."""+ mock_plx_config.inline_tables.spaces_inside_curly_braces = False+ _mock_config = mocker.patch.object(PlxFactory, '_plx_config', return_value=mock_plx_config)+ mock_add_spaces = mocker.patch.object(PlxFactory, 'add_spaces_to_inline_tables')++ data = {"domain": "test", "definition": "test domain"}++ result = PlxFactory.dict_to_plx_styled_toml(data)++ # Should not call add_spaces_to_inline_tables+ mock_add_spaces.assert_not_called()+ assert isinstance(result, str)++ def test_dict_to_plx_styled_toml_empty_sections(self, mocker: MockerFixture, mock_plx_config: PlxConfig):+ """Test handling of empty sections."""+ _mock_config = mocker.patch.object(PlxFactory, '_plx_config', return_value=mock_plx_config)++ data: dict[str, Any] = {+ "domain": "test",+ "concept": {}, # Empty concept section+ "pipe": {} # Empty pipe section+ }++ result = PlxFactory.dict_to_plx_styled_toml(data)++ # Empty sections should be skipped+ assert "concept" not in result+ assert "pipe" not in result+ assert "domain" in result++ def test_dict_to_plx_styled_toml_with_pipe_section(self, mocker: MockerFixture, mock_plx_config: PlxConfig):+ """Test dict to PLX styled TOML with pipe section."""+ _mock_config = mocker.patch.object(PlxFactory, '_plx_config', return_value=mock_plx_config)++ data = {+ "domain": "test",+ "pipe": {+ "test_pipe": {+ "type": "PipeLLM",+ "definition": "Test pipe"+ }+ }+ }++ result = PlxFactory.dict_to_plx_styled_toml(data)++ assert "domain" in result+ assert "[pipe.test_pipe]" in result+ assert "type" in result+ assert "definition" in result++ def test_dict_to_plx_styled_toml_with_concept_section(self, mocker: MockerFixture, mock_plx_config: PlxConfig):+ """Test dict to PLX styled TOML with concept section."""+ _mock_config = mocker.patch.object(PlxFactory, '_plx_config', return_value=mock_plx_config)++ data = {+ "domain": "test",+ "concept": {+ "TestConcept": "A test concept"+ }+ }++ result = PlxFactory.dict_to_plx_styled_toml(data)++ assert "domain" in result+ assert "[concept]" in result+ assert "TestConcept" in resultdiff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_data.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_data.pyindex 25023d6eb..63fbca416 100644--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_data.py+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_data.py@@ -9,8 +9,8 @@ from pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprintclass PipeParallelTestCases:- SIMPLE_PARALLEL = (- "simple_parallel",+ PARALLEL_WITH_EACH_OUTPUT = (+ "parallel_with_each_output",PipeParallelSpec(the_pipe_code="parallel_processor",definition="Run pipes in parallel",@@ -21,6 +21,7 @@ class PipeParallelTestCases:SubPipeSpec(the_pipe_code="transform_data", result="transformed"),SubPipeSpec(the_pipe_code="validate_data", result="validation"),],+ add_each_output=True,),PipeParallelBlueprint(definition="Run pipes in parallel",@@ -35,6 +36,7 @@ class PipeParallelTestCases:],add_each_output=True,combined_output=None,+ source="tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_data.py PipeParallelTestCases.PARALLEL_WITH_EACH_OUTPUT",),)@@ -64,10 +66,42 @@ class PipeParallelTestCases:],add_each_output=False,combined_output="MergedData",+ source="tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_data.py PipeParallelTestCases.PARALLEL_WITH_COMBINED",+ ),+ )++ PARALLEL_WITH_BOTH_EACH_OUTPUT_AND_COMBINED = (+ "parallel_with_both_each_output_and_combined",+ PipeParallelSpec(+ the_pipe_code="combined_parallel",+ definition="Parallel with combined output",+ inputs={"input": InputRequirementSpec(concept="Input")},+ output="CombinedResult",+ parallels=[+ SubPipeSpec(the_pipe_code="pipe1", result="result1"),+ SubPipeSpec(the_pipe_code="pipe2", result="result2"),+ ],+ add_each_output=True,+ combined_output="MergedData",+ ),+ PipeParallelBlueprint(+ definition="Parallel with combined output",+ inputs={"input": InputRequirementBlueprint(concept="Input")},+ output="CombinedResult",+ type="PipeParallel",+ category="PipeController",+ parallels=[+ SubPipeBlueprint(pipe="pipe1", result="result1"),+ SubPipeBlueprint(pipe="pipe2", result="result2"),+ ],+ add_each_output=True,+ combined_output="MergedData",+ source="tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_data.py PipeParallelTestCases.PARALLEL_WITH_BOTH_EACH_OUTPUT_AND_COMBINED",),)TEST_CASES: ClassVar[list[tuple[str, PipeParallelSpec, PipeParallelBlueprint]]] = [- SIMPLE_PARALLEL,+ PARALLEL_WITH_EACH_OUTPUT,PARALLEL_WITH_COMBINED,+ PARALLEL_WITH_BOTH_EACH_OUTPUT_AND_COMBINED,]diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_pipe_parallel.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_pipe_parallel.pyindex 04d999a7f..e35dd1669 100644--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_pipe_parallel.py+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_pipe_parallel.py@@ -4,7 +4,7 @@ from pipelex.libraries.pipelines.builder.pipe.pipe_parallel_spec import PipeParafrom pipelex.pipe_controllers.parallel.pipe_parallel_blueprint import PipeParallelBlueprintfrom .test_data import PipeParallelTestCases-+from pipelex import logclass TestPipeParallelBlueprintConversion:@pytest.mark.parametrize(@@ -17,5 +17,7 @@ class TestPipeParallelBlueprintConversion:pipe_spec: PipeParallelSpec,expected_blueprint: PipeParallelBlueprint,):+ log.verbose(f"Testing {test_name}")result = pipe_spec.to_blueprint()+ expected_blueprint.source = Noneassert result == expected_blueprintdiff --git a/tests/unit/pipelex/tools/misc/test_toml_utils.py b/tests/unit/pipelex/tools/misc/test_toml_utils.pyindex a19a20c18..9980dea8e 100644--- a/tests/unit/pipelex/tools/misc/test_toml_utils.py+++ b/tests/unit/pipelex/tools/misc/test_toml_utils.py@@ -4,7 +4,7 @@ import pytestfrom pipelex.tools.misc.toml_utils import (TOMLValidationError,- failable_load_toml_from_path,+ load_toml_from_path_if_exists,load_toml_from_path,validate_toml_file,)@@ -104,12 +104,12 @@ definition = "Test with trailing space"assert isinstance(result, dict)assert result["domain"] == "test"- def test_failable_load_toml_from_path_nonexistent_file(self) -> None:+ def test_load_toml_from_path_if_exists_nonexistent_file(self) -> None:"""Test failable loading with non-existent file."""- result = failable_load_toml_from_path("/nonexistent/path.toml")+ result = load_toml_from_path_if_exists("/nonexistent/path.toml")assert result is None- def test_failable_load_toml_from_path_valid_file(self, tmp_path: Path) -> None:+ def test_load_toml_from_path_if_exists_valid_file(self, tmp_path: Path) -> None:"""Test failable loading with valid file."""toml_content = """domain = "test"definition = "Test definition"@@ -117,12 +117,12 @@ definition = "Test definition"toml_file = tmp_path / "valid.toml"toml_file.write_text(toml_content)- result = failable_load_toml_from_path(str(toml_file))+ result = load_toml_from_path_if_exists(str(toml_file))assert result is not Noneassert result["domain"] == "test"- def test_failable_load_toml_from_path_with_whitespace_works(self, tmp_path: Path) -> None:+ def test_load_toml_from_path_if_exists_with_whitespace_works(self, tmp_path: Path) -> None:"""Test failable loading works normally with whitespace."""toml_content = """domain = "test""""@@ -130,7 +130,7 @@ definition = "Test definition"toml_file.write_text(toml_content)# Should work fine - loading doesn't validate by default- result = failable_load_toml_from_path(str(toml_file))+ result = load_toml_from_path_if_exists(str(toml_file))assert result is not Noneassert result["domain"] == "test"