Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

Use tomlkit to generate PLX and improve library error handling (#295)

Louis Choquel844fa8d3db4b
diff --git a/.badges/tests.json b/.badges/tests.json
index d43b5371e..056566a85 100644
--- a/.badges/tests.json
+++ b/.badges/tests.json
@@ -1,7 +1,7 @@
{
- "schemaVersion": 1,
- "label": "tests",
- "message": "1121",
- "color": "blue",
- "cacheSeconds": 300
- }
\ No newline at end of file
+ "schemaVersion": 1,
+ "label": "tests",
+ "message": "1045",
+ "color": "blue",
+ "cacheSeconds": 300
+}
\ No newline at end of file
diff --git a/Makefile b/Makefile
index d9b38e0e3..ab8a0cd3a 100644
--- a/Makefile
+++ b/Makefile
@@ -455,11 +455,11 @@ docs-deploy: env
### SHORTHANDS
##########################################################################################
-c: format lint pyright pylint mypy
+c: format lint pyright mypy pylint
@echo "> done: c = check"
cc: cleanderived c
- @echo "> done: cc = cleanderived format lint pyright pylint mypy"
+ @echo "> done: cc = cleanderived format lint pyright mypy pylint"
check: cc check-unused-imports
@echo "> done: check"
diff --git a/pipelex/cli/commands/build_cmd.py b/pipelex/cli/commands/build_cmd.py
index fb64deadb..567c83224 100644
--- a/pipelex/cli/commands/build_cmd.py
+++ b/pipelex/cli/commands/build_cmd.py
@@ -5,11 +5,12 @@ from typing import Annotated
import typer
from pipelex import pretty_print
-from pipelex.core.interpreter import PipelexInterpreter
from pipelex.hub import get_report_delegate
+from pipelex.language.plx_factory import PlxFactory
from pipelex.libraries.pipelines.builder.builder import PipelexBundleSpec
from pipelex.pipelex import Pipelex
from pipelex.pipeline.execute import execute_pipeline
+from pipelex.tools.misc.file_utils import ensure_directory_for_file_path, save_text_to_path
build_app = typer.Typer(help="Build artifacts like pipelines", no_args_is_help=True)
@@ -30,9 +31,13 @@ def build_pipe_cmd(
typer.Argument(help="Brief description of what the pipeline should do"),
],
output_path: Annotated[
- str | None,
- typer.Option("--output", "-o", help="Path to save the generated PLX file (use --output='' to skip saving)"),
- ] = "./generated_pipeline.plx",
+ str,
+ typer.Option("--output", "-o", help="Path to save the generated PLX file"),
+ ] = "./results/generated_pipeline.plx",
+ no_output: Annotated[
+ bool,
+ typer.Option("--no-output", help="Skip saving the pipeline to file"),
+ ] = False,
) -> None:
Pipelex.make(relative_config_folder_path="../../../pipelex/libraries", from_file=True)
typer.echo("=" * 70)
@@ -40,21 +45,29 @@ def build_pipe_cmd(
typer.echo("")
async def run_pipeline():
+ if not no_output:
+ if not output_path:
+ typer.echo(typer.style("\n⚠️ Cannot save a pipeline to an empty file name", fg=typer.colors.RED))
+ raise typer.Exit(1)
+
+ ensure_directory_for_file_path(file_path=output_path)
+ else:
+ typer.echo(typer.style("\n⚠️ Pipeline will not be saved to file (--no-output specified)", fg=typer.colors.YELLOW))
+
pipe_output = await execute_pipeline(
pipe_code="pipe_builder",
input_memory={"brief": brief},
)
pretty_print(pipe_output, title="Pipe Output")
pipelex_bundle_spec = pipe_output.working_memory.get_stuff_as(name="pipelex_bundle_spec", content_type=PipelexBundleSpec)
- plx_content = PipelexInterpreter.make_plx_content(blueprint=pipelex_bundle_spec.to_blueprint())
+ plx_content = PlxFactory.make_plx_content(blueprint=pipelex_bundle_spec.to_blueprint())
- # Save to file unless explicitly disabled with empty string
- if output_path and output_path != "":
- with open(output_path, "w") as f:
- f.write(plx_content)
+ # Save to file unless explicitly disabled with --no-output
+ if not no_output:
+ save_text_to_path(text=plx_content, path=output_path)
typer.echo(typer.style(f"\n✅ Pipeline saved to: {output_path}", fg=typer.colors.GREEN))
- elif output_path == "":
- typer.echo(typer.style("\n⚠️ Pipeline not saved to file (--output='' specified)", fg=typer.colors.YELLOW))
+ else:
+ typer.echo(typer.style("\n⚠️ Pipeline not saved to file (--no-output specified)", fg=typer.colors.YELLOW))
start_time = time.time()
asyncio.run(run_pipeline())
diff --git a/pipelex/cli/commands/your-pipe.plx b/pipelex/cli/commands/your-pipe.plx
deleted file mode 100644
index e69de29bb..000000000
diff --git a/pipelex/config.py b/pipelex/config.py
index a41a7def0..96d062275 100644
--- a/pipelex/config.py
+++ b/pipelex/config.py
@@ -7,6 +7,7 @@ from pipelex.cogt.config_cogt import Cogt
from pipelex.cogt.model_backends.prompting_target import PromptingTarget
from pipelex.exceptions import PipelexConfigError, StaticValidationErrorType
from pipelex.hub import get_required_config
+from pipelex.language.plx_config import PlxConfig
from pipelex.libraries.library_config import LibraryConfig
from pipelex.pipeline.track.tracker_config import TrackerConfig
from pipelex.tools.aws.aws_config import AwsConfig
@@ -110,6 +111,7 @@ class Pipelex(ConfigModel):
tracker_config: TrackerConfig
structure_config: StructureConfig
prompting_config: PromptingConfig
+ plx_config: PlxConfig
dry_run_config: DryRunConfig
pipe_run_config: PipeRunConfig
diff --git a/pipelex/core/bundles/pipelex_bundle_blueprint.py b/pipelex/core/bundles/pipelex_bundle_blueprint.py
index 66ce37b84..91cb86c39 100644
--- a/pipelex/core/bundles/pipelex_bundle_blueprint.py
+++ b/pipelex/core/bundles/pipelex_bundle_blueprint.py
@@ -31,6 +31,7 @@ PipeBlueprintUnion = Annotated[
class PipelexBundleBlueprint(BaseModel):
model_config = ConfigDict(extra="forbid")
+ source: str | None = None
domain: str
definition: str | None = None
system_prompt: str | None = None
diff --git a/pipelex/core/concepts/concept_blueprint.py b/pipelex/core/concepts/concept_blueprint.py
index 493d4c7b0..eaec48747 100644
--- a/pipelex/core/concepts/concept_blueprint.py
+++ b/pipelex/core/concepts/concept_blueprint.py
@@ -35,7 +35,7 @@ class ConceptStructureBlueprint(BaseModel):
item_type: str | None = None
key_type: str | None = None
value_type: str | None = None
- choices: list[str] | None = Field(default_factory=list)
+ choices: list[str] | None = Field(default=None)
required: bool | None = Field(default=True)
default_value: Any | None = None
diff --git a/pipelex/core/concepts/concept_factory.py b/pipelex/core/concepts/concept_factory.py
index f501974be..b80758831 100644
--- a/pipelex/core/concepts/concept_factory.py
+++ b/pipelex/core/concepts/concept_factory.py
@@ -13,7 +13,7 @@ from pipelex.core.concepts.concept_native import NativeConceptEnumData, NativeCo
from pipelex.core.concepts.structure_generator import StructureGenerator
from pipelex.core.domains.domain import SpecialDomain
from pipelex.core.stuffs.stuff_content import TextContent
-from pipelex.exceptions import ConceptFactoryError, StructureClassError
+from pipelex.exceptions import ConceptCodeError, ConceptDefinitionError, ConceptRefineError, StructureClassError
class DomainAndConceptCode(BaseModel):
@@ -106,16 +106,10 @@ class ConceptFactory:
@classmethod
def make_refine(cls, refine: str) -> str:
- if NativeConceptManager.is_native_concept(concept_string_or_code=refine):
- return NativeConceptManager.get_native_concept_string(concept_string_or_code=refine)
- msg = f"Refine '{refine}' is not a native concept"
- raise ConceptFactoryError(msg)
-
- @classmethod
- def make_refines(cls, blueprint: ConceptBlueprint) -> str | None:
- if blueprint.refines:
- return cls.make_refine(refine=blueprint.refines)
- return None
+ if not NativeConceptManager.is_native_concept(concept_string_or_code=refine):
+ msg = f"Refine '{refine}' is not a native concept"
+ raise ConceptRefineError(msg)
+ return NativeConceptManager.get_native_concept_string(concept_string_or_code=refine)
@classmethod
def make_from_blueprint(
@@ -125,7 +119,11 @@ class ConceptFactory:
blueprint: ConceptBlueprint,
concept_codes_from_the_same_domain: list[str] | None = None,
) -> Concept:
- ConceptBlueprint.validate_concept_code(concept_code=concept_code)
+ try:
+ ConceptBlueprint.validate_concept_code(concept_code=concept_code)
+ except ConceptCodeError as exc:
+ msg = f"Could not validate concept code '{concept_code}' in domain '{domain}': {exc}"
+ raise ConceptDefinitionError(msg) from exc
structure_class_name: str
current_refine: str | None = None
@@ -163,7 +161,7 @@ class ConceptFactory:
except Exception as exc:
msg = f"Error generating structure class for concept '{concept_code}' in domain '{domain}': {exc}"
- raise ConceptFactoryError(msg) from exc
+ raise ConceptDefinitionError(msg) from exc
# Handle refines definition
elif blueprint.refines:
@@ -173,8 +171,12 @@ class ConceptFactory:
f"Concept '{concept_code}' in domain '{domain}' has refines but also has a structure class registered. "
"A concept cannot have both structure and refines."
)
- raise ConceptFactoryError(msg)
- current_refine = cls.make_refines(blueprint=blueprint)
+ raise ConceptDefinitionError(msg)
+ try:
+ current_refine = cls.make_refine(refine=blueprint.refines)
+ except ConceptRefineError as exc:
+ msg = f"Could not validate refine '{blueprint.refines}' for concept '{concept_code}' in domain '{domain}': {exc}"
+ raise ConceptDefinitionError(msg) from exc
structure_class_name = current_refine.split(".")[1] + "Content" if current_refine else TextContent.__name__
# Handle neither structure nor refines - check the class registry
# If there is a class, use it. structure_class_name is then the concept_code
diff --git a/pipelex/core/domains/domain_factory.py b/pipelex/core/domains/domain_factory.py
index 449349ea2..bc9dfc7f6 100644
--- a/pipelex/core/domains/domain_factory.py
+++ b/pipelex/core/domains/domain_factory.py
@@ -1,14 +1,23 @@
+from pydantic import ValidationError
+
from pipelex.core.domains.domain import Domain
from pipelex.core.domains.domain_blueprint import DomainBlueprint
+from pipelex.exceptions import DomainDefinitionError
+from pipelex.tools.typing.pydantic_utils import format_pydantic_validation_error
class DomainFactory:
@classmethod
def make_from_blueprint(cls, blueprint: DomainBlueprint) -> Domain:
- return Domain(
- code=blueprint.code,
- definition=blueprint.definition,
- system_prompt=blueprint.system_prompt,
- system_prompt_to_structure=blueprint.system_prompt_to_structure,
- prompt_template_to_structure=blueprint.prompt_template_to_structure,
- )
+ try:
+ return Domain(
+ code=blueprint.code,
+ definition=blueprint.definition,
+ system_prompt=blueprint.system_prompt,
+ system_prompt_to_structure=blueprint.system_prompt_to_structure,
+ prompt_template_to_structure=blueprint.prompt_template_to_structure,
+ )
+ except ValidationError as exc:
+ formatted_error_msg = format_pydantic_validation_error(exc)
+ msg = f"Could not make domain from blueprint: {formatted_error_msg}"
+ raise DomainDefinitionError(msg) from exc
diff --git a/pipelex/core/domains/domain_library.py b/pipelex/core/domains/domain_library.py
index f9a0abe14..9574d0026 100644
--- a/pipelex/core/domains/domain_library.py
+++ b/pipelex/core/domains/domain_library.py
@@ -28,6 +28,10 @@ class DomainLibrary(RootModel[DomainLibraryRoot], DomainProviderAbstract):
else:
self.root[domain_code] = domain
+ def add_domains(self, domains: list[Domain]):
+ for domain in domains:
+ self.add_domain(domain=domain)
+
def remove_domain_by_code(self, domain_code: str) -> None:
if domain_code in self.root:
del self.root[domain_code]
diff --git a/pipelex/core/interpreter.py b/pipelex/core/interpreter.py
index 0d6d9509b..383ce55e0 100644
--- a/pipelex/core/interpreter.py
+++ b/pipelex/core/interpreter.py
@@ -1,32 +1,13 @@
-from collections.abc import Mapping
from pathlib import Path
-from typing import Any, cast
+from typing import Any
import toml
from pydantic import BaseModel, model_validator
from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint
-from pipelex.core.concepts.concept_blueprint import ConceptBlueprint, ConceptStructureBlueprint
from pipelex.core.exceptions import (
PipelexConfigurationError,
- PipelexUnknownPipeError,
)
-from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
-from pipelex.pipe_controllers.batch.pipe_batch_blueprint import PipeBatchBlueprint
-from pipelex.pipe_controllers.condition.pipe_condition_blueprint import (
- PipeConditionBlueprint,
-)
-from pipelex.pipe_controllers.parallel.pipe_parallel_blueprint import (
- PipeParallelBlueprint,
-)
-from pipelex.pipe_controllers.sequence.pipe_sequence_blueprint import (
- PipeSequenceBlueprint,
-)
-from pipelex.pipe_operators.func.pipe_func_blueprint import PipeFuncBlueprint
-from pipelex.pipe_operators.img_gen.pipe_img_gen_blueprint import PipeImgGenBlueprint
-from pipelex.pipe_operators.jinja2.pipe_jinja2_blueprint import PipeJinja2Blueprint
-from pipelex.pipe_operators.llm.pipe_llm_blueprint import PipeLLMBlueprint
-from pipelex.pipe_operators.ocr.pipe_ocr_blueprint import PipeOcrBlueprint
from pipelex.tools.misc.toml_utils import (
clean_trailing_whitespace,
validate_toml_content,
@@ -142,866 +123,3 @@ class PipelexInterpreter(BaseModel):
file_content = self._read_and_reformat()
plx_data = self._parse_plx_content(file_content)
return PipelexBundleBlueprint.model_validate(plx_data)
-
- @staticmethod
- def make_plx_content(blueprint: PipelexBundleBlueprint) -> str:
- """Convert a PipelexBundleBlueprint to properly formatted PLX content."""
- plx_parts: list[str] = []
-
- # Domain-level fields
- domain_fields: list[str] = []
- domain_fields.append(f'domain = "{PipelexInterpreter.escape_plx_string(blueprint.domain)}"')
- if blueprint.definition:
- domain_fields.append(f'definition = "{PipelexInterpreter.escape_plx_string(blueprint.definition)}"')
- if blueprint.system_prompt:
- domain_fields.append(f'system_prompt = "{PipelexInterpreter.escape_plx_string(blueprint.system_prompt)}"')
- if blueprint.system_prompt_to_structure:
- domain_fields.append(f'system_prompt_to_structure = "{PipelexInterpreter.escape_plx_string(blueprint.system_prompt_to_structure)}"')
- if blueprint.prompt_template_to_structure:
- domain_fields.append(f'prompt_template_to_structure = "{PipelexInterpreter.escape_plx_string(blueprint.prompt_template_to_structure)}"')
-
- plx_parts.append("\n".join(domain_fields))
-
- # Concepts section
- if blueprint.concept:
- concept_plx = PipelexInterpreter.concepts_to_plx_string(blueprint.concept)
- if concept_plx: # Only add if not empty
- plx_parts.append(concept_plx)
-
- # Pipes section
- if blueprint.pipe:
- pipes_plx = PipelexInterpreter.pipes_to_plx_string(blueprint.pipe)
- if pipes_plx: # Only add if not empty
- plx_parts.append(pipes_plx)
-
- result = "\n\n".join(plx_parts)
- if result and not result.endswith("\n"):
- result += "\n"
- return result
-
- @staticmethod
- def concepts_to_plx_string(concepts: dict[str, ConceptBlueprint | str]) -> str:
- """Convert concepts dict to PLX string."""
- if not concepts:
- return ""
-
- plx_parts: list[str] = []
- simple_concepts: list[str] = []
- complex_concepts: list[str] = []
-
- for concept_name, concept_blueprint in concepts.items():
- if isinstance(concept_blueprint, str):
- simple_concepts.append(f'{concept_name} = "{PipelexInterpreter.escape_plx_string(concept_blueprint)}"')
- # Handle ConceptBlueprint objects
- elif concept_blueprint.structure is None and concept_blueprint.refines is None:
- # Simple concept with just definition
- simple_concepts.append(f'{concept_name} = "{PipelexInterpreter.escape_plx_string(concept_blueprint.definition)}"')
- else:
- # Complex concept needs its own section
- complex_concept_plx = PipelexInterpreter.complex_concept_to_plx_string(concept_name, concept_blueprint)
- complex_concepts.append(complex_concept_plx)
-
- # Add simple concepts section if we have any
- if simple_concepts:
- plx_parts.append("[concept]")
- plx_parts.extend(simple_concepts)
-
- # Add complex concepts with proper spacing
- if complex_concepts:
- # If we had simple concepts, add an empty line before complex ones
- if simple_concepts:
- plx_parts.append("")
- # Add each complex concept with empty lines between them
- for i, complex_concept in enumerate(complex_concepts):
- if i > 0: # Add empty line between complex concepts
- plx_parts.append("")
- plx_parts.append(complex_concept)
-
- return "\n".join(plx_parts)
-
- @staticmethod
- def complex_concept_to_plx_string(concept_name: str, concept: ConceptBlueprint) -> str:
- """Convert a complex ConceptBlueprint to PLX string."""
- lines: list[str] = [f"[concept.{concept_name}]"]
-
- if concept.definition:
- lines.append(f'definition = "{PipelexInterpreter.escape_plx_string(concept.definition)}"')
-
- if concept.refines:
- lines.append(f'refines = "{PipelexInterpreter.escape_plx_string(concept.refines)}"')
-
- if concept.structure:
- if isinstance(concept.structure, str):
- lines.append(f'structure = "{PipelexInterpreter.escape_plx_string(concept.structure)}"')
- else:
- # Complex structure with fields
- lines.append("")
- lines.append(f"[concept.{concept_name}.structure]")
- for field_name, field_def in concept.structure.items():
- field_plx = PipelexInterpreter.structure_field_to_plx_string(field_name, field_def)
- lines.append(field_plx)
-
- return "\n".join(lines)
-
- @staticmethod
- def structure_field_to_plx_string(field_name: str, field_def: Any) -> str:
- """Convert a structure field to PLX string."""
- if isinstance(field_def, str):
- return f'{field_name} = "{PipelexInterpreter.escape_plx_string(field_def)}"'
- # Complex field with type, definition, required, etc.
- field_parts: list[str] = []
-
- if hasattr(field_def, "type") and field_def.type:
- type_value = field_def.type.value if hasattr(field_def.type, "value") else field_def.type
- field_parts.append(f'type = "{PipelexInterpreter.escape_plx_string(str(type_value))}"')
-
- if hasattr(field_def, "definition") and field_def.definition:
- field_parts.append(f'definition = "{PipelexInterpreter.escape_plx_string(field_def.definition)}"')
-
- if hasattr(field_def, "required") and field_def.required is not None:
- field_parts.append(f"required = {str(field_def.required).lower()}")
-
- return f"{field_name} = {{ {', '.join(field_parts)} }}"
-
- @staticmethod
- def pipes_to_plx_string(pipes: dict[str, Any]) -> str:
- """Convert pipes dict to PLX string."""
- plx_parts: list[str] = []
- for pipe_name, blueprint in pipes.items():
- pipe_plx = PipelexInterpreter.pipe_to_plx_string(pipe_name, blueprint)
- plx_parts.append(pipe_plx)
- return "\n\n".join(plx_parts)
-
- @staticmethod
- def pipe_to_plx_string(pipe_name: str, blueprint: Any) -> str:
- """Convert a single pipe blueprint to PLX string."""
- if isinstance(blueprint, PipeLLMBlueprint):
- return PipelexInterpreter.llm_pipe_to_plx_string(pipe_name, blueprint)
- elif isinstance(blueprint, PipeSequenceBlueprint):
- return PipelexInterpreter.sequence_pipe_to_plx_string(pipe_name, blueprint)
- elif isinstance(blueprint, PipeOcrBlueprint):
- return PipelexInterpreter.ocr_pipe_to_plx_string(pipe_name, blueprint)
- elif isinstance(blueprint, PipeFuncBlueprint):
- return PipelexInterpreter.func_pipe_to_plx_string(pipe_name, blueprint)
- elif isinstance(blueprint, PipeImgGenBlueprint):
- return PipelexInterpreter.img_gen_pipe_to_plx_string(pipe_name, blueprint)
- elif isinstance(blueprint, PipeJinja2Blueprint):
- return PipelexInterpreter.jinja2_pipe_to_plx_string(pipe_name, blueprint)
- elif isinstance(blueprint, PipeConditionBlueprint):
- return PipelexInterpreter.condition_pipe_to_plx_string(pipe_name, blueprint)
- elif isinstance(blueprint, PipeParallelBlueprint):
- return PipelexInterpreter.parallel_pipe_to_plx_string(pipe_name, blueprint)
- elif isinstance(blueprint, PipeBatchBlueprint):
- return PipelexInterpreter.batch_pipe_to_plx_string(pipe_name, blueprint)
- else:
- # Fallback to old dict approach for unknown pipe types
- pipe_dict = PipelexInterpreter.serialize_pipe(blueprint)
- return f"[pipe.{pipe_name}]\n" + "\n".join([f'{k} = "{v}"' if isinstance(v, str) else f"{k} = {v}" for k, v in pipe_dict.items()])
-
- @staticmethod
- def llm_pipe_to_plx_string(pipe_name: str, pipe: PipeLLMBlueprint) -> str:
- """Convert a PipeLLM blueprint directly to PLX section string."""
- lines: list[str] = [
- f"[pipe.{pipe_name}]",
- f'type = "{PipelexInterpreter.escape_plx_string(pipe.type)}"',
- f'definition = "{PipelexInterpreter.escape_plx_string(pipe.definition)}"',
- ]
-
- # Add inputs first if they exist
- PipelexInterpreter.add_inputs_to_lines_if_exist(lines, pipe.inputs)
-
- # Add output after inputs (or immediately if no inputs)
- lines.append(f'output = "{PipelexInterpreter.escape_plx_string(pipe.output)}"')
-
- # Add optional fields
- if pipe.nb_output is not None:
- lines.append(f"nb_output = {pipe.nb_output}")
- if pipe.multiple_output is not None:
- lines.append(f"multiple_output = {str(pipe.multiple_output).lower()}")
- if pipe.system_prompt_template:
- lines.append(f'system_prompt_template = "{PipelexInterpreter.escape_plx_string(pipe.system_prompt_template)}"')
- if pipe.system_prompt:
- lines.append(f'system_prompt = "{PipelexInterpreter.escape_plx_string(pipe.system_prompt)}"')
- if pipe.prompt_template:
- lines.append(f'prompt_template = "{PipelexInterpreter.escape_plx_string(pipe.prompt_template)}"')
- if pipe.template_name:
- lines.append(f'template_name = "{PipelexInterpreter.escape_plx_string(pipe.template_name)}"')
- if pipe.prompt:
- lines.append(f'prompt = "{PipelexInterpreter.escape_plx_string(pipe.prompt)}"')
-
- return "\n".join(lines)
-
- @staticmethod
- def ocr_pipe_to_plx_string(pipe_name: str, pipe: PipeOcrBlueprint) -> str:
- """Convert a PipeOcr blueprint directly to PLX section string."""
- lines: list[str] = [
- f"[pipe.{pipe_name}]",
- f'type = "{pipe.type}"',
- f'definition = "{pipe.definition}"',
- ]
-
- # Add inputs if they exist
- PipelexInterpreter.add_inputs_to_lines_if_exist(lines, pipe.inputs)
-
- lines.append(f'output = "{PipelexInterpreter.escape_plx_string(pipe.output)}"')
- lines.append(f'ocr = "{pipe.ocr}"')
-
- return "\n".join(lines)
-
- @staticmethod
- def func_pipe_to_plx_string(pipe_name: str, pipe: PipeFuncBlueprint) -> str:
- """Convert a PipeFunc blueprint directly to PLX section string."""
- lines: list[str] = [
- f"[pipe.{pipe_name}]",
- f'type = "{pipe.type}"',
- f'definition = "{pipe.definition}"',
- ]
-
- # Add inputs if they exist
- PipelexInterpreter.add_inputs_to_lines_if_exist(lines, pipe.inputs)
-
- lines.append(f'output = "{PipelexInterpreter.escape_plx_string(pipe.output)}"')
- lines.append(f'function_name = "{PipelexInterpreter.escape_plx_string(pipe.function_name)}"')
-
- return "\n".join(lines)
-
- @staticmethod
- def img_gen_pipe_to_plx_string(pipe_name: str, pipe: PipeImgGenBlueprint) -> str:
- """Convert a PipeImgGen blueprint directly to PLX section string."""
- lines: list[str] = [
- f"[pipe.{pipe_name}]",
- f'type = "{pipe.type}"',
- f'definition = "{pipe.definition}"',
- ]
-
- # Add inputs if they exist
- PipelexInterpreter.add_inputs_to_lines_if_exist(lines, pipe.inputs)
-
- lines.append(f'output = "{PipelexInterpreter.escape_plx_string(pipe.output)}"')
-
- # Add optional fields
- if pipe.img_gen_prompt:
- lines.append(f'img_gen_prompt = "{PipelexInterpreter.escape_plx_string(pipe.img_gen_prompt)}"')
- if pipe.aspect_ratio:
- lines.append(f'aspect_ratio = "{pipe.aspect_ratio}"')
- if pipe.is_raw is not None:
- lines.append(f"is_raw = {str(pipe.is_raw).lower()}")
- if pipe.seed:
- lines.append(f"seed = {pipe.seed}")
- if pipe.nb_output:
- lines.append(f"nb_output = {pipe.nb_output}")
-
- return "\n".join(lines)
-
- @staticmethod
- def jinja2_pipe_to_plx_string(pipe_name: str, pipe: PipeJinja2Blueprint) -> str:
- """Convert a PipeJinja2 blueprint directly to PLX section string."""
- lines: list[str] = [
- f"[pipe.{pipe_name}]",
- f'type = "{pipe.type}"',
- f'definition = "{pipe.definition}"',
- ]
-
- # Add inputs if they exist
- PipelexInterpreter.add_inputs_to_lines_if_exist(lines, pipe.inputs)
-
- lines.append(f'output = "{PipelexInterpreter.escape_plx_string(pipe.output)}"')
-
- # Add jinja2 template
- if pipe.jinja2:
- lines.append(f'jinja2 = "{PipelexInterpreter.escape_plx_string(pipe.jinja2)}"')
- if pipe.jinja2_name:
- lines.append(f'jinja2_name = "{PipelexInterpreter.escape_plx_string(pipe.jinja2_name)}"')
-
- return "\n".join(lines)
-
- @staticmethod
- def condition_pipe_to_plx_string(pipe_name: str, pipe: PipeConditionBlueprint) -> str:
- """Convert a PipeCondition blueprint directly to PLX section string."""
- lines: list[str] = [
- f"[pipe.{pipe_name}]",
- f'type = "{pipe.type}"',
- f'definition = "{pipe.definition}"',
- ]
-
- # Add inputs if they exist
- PipelexInterpreter.add_inputs_to_lines_if_exist(lines, pipe.inputs)
-
- lines.append(f'output = "{PipelexInterpreter.escape_plx_string(pipe.output)}"')
-
- # Add pipe_map
- if pipe.pipe_map:
- pipe_map_parts: list[str] = []
- pipe_map_dict = pipe.pipe_map.root
- for key, value in pipe_map_dict.items():
- pipe_map_parts.append(f'{key} = "{value}"')
- lines.append(f"pipe_map = {{ {', '.join(pipe_map_parts)} }}")
-
- # Add optional fields
- if pipe.expression_template:
- lines.append(f'expression_template = "{pipe.expression_template}"')
- if pipe.expression:
- lines.append(f'expression = "{pipe.expression}"')
- if pipe.default_pipe_code:
- lines.append(f'default_pipe_code = "{pipe.default_pipe_code}"')
- if pipe.add_alias_from_expression_to:
- lines.append(f'add_alias_from_expression_to = "{pipe.add_alias_from_expression_to}"')
-
- return "\n".join(lines)
-
- @staticmethod
- def parallel_pipe_to_plx_string(pipe_name: str, pipe: PipeParallelBlueprint) -> str:
- """Convert a PipeParallel blueprint directly to PLX section string."""
- lines: list[str] = [
- f"[pipe.{pipe_name}]",
- f'type = "{pipe.type}"',
- f'definition = "{pipe.definition}"',
- ]
-
- # Add inputs if they exist
- PipelexInterpreter.add_inputs_to_lines_if_exist(lines, pipe.inputs)
-
- lines.append(f'output = "{PipelexInterpreter.escape_plx_string(pipe.output)}"')
-
- # Add parallels array
- if pipe.parallels:
- lines.append("parallels = [")
- for parallel in pipe.parallels:
- parallel_string = PipelexInterpreter.sub_pipe_to_plx_string(parallel)
- lines.append(f" {parallel_string},")
- lines.append("]")
-
- # Add optional fields
- if pipe.add_each_output is not True: # Only include if not default True
- lines.append(f"add_each_output = {str(pipe.add_each_output).lower()}")
- if pipe.combined_output:
- lines.append(f'combined_output = "{pipe.combined_output}"')
-
- return "\n".join(lines)
-
- @staticmethod
- def batch_pipe_to_plx_string(pipe_name: str, pipe: PipeBatchBlueprint) -> str:
- """Convert a PipeBatch blueprint directly to PLX section string."""
- lines: list[str] = [
- f"[pipe.{pipe_name}]",
- f'type = "{pipe.type}"',
- f'definition = "{pipe.definition}"',
- ]
-
- # Add inputs if they exist
- PipelexInterpreter.add_inputs_to_lines_if_exist(lines, pipe.inputs)
-
- lines.append(f'output = "{PipelexInterpreter.escape_plx_string(pipe.output)}"')
- lines.append(f'branch_pipe_code = "{pipe.branch_pipe_code}"')
-
- # Add optional fields
- if pipe.input_list_name:
- lines.append(f'input_list_name = "{pipe.input_list_name}"')
- if pipe.input_item_name:
- lines.append(f'input_item_name = "{pipe.input_item_name}"')
-
- return "\n".join(lines)
-
- @staticmethod
- def serialize_concept_structure_field(field_value: Any) -> Any:
- """Serialize a single concept structure field."""
- if isinstance(field_value, str):
- return field_value
- if isinstance(field_value, ConceptStructureBlueprint):
- field_data: dict[str, Any] = {
- "definition": field_value.definition,
- "required": field_value.required,
- }
- if field_value.type is not None:
- field_data["type"] = field_value.type
- if field_value.item_type is not None:
- field_data["item_type"] = field_value.item_type
- if field_value.key_type is not None:
- field_data["key_type"] = field_value.key_type
- if field_value.value_type is not None:
- field_data["value_type"] = field_value.value_type
- if field_value.choices:
- field_data["choices"] = field_value.choices
- if field_value.default_value is not None:
- field_data["default_value"] = field_value.default_value
- return field_data
- return field_value
-
- @staticmethod
- def serialize_concept_structure(
- structure: str | dict[str, Any],
- ) -> str | dict[str, Any]:
- """Serialize concept structure field."""
- if isinstance(structure, str):
- return structure
- result: dict[str, Any] = {}
- for field_name, field_value in structure.items():
- result[field_name] = PipelexInterpreter.serialize_concept_structure_field(field_value)
- return result
-
- @staticmethod
- def serialize_single_concept(
- concept_blueprint: ConceptBlueprint | str,
- ) -> str | dict[str, Any]:
- """Serialize a single concept blueprint."""
- if isinstance(concept_blueprint, str):
- return concept_blueprint
-
- # Handle ConceptBlueprint object
- if concept_blueprint.structure is not None:
- # Structured concept
- return {
- "definition": concept_blueprint.definition,
- "structure": PipelexInterpreter.serialize_concept_structure(concept_blueprint.structure),
- }
- if concept_blueprint.refines is not None:
- # Concept with refines
- return {
- "definition": concept_blueprint.definition,
- "refines": concept_blueprint.refines,
- }
- # Simple concept with just definition
- return concept_blueprint.definition
-
- @staticmethod
- def serialize_concepts(concepts: dict[str, ConceptBlueprint | str] | None) -> dict[str, Any]:
- """Serialize concepts section with domain context."""
- if concepts is None:
- return {}
-
- result: dict[str, Any] = {}
- for concept_name, concept_blueprint in concepts.items():
- result[concept_name] = PipelexInterpreter.serialize_single_concept(concept_blueprint)
- return result
-
- @staticmethod
- def serialize_pipes(pipes: dict[str, Any]) -> dict[str, Any]:
- """Serialize pipes section with domain context."""
- result: dict[str, Any] = {}
- for pipe_name, blueprint in pipes.items():
- result[pipe_name] = PipelexInterpreter.serialize_pipe(blueprint)
- return result
-
- @staticmethod
- def serialize_pipe(blueprint: Any) -> dict[str, Any]:
- if isinstance(blueprint, PipeLLMBlueprint):
- return PipelexInterpreter.serialize_llm_pipe(blueprint)
- elif isinstance(blueprint, PipeOcrBlueprint):
- return PipelexInterpreter.serialize_ocr_pipe(blueprint)
- elif isinstance(blueprint, PipeFuncBlueprint):
- return PipelexInterpreter._serialize_func_pipe(blueprint)
- elif isinstance(blueprint, PipeImgGenBlueprint):
- return PipelexInterpreter._serialize_img_gen_pipe(blueprint)
- elif isinstance(blueprint, PipeJinja2Blueprint):
- return PipelexInterpreter.serialize_jinja2_pipe(blueprint)
- elif isinstance(blueprint, PipeSequenceBlueprint):
- return PipelexInterpreter.serialize_sequence_pipe(blueprint)
- elif isinstance(blueprint, PipeConditionBlueprint):
- return PipelexInterpreter._serialize_condition_pipe(blueprint)
- elif isinstance(blueprint, PipeParallelBlueprint):
- return PipelexInterpreter._serialize_parallel_pipe(blueprint)
- elif isinstance(blueprint, PipeBatchBlueprint):
- return PipelexInterpreter._serialize_batch_pipe(blueprint)
- else:
- msg = f"Unknown pipe blueprint type: {type(blueprint)}"
- raise PipelexUnknownPipeError(msg)
-
- @staticmethod
- def serialize_llm_pipe(pipe: PipeLLMBlueprint) -> dict[str, Any]:
- """Serialize PipeLLM blueprint."""
- result: dict[str, Any] = {
- "type": pipe.type,
- "definition": pipe.definition,
- }
-
- # Add inputs first if they exist
- if pipe.inputs:
- result["inputs"] = PipelexInterpreter.serialize_inputs(pipe.inputs)
-
- # Then output
- result["output"] = pipe.output
-
- # Add optional fields in expected order based on test cases
- if pipe.nb_output is not None:
- result["nb_output"] = pipe.nb_output
- if pipe.multiple_output is not None:
- result["multiple_output"] = pipe.multiple_output
-
- if pipe.system_prompt_template:
- result["system_prompt_template"] = pipe.system_prompt_template
- if pipe.system_prompt:
- result["system_prompt"] = pipe.system_prompt
-
- if pipe.prompt_template:
- result["prompt_template"] = pipe.prompt_template
- if pipe.template_name:
- result["template_name"] = pipe.template_name
- if pipe.prompt:
- result["prompt"] = pipe.prompt
-
- # Less common fields
- if pipe.system_prompt_template_name:
- result["system_prompt_template_name"] = pipe.system_prompt_template_name
- if pipe.system_prompt_name:
- result["system_prompt_name"] = pipe.system_prompt_name
- if pipe.prompt_name:
- result["prompt_name"] = pipe.prompt_name
- if pipe.llm:
- result["llm"] = pipe.llm
- if pipe.llm_to_structure:
- result["llm_to_structure"] = pipe.llm_to_structure
- if pipe.structuring_method:
- result["structuring_method"] = pipe.structuring_method
- if pipe.prompt_template_to_structure:
- result["prompt_template_to_structure"] = pipe.prompt_template_to_structure
- if pipe.system_prompt_to_structure:
- result["system_prompt_to_structure"] = pipe.system_prompt_to_structure
-
- return result
-
- @staticmethod
- def add_common_pipe_fields(result: dict[str, Any], pipe: Any) -> None:
- """Add common pipe fields (inputs) to result dict if they exist."""
- if hasattr(pipe, "inputs") and pipe.inputs:
- result["inputs"] = PipelexInterpreter.serialize_inputs(pipe.inputs)
-
- @staticmethod
- def _serialize_ocr_pipe(pipe: PipeOcrBlueprint) -> dict[str, Any]:
- """Serialize PipeOcr blueprint."""
- result: dict[str, Any] = {
- "type": pipe.type,
- "definition": pipe.definition,
- "output": pipe.output,
- }
-
- PipelexInterpreter.add_common_pipe_fields(result, pipe)
- return result
-
- @staticmethod
- def _serialize_func_pipe(pipe: PipeFuncBlueprint) -> dict[str, Any]:
- """Serialize PipeFunc blueprint."""
- result: dict[str, Any] = {
- "type": pipe.type,
- "definition": pipe.definition,
- "output": pipe.output,
- "function_name": pipe.function_name,
- }
-
- PipelexInterpreter.add_common_pipe_fields(result, pipe)
- return result
-
- @staticmethod
- def _serialize_img_gen_pipe(pipe: PipeImgGenBlueprint) -> dict[str, Any]:
- """Serialize PipeImgGen blueprint."""
- result: dict[str, Any] = {
- "type": pipe.type,
- "definition": pipe.definition,
- "output": pipe.output,
- }
-
- PipelexInterpreter.add_common_pipe_fields(result, pipe)
-
- # Add optional fields only if they have values
- if pipe.img_gen_prompt:
- result["img_gen_prompt"] = pipe.img_gen_prompt
- if pipe.aspect_ratio:
- result["aspect_ratio"] = pipe.aspect_ratio
- if pipe.is_raw is not None:
- result["is_raw"] = pipe.is_raw
- if pipe.seed:
- result["seed"] = pipe.seed
- if pipe.nb_output:
- result["nb_output"] = pipe.nb_output
-
- return result
-
- @staticmethod
- def serialize_ocr_pipe(pipe: PipeOcrBlueprint) -> dict[str, Any]:
- """Serialize a PipeOcr blueprint."""
- result: dict[str, Any] = {
- "type": pipe.type,
- "definition": pipe.definition,
- }
-
- # Add common pipe fields
- PipelexInterpreter.add_common_pipe_fields(result, pipe)
-
- # Add output
- result["output"] = pipe.output
-
- # Add required fields
- result["ocr"] = pipe.ocr
-
- # Add optional fields
- if pipe.page_images:
- result["page_images"] = pipe.page_images
- if pipe.page_image_captions:
- result["page_image_captions"] = pipe.page_image_captions
- if pipe.page_views:
- result["page_views"] = pipe.page_views
- if pipe.page_views_dpi:
- result["page_views_dpi"] = pipe.page_views_dpi
-
- return result
-
- @staticmethod
- def serialize_sequence_pipe(pipe: PipeSequenceBlueprint) -> dict[str, Any]:
- """Serialize a PipeSequence blueprint."""
- result: dict[str, Any] = {
- "type": pipe.type,
- "definition": pipe.definition,
- }
-
- # Add common pipe fields
- PipelexInterpreter.add_common_pipe_fields(result, pipe)
-
- # Add output
- result["output"] = pipe.output
-
- # Add steps
- if pipe.steps:
- result["steps"] = [PipelexInterpreter.serialize_sub_pipe(step) for step in pipe.steps]
-
- return result
-
- @staticmethod
- def serialize_jinja2_pipe(pipe: PipeJinja2Blueprint) -> dict[str, Any]:
- """Serialize PipeJinja2 blueprint."""
- result: dict[str, Any] = {
- "type": pipe.type,
- "definition": pipe.definition,
- "output": pipe.output,
- }
-
- PipelexInterpreter.add_common_pipe_fields(result, pipe)
-
- # Add optional fields only if they have values
- if pipe.jinja2_name:
- result["jinja2_name"] = pipe.jinja2_name
- if pipe.jinja2:
- result["jinja2"] = pipe.jinja2
- if pipe.prompting_style:
- result["prompting_style"] = pipe.prompting_style
- result["template_category"] = pipe.template_category
-
- return result
-
- @staticmethod
- def serialize_sub_pipe(sub_pipe: Any) -> dict[str, Any]:
- """Serialize a sub pipe (step or parallel) blueprint."""
- step_data: dict[str, Any] = {
- "pipe": sub_pipe.pipe,
- "result": sub_pipe.result,
- }
- # Only include optional fields if they have non-default values
- if sub_pipe.nb_output is not None:
- step_data["nb_output"] = sub_pipe.nb_output
- if sub_pipe.multiple_output is not None:
- step_data["multiple_output"] = sub_pipe.multiple_output
- if sub_pipe.batch_over is not False: # Only include if not default False
- step_data["batch_over"] = sub_pipe.batch_over
- if sub_pipe.batch_as is not None:
- step_data["batch_as"] = sub_pipe.batch_as
- return step_data
-
- @staticmethod
- def sub_pipe_to_plx_string(sub_pipe: Any) -> str:
- """Convert a sub pipe blueprint directly to a PLX inline table string."""
- parts = [f'pipe = "{sub_pipe.pipe}"', f'result = "{sub_pipe.result}"']
-
- # Add optional fields if they have non-default values
- if sub_pipe.nb_output is not None:
- parts.append(f"nb_output = {sub_pipe.nb_output}")
- if sub_pipe.multiple_output is not None:
- parts.append(f"multiple_output = {str(sub_pipe.multiple_output).lower()}")
- if sub_pipe.batch_over is not False: # Only include if not default False
- if isinstance(sub_pipe.batch_over, str):
- parts.append(f'batch_over = "{sub_pipe.batch_over}"')
- else:
- parts.append(f"batch_over = {str(sub_pipe.batch_over).lower()}")
- if sub_pipe.batch_as is not None:
- parts.append(f'batch_as = "{sub_pipe.batch_as}"')
-
- return "{ " + ", ".join(parts) + " }"
-
- @staticmethod
- def _serialize_sequence_pipe(pipe: PipeSequenceBlueprint) -> dict[str, Any]:
- """Serialize PipeSequence blueprint."""
- result: dict[str, Any] = {
- "type": pipe.type,
- "definition": pipe.definition,
- "output": pipe.output,
- }
-
- PipelexInterpreter.add_common_pipe_fields(result, pipe)
-
- if pipe.steps:
- result["steps"] = [PipelexInterpreter.serialize_sub_pipe(step) for step in pipe.steps]
-
- return result
-
- @staticmethod
- def sequence_pipe_to_plx_string(pipe_name: str, pipe: PipeSequenceBlueprint) -> str:
- """Convert a PipeSequence blueprint directly to PLX section string."""
- lines: list[str] = [
- f"[pipe.{pipe_name}]",
- f'type = "{pipe.type}"',
- f'definition = "{pipe.definition}"',
- ]
-
- # Add inputs first if they exist
- if hasattr(pipe, "inputs"):
- PipelexInterpreter.add_inputs_to_lines_if_exist(lines, pipe.inputs)
-
- # Add output after inputs
- lines.append(f'output = "{pipe.output}"')
-
- # Add steps array with proper spacing
- if pipe.steps:
- lines.append("steps = [")
- for step in pipe.steps:
- step_string = PipelexInterpreter.sub_pipe_to_plx_string(step)
- lines.append(f" {step_string},")
- lines.append("]")
-
- return "\n".join(lines)
-
- @staticmethod
- def _serialize_condition_pipe(pipe: PipeConditionBlueprint) -> dict[str, Any]:
- """Serialize PipeCondition blueprint."""
- result: dict[str, Any] = {
- "type": pipe.type,
- "definition": pipe.definition,
- "output": pipe.output,
- "pipe_map": pipe.pipe_map,
- }
-
- PipelexInterpreter.add_common_pipe_fields(result, pipe)
-
- # Add optional fields only if they have values
- if pipe.expression_template:
- result["expression_template"] = pipe.expression_template
- if pipe.expression:
- result["expression"] = pipe.expression
- if pipe.default_pipe_code:
- result["default_pipe_code"] = pipe.default_pipe_code
- if pipe.add_alias_from_expression_to:
- result["add_alias_from_expression_to"] = pipe.add_alias_from_expression_to
-
- return result
-
- @staticmethod
- def _serialize_parallel_pipe(pipe: PipeParallelBlueprint) -> dict[str, Any]:
- """Serialize PipeParallel blueprint."""
- result: dict[str, Any] = {
- "type": pipe.type,
- "definition": pipe.definition,
- "output": pipe.output,
- }
-
- PipelexInterpreter.add_common_pipe_fields(result, pipe)
-
- if pipe.parallels:
- result["parallels"] = [PipelexInterpreter.serialize_sub_pipe(parallel) for parallel in pipe.parallels]
- if pipe.add_each_output is not True: # Only include if not default True
- result["add_each_output"] = pipe.add_each_output
- if pipe.combined_output:
- result["combined_output"] = pipe.combined_output
-
- return result
-
- @staticmethod
- def _serialize_batch_pipe(pipe: PipeBatchBlueprint) -> dict[str, Any]:
- """Serialize PipeBatch blueprint."""
- result: dict[str, Any] = {
- "type": pipe.type,
- "definition": pipe.definition,
- "output": pipe.output,
- "branch_pipe_code": pipe.branch_pipe_code,
- }
-
- PipelexInterpreter.add_common_pipe_fields(result, pipe)
-
- if pipe.input_list_name:
- result["input_list_name"] = pipe.input_list_name
- if pipe.input_item_name:
- result["input_item_name"] = pipe.input_item_name
-
- return result
-
- @staticmethod
- def serialize_input_requirement(
- input_req: InputRequirementBlueprint,
- ) -> dict[str, Any]:
- """Serialize a single InputRequirementBlueprint to PLX format."""
- result: dict[str, Any] = {"concept": input_req.concept}
- if input_req.multiplicity is not None:
- result["multiplicity"] = input_req.multiplicity
- return result
-
- @staticmethod
- def serialize_inputs(
- inputs: Mapping[str, str | InputRequirementBlueprint],
- ) -> dict[str, Any]:
- """Convert InputRequirementBlueprint objects to proper PLX format for serialization."""
- result: dict[str, Any] = {}
- for key, value in inputs.items():
- if isinstance(value, InputRequirementBlueprint):
- result[key] = PipelexInterpreter.serialize_input_requirement(value)
- else:
- # Already a string - simple case
- result[key] = str(value)
- return result
-
- @staticmethod
- def inputs_to_plx_string(
- inputs: Mapping[str, str | InputRequirementBlueprint],
- ) -> str:
- """Convert inputs dictionary to PLX string format."""
- inputs_dict = PipelexInterpreter.serialize_inputs(inputs)
- inputs_parts: list[str] = []
-
- for key, value in inputs_dict.items():
- if isinstance(value, dict):
- # Nested dict (like InputRequirementBlueprint)
- nested_parts: list[str] = []
- nested_dict: dict[str, Any] = cast("dict[str, Any]", value)
- for nested_key, nested_value in nested_dict.items():
- if isinstance(nested_value, str):
- nested_parts.append(f'{nested_key} = "{PipelexInterpreter.escape_plx_string(nested_value)}"')
- else:
- nested_parts.append(f"{nested_key} = {str(nested_value).lower()}")
- inputs_parts.append(f"{key} = {{ {', '.join(nested_parts)} }}")
- # Simple string value
- elif isinstance(value, str):
- inputs_parts.append(f'{key} = "{PipelexInterpreter.escape_plx_string(value)}"')
- else:
- # Fallback for any other type
- inputs_parts.append(f'{key} = "{PipelexInterpreter.escape_plx_string(str(value))}"')
-
- return f"{{ {', '.join(inputs_parts)} }}"
-
- @staticmethod
- def add_inputs_to_lines_if_exist(
- lines: list[str],
- pipe_inputs: Mapping[str, str | InputRequirementBlueprint] | None,
- ) -> None:
- """Add inputs line to PLX lines if inputs exist."""
- if pipe_inputs:
- inputs_dict = PipelexInterpreter.serialize_inputs(pipe_inputs)
- inputs_parts: list[str] = []
- for key, value in inputs_dict.items():
- if isinstance(value, dict):
- nested_parts: list[str] = []
- nested_dict: dict[str, Any] = cast("dict[str, Any]", value)
- for nested_key, nested_value in nested_dict.items():
- if isinstance(nested_value, str):
- nested_parts.append(f'{nested_key} = "{PipelexInterpreter.escape_plx_string(nested_value)}"')
- else:
- nested_parts.append(f"{nested_key} = {str(nested_value).lower()}")
- inputs_parts.append(f"{key} = {{ {', '.join(nested_parts)} }}")
- elif isinstance(value, str):
- inputs_parts.append(f'{key} = "{PipelexInterpreter.escape_plx_string(value)}"')
- else:
- inputs_parts.append(f'{key} = "{PipelexInterpreter.escape_plx_string(str(value))}"')
- lines.append(f"inputs = {{ {', '.join(inputs_parts)} }}")
diff --git a/pipelex/core/pipes/pipe_blueprint.py b/pipelex/core/pipes/pipe_blueprint.py
index d684c82ed..07e1c5fe8 100644
--- a/pipelex/core/pipes/pipe_blueprint.py
+++ b/pipelex/core/pipes/pipe_blueprint.py
@@ -37,6 +37,7 @@ class AllowedPipeTypes(StrEnum):
class PipeBlueprint(BaseModel):
+ source: str | None = None
category: Any
type: Any # TODO: Find a better way to handle this.
definition: str | None = None
diff --git a/pipelex/core/pipes/pipe_run_params.py b/pipelex/core/pipes/pipe_run_params.py
index 2a71d40d4..34c0ac93e 100644
--- a/pipelex/core/pipes/pipe_run_params.py
+++ b/pipelex/core/pipes/pipe_run_params.py
@@ -6,6 +6,7 @@ from pydantic import BaseModel, Field, field_validator
from pipelex import log
from pipelex.core.memory.working_memory import BATCH_ITEM_STUFF_NAME, MAIN_STUFF_NAME
+from pipelex.exceptions import BatchParamsError
from pipelex.types import Self, StrEnum
@@ -177,22 +178,20 @@ class BatchParams(BaseModel):
@classmethod
def make_optional_batch_params(
cls,
- input_list_name: bool | str,
- input_item_name: str | None = None,
+ input_list_name: str | None,
+ input_item_name: str | None,
) -> BatchParams | None:
- the_batch_params: BatchParams | None = None
- if input_list_name or input_item_name:
- input_list_stuff_name: str
- if isinstance(input_list_name, str):
- input_list_stuff_name = input_list_name
- else:
- input_list_stuff_name = MAIN_STUFF_NAME
+ if input_list_name:
input_item_stuff_name = input_item_name or BATCH_ITEM_STUFF_NAME
- the_batch_params = BatchParams(
- input_list_stuff_name=input_list_stuff_name,
+ return BatchParams(
+ input_list_stuff_name=input_list_name,
input_item_stuff_name=input_item_stuff_name,
)
- return the_batch_params
+ elif input_item_name:
+ msg = "input_item_name is required when input_list_name is not provided"
+ raise BatchParamsError(msg)
+ else:
+ return None
@classmethod
def make_default(cls) -> BatchParams:
diff --git a/pipelex/exceptions.py b/pipelex/exceptions.py
index 1b83b4d3e..c8dfbdb45 100644
--- a/pipelex/exceptions.py
+++ b/pipelex/exceptions.py
@@ -104,19 +104,15 @@ class ClientAuthenticationError(PipelexError):
pass
-class DomainDefinitionError(PipelexError):
- pass
-
-
class ConceptLibraryConceptNotFoundError(PipelexError):
pass
-class ConceptFactoryError(PipelexError):
+class LibraryError(PipelexError):
pass
-class LibraryError(PipelexError):
+class LibraryLoadingError(LibraryError):
pass
@@ -144,10 +140,48 @@ class LibraryParsingError(LibraryError):
pass
-class PipeDefinitionError(PipelexError):
+# TODO: add details from all cases raising this error
+class DomainDefinitionError(PipelexError):
pass
+# TODO: add details from all cases raising this error
+class ConceptDefinitionError(PipelexError):
+ pass
+
+
+# TODO: add details from all cases raising this error
+class PipeDefinitionError(PipelexError):
+ def __init__(
+ self,
+ message: str,
+ domain: str | None = None,
+ pipe_code: str | None = None,
+ description: str | None = None,
+ source: str | None = None,
+ ):
+ self.domain = domain
+ self.pipe_code = pipe_code
+ self.description = description
+ self.source = source
+ message = message + " • " + self.pipe_details()
+ super().__init__(message)
+
+ def pipe_details(self) -> str:
+ if not self.domain and not self.pipe_code and not self.description and not self.source:
+ return "No pipe details provided"
+ details = "Pipe details:"
+ if self.domain:
+ details += f" • domain='{self.domain}'"
+ if self.pipe_code:
+ details += f" • pipe='{self.pipe_code}'"
+ if self.description:
+ details += f" • description='{self.description}'"
+ if self.source:
+ details += f" • source='{self.source}'"
+ return details
+
+
class UnexpectedPipeDefinitionError(PipeDefinitionError):
pass
@@ -187,6 +221,10 @@ class DryRunError(PipeExecutionError):
super().__init__(message)
+class BatchParamsError(PipelexError):
+ pass
+
+
class PipeConditionError(PipelexError):
pass
@@ -239,6 +277,10 @@ class ConceptCodeError(ConceptError):
pass
+class ConceptRefineError(ConceptError):
+ pass
+
+
class PipelineManagerNotFoundError(PipelexError):
pass
diff --git a/tests/unit/pipelex/core/concepts/concept_factory.py/__init__.py b/pipelex/language/__init__.py
similarity index 100%
rename from tests/unit/pipelex/core/concepts/concept_factory.py/__init__.py
rename to pipelex/language/__init__.py
diff --git a/pipelex/language/plx_config.py b/pipelex/language/plx_config.py
new file mode 100644
index 000000000..f5a8f65ea
--- /dev/null
+++ b/pipelex/language/plx_config.py
@@ -0,0 +1,28 @@
+from pipelex.tools.config.config_model import ConfigModel
+
+
+class PlxConfigStrings(ConfigModel):
+ prefer_literal: bool
+ force_multiline: bool
+ length_limit_to_multiline: int
+ ensure_trailing_newline: bool
+ ensure_leading_blank_line: bool
+
+
+class PlxConfigInlineTables(ConfigModel):
+ spaces_inside_curly_braces: bool
+
+
+class PlxConfigForConcepts(ConfigModel):
+ structure_field_ordering: list[str]
+
+
+class PlxConfigForPipes(ConfigModel):
+ field_ordering: list[str]
+
+
+class PlxConfig(ConfigModel):
+ strings: PlxConfigStrings
+ inline_tables: PlxConfigInlineTables
+ concepts: PlxConfigForConcepts
+ pipes: PlxConfigForPipes
diff --git a/pipelex/language/plx_factory.py b/pipelex/language/plx_factory.py
new file mode 100644
index 000000000..294985259
--- /dev/null
+++ b/pipelex/language/plx_factory.py
@@ -0,0 +1,305 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, Mapping, cast
+
+import tomlkit
+from tomlkit import array, document, inline_table, table
+from tomlkit import string as tomlkit_string
+
+from pipelex import log
+from pipelex.config import PlxConfig, get_config
+from pipelex.tools.misc.json_utils import remove_none_values_from_dict
+from pipelex.types import StrEnum
+
+if TYPE_CHECKING:
+ from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint
+
+
+class SectionKey(StrEnum):
+ PIPE = "pipe"
+ CONCEPT = "concept"
+
+
+CONCEPT_STRUCTURE_FIELD_KEY = "structure"
+
+
+class PlxFactory:
+ @classmethod
+ def _plx_config(cls) -> PlxConfig:
+ return get_config().pipelex.plx_config
+
+ @classmethod
+ def format_tomlkit_string(cls, text: str) -> Any: # Can't type this because of tomlkit
+ r"""Build a tomlkit string node.
+ - If `force_multiline` or the text contains '\\n', we emit a triple-quoted multiline string.
+ - When multiline, `ensure_trailing_newline` puts the closing quotes on their own line.
+ - When multiline, `ensure_leading_blank_line` inserts a real blank line at the start of the string.
+ """
+ strings_config = cls._plx_config().strings
+
+ needs_multiline = strings_config.force_multiline or ("\n" in text) or len(text) > strings_config.length_limit_to_multiline
+ normalized = text
+
+ if needs_multiline:
+ if strings_config.ensure_leading_blank_line and not normalized.startswith("\n"):
+ normalized = "\n" + normalized
+ if strings_config.ensure_trailing_newline and not normalized.endswith("\n"):
+ normalized = normalized + "\n"
+
+ use_literal = strings_config.prefer_literal and ("'''" not in normalized)
+ return tomlkit_string(normalized, multiline=needs_multiline, literal=use_literal)
+
+ @classmethod
+ def convert_dicts_to_inline_tables(cls, value: Any, field_ordering: list[str] | None = None) -> Any: # Can't type this because of tomlkit
+ """Recursively convert Python values; dicts -> inline tables; lists kept as arrays."""
+ if isinstance(value, Mapping):
+ value = cast("Mapping[str, Any]", value)
+ inline_table_obj = inline_table()
+ if field_ordering:
+ for key in field_ordering:
+ if key in value:
+ inline_table_obj[key] = cls.convert_dicts_to_inline_tables(value=value[key])
+ else:
+ for key, value_item in value.items():
+ inline_table_obj[key] = cls.convert_dicts_to_inline_tables(value=value_item)
+ return inline_table_obj
+
+ elif isinstance(value, list):
+ value = cast("list[Any]", value)
+ array_obj = array()
+ array_obj.multiline(True) # set to False for single-line arrays
+ for element in value:
+ if isinstance(element, Mapping):
+ element = cast("Mapping[str, Any]", element)
+ inline_element = inline_table()
+ for inner_key, inner_value in element.items():
+ inline_element[inner_key] = cls.convert_dicts_to_inline_tables(value=inner_value)
+ array_obj.append(inline_element) # pyright: ignore[reportUnknownMemberType]
+ else:
+ array_obj.append(cls.convert_dicts_to_inline_tables(value=element)) # pyright: ignore[reportUnknownMemberType]
+ return array_obj
+
+ elif isinstance(value, str):
+ return cls.format_tomlkit_string(text=value)
+ else:
+ return value
+
+ @classmethod
+ def convert_mapping_to_table(
+ cls, mapping: Mapping[str, Any], field_ordering: list[str] | None = None
+ ) -> Any: # Can't type this because of tomlkit
+ """Convert a mapping into a TOML Table where any nested mappings (third level+)
+ are converted to inline tables.
+
+ This creates a second-level standard table, and only uses inline tables for
+ third level and deeper structures.
+ """
+ tbl = table()
+
+ # If field ordering is provided, add fields in the specified order first
+ if field_ordering:
+ for field_key in field_ordering:
+ if field_key in mapping and field_key != "category": # Skip category field
+ field_value = mapping[field_key]
+ if isinstance(field_value, Mapping):
+ # Third-level mapping -> inline table
+ tbl.add(field_key, cls.convert_dicts_to_inline_tables(field_value))
+ else:
+ tbl.add(field_key, cls.convert_dicts_to_inline_tables(field_value))
+
+ # Add any remaining fields not in the ordering
+ for field_key, field_value in mapping.items():
+ if field_key not in field_ordering and field_key != "category":
+ if isinstance(field_value, Mapping):
+ # Third-level mapping -> inline table
+ tbl.add(field_key, cls.convert_dicts_to_inline_tables(field_value))
+ else:
+ tbl.add(field_key, cls.convert_dicts_to_inline_tables(field_value))
+ else:
+ # No field ordering provided, use original logic
+ for field_key, field_value in mapping.items():
+ # Skip the category field as it's not needed in PLX output
+ if field_key == "category":
+ continue
+
+ if isinstance(field_value, Mapping):
+ # Third-level mapping -> inline table
+ tbl.add(field_key, cls.convert_dicts_to_inline_tables(field_value))
+ else:
+ tbl.add(field_key, cls.convert_dicts_to_inline_tables(field_value))
+ return tbl
+
+ # TODO: replace this by a proper toml formatter utility
+ @classmethod
+ def add_spaces_to_inline_tables(cls, toml_string: str) -> str:
+ """Add spaces inside inline table curly braces.
+
+ Converts {key = value} to { key = value }.
+ Only adds spaces if they're not already present.
+ Handles nested inline tables properly.
+ """
+
+ def find_and_replace_inline_tables(text: str) -> str:
+ """Find inline tables using proper brace matching and add spaces."""
+ result = ""
+ char_index = 0
+
+ while char_index < len(text):
+ if text[char_index] == "{":
+ # Check if this is a Jinja2 template (double braces)
+ if char_index + 1 < len(text) and text[char_index + 1] == "{":
+ # This is a Jinja2 template, find the closing }}
+ jinja_end = text.find("}}", char_index + 2)
+ if jinja_end != -1:
+ # Add the entire Jinja2 template as-is
+ result += text[char_index : jinja_end + 2]
+ char_index = jinja_end + 2
+ continue
+
+ # Found start of inline table, find the matching closing brace
+ brace_count = 1
+ start = char_index
+ char_index += 1
+
+ while char_index < len(text) and brace_count > 0:
+ if text[char_index] == "{":
+ brace_count += 1
+ elif text[char_index] == "}":
+ brace_count -= 1
+ char_index += 1
+
+ if brace_count == 0:
+ # Found complete inline table
+ content = text[start + 1 : char_index - 1] # Content between braces
+
+ # Recursively process nested inline tables first
+ content = find_and_replace_inline_tables(content)
+
+ # Add spaces if not already present
+ if content.startswith(" ") and content.endswith(" "):
+ result += "{" + content + "}"
+ elif content.startswith(" "):
+ result += "{" + content + " }"
+ elif content.endswith(" "):
+ result += "{ " + content + "}"
+ else:
+ result += "{ " + content + " }"
+ else:
+ # Unmatched brace, add as-is
+ result += text[start:char_index]
+ else:
+ result += text[char_index]
+ char_index += 1
+
+ return result
+
+ return find_and_replace_inline_tables(toml_string)
+
+ @classmethod
+ def make_table_obj_for_pipe(cls, section_value: Mapping[str, Any]) -> Any:
+ """Make a table object for a pipe section."""
+ log.debug("******** Making table object for pipe section ********")
+ table_obj = table()
+ for field_key, field_value in section_value.items():
+ log.debug(f"------ Field {field_key} is a {type(field_value)}")
+ if not isinstance(field_value, Mapping):
+ log.debug(f"Field is not a mapping: key = {field_key}, value = {field_value}")
+ table_obj.add(field_key, cls.convert_dicts_to_inline_tables(field_value))
+ continue
+ log.debug(f"Field is a mapping: key = {field_key}, value = {field_value}")
+ field_value = cast("Mapping[str, Any]", field_value)
+ table_obj.add(field_key, cls.convert_mapping_to_table(field_value, field_ordering=cls._plx_config().pipes.field_ordering))
+ return table_obj
+
+ @classmethod
+ def make_table_obj_for_concept(cls, section_value: Mapping[str, Any]) -> Any:
+ """Make a table object for a concept section."""
+ log.debug("******** Making table object for concept section ********")
+ table_obj = table()
+ for concept_key, concept_value in section_value.items():
+ if isinstance(concept_value, str):
+ log.debug(f"Concept '{concept_key}' is a string: {concept_value}")
+ table_obj.add(concept_key, concept_value)
+ continue
+ if not isinstance(concept_value, Mapping):
+ msg = f"Concept field value is not a mapping: key = {concept_key}, value = {concept_value}"
+ raise TypeError(msg)
+ log.debug(f"Concept '{concept_key}' is a mapping: {concept_value}")
+ concept_value = cast("Mapping[str, Any]", concept_value)
+ concept_table_obj = table()
+ for concept_field_key, concept_field_value in concept_value.items():
+ if concept_field_key == CONCEPT_STRUCTURE_FIELD_KEY:
+ if isinstance(concept_field_value, str):
+ log.debug(f"Structure for concept '{concept_key}' is a string: {concept_field_value}")
+ concept_table_obj.add("structure", concept_field_value)
+ continue
+ if not isinstance(concept_field_value, Mapping):
+ msg = f"Structure field value is not a mapping: key = {concept_field_key}, value = {concept_field_value}"
+ raise TypeError(msg)
+ log.debug(f"Structure for concept '{concept_key}' is a mapping: {concept_field_value}")
+ structure_value = cast("Mapping[str, Any]", concept_field_value)
+ structure_table_obj = table()
+ for structure_field_key, structure_field_value in structure_value.items():
+ if isinstance(structure_field_value, str):
+ log.debug(f"Structure '{structure_field_key}' is a string: {structure_field_value}")
+ structure_table_obj.add(structure_field_key, structure_field_value)
+ continue
+ if not isinstance(structure_field_value, Mapping):
+ msg = (
+ f"Structure field value is neither a string nor a mapping: "
+ f"key = {structure_field_key}, value = {structure_field_value}"
+ )
+ raise TypeError(msg)
+ log.debug(f"Structure for '{concept_key}' is a mapping: {structure_field_value}")
+ structure_table_obj.add(
+ structure_field_key,
+ cls.convert_dicts_to_inline_tables(
+ value=structure_field_value, field_ordering=cls._plx_config().concepts.structure_field_ordering
+ ),
+ )
+ concept_table_obj.add("structure", structure_table_obj)
+ else:
+ # sub_table = _convert_mapping_to_table(concept_field_value)
+ log.debug(f"{concept_key}/'{concept_field_key}' is inline: {concept_field_value}")
+ concept_table_obj.add(concept_field_key, cls.convert_dicts_to_inline_tables(concept_field_value))
+ table_obj.add(concept_key, concept_table_obj)
+ return table_obj
+
+ @classmethod
+ def dict_to_plx_styled_toml(cls, data: Mapping[str, Any]) -> str:
+ """Top-level keys become tables; second-level mappings become tables; inline tables start at third level."""
+ log.debug("=" * 100)
+ data = remove_none_values_from_dict(data=data)
+ document_root = document()
+ for root_key, root_value in data.items():
+ if not isinstance(root_value, Mapping):
+ log.debug(f"Root root_key is not a mapping: key = {root_key}, value = {root_value}")
+ document_root.add(root_key, cls.convert_dicts_to_inline_tables(root_value))
+ continue
+
+ # It's a mapping, therefore it's a section
+ log.debug(f"Root {root_key} is a section -------------------")
+
+ section_key = SectionKey(root_key)
+ section_value = cast("Mapping[str, Any]", root_value)
+ # Skip empty mappings (empty concept and pipe sections)
+ if not section_value:
+ log.debug(f"Section {section_key} is empty, skipping")
+ continue
+ match section_key:
+ case SectionKey.PIPE:
+ table_obj_for_pipe = cls.make_table_obj_for_pipe(section_value)
+ document_root.add(section_key, table_obj_for_pipe)
+ case SectionKey.CONCEPT:
+ table_obj_for_concept = cls.make_table_obj_for_concept(section_value)
+ document_root.add(section_key, table_obj_for_concept)
+
+ toml_output = tomlkit.dumps(document_root) # pyright: ignore[reportUnknownMemberType]
+ if cls._plx_config().inline_tables.spaces_inside_curly_braces:
+ return cls.add_spaces_to_inline_tables(toml_output)
+ return toml_output
+
+ @classmethod
+ def make_plx_content(cls, blueprint: PipelexBundleBlueprint) -> str:
+ blueprint_dict = blueprint.model_dump(serialize_as_any=True)
+ return cls.dict_to_plx_styled_toml(data=blueprint_dict)
diff --git a/pipelex/libraries/library_config.py b/pipelex/libraries/library_config.py
index bfb5650b4..65e176f9b 100644
--- a/pipelex/libraries/library_config.py
+++ b/pipelex/libraries/library_config.py
@@ -1,3 +1,4 @@
+from pathlib import Path
from typing import ClassVar
from pipelex.tools.config.config_model import ConfigModel
@@ -27,8 +28,8 @@ class LibraryConfig(ConfigModel):
return "tests/test_pipelines"
@property
- def failing_pipelines_file_paths(self) -> list[str]:
- return ["tests/test_pipelines/failing_pipelines.plx"]
+ def failing_pipelines_file_paths(self) -> set[Path]:
+ return {Path("tests/test_pipelines/failing_pipelines.plx")}
def get_templates_paths(self) -> list[str]:
return [str(path) for path in find_files_in_dir(dir_path=self.templates_dir_path, pattern="*.toml", is_recursive=True)]
diff --git a/pipelex/libraries/library_manager.py b/pipelex/libraries/library_manager.py
index 164fb76b5..5bdf0a9fa 100644
--- a/pipelex/libraries/library_manager.py
+++ b/pipelex/libraries/library_manager.py
@@ -21,9 +21,12 @@ from pipelex.core.pipes.pipe_abstract import PipeAbstract
from pipelex.core.pipes.pipe_factory import PipeFactory
from pipelex.core.pipes.pipe_library import PipeLibrary
from pipelex.exceptions import (
+ ConceptDefinitionError,
ConceptLibraryError,
+ DomainDefinitionError,
LibraryError,
- PipelexSetupError,
+ LibraryLoadingError,
+ PipeDefinitionError,
PipeLibraryError,
)
from pipelex.libraries.library_config import LibraryConfig
@@ -140,29 +143,31 @@ class LibraryManager(LibraryManagerAbstract):
return all_plx_paths
- @override
- def load_from_file(self, plx_path: Path) -> None:
- """Load a single file - this method is kept for compatibility."""
- if not PipelexInterpreter.is_pipelex_file(plx_path):
- msg = f"File is not a valid Pipelex PLX file: {plx_path}"
- raise LibraryError(msg)
-
- blueprint = PipelexInterpreter(file_path=plx_path).make_pipelex_bundle_blueprint()
- self.load_from_blueprint(blueprint)
-
@override
def load_from_blueprint(self, blueprint: PipelexBundleBlueprint) -> list[PipeAbstract]:
"""Load a blueprint."""
# Create and load domain
- domain = self._load_domain_from_blueprint(blueprint)
+ try:
+ domain = self._load_domain_from_blueprint(blueprint)
+ except DomainDefinitionError as exc:
+ msg = f"Could not load domain from PLX blueprint at '{blueprint.source}', domain code: '{blueprint.domain}': {exc}"
+ raise LibraryLoadingError(msg) from exc
self.domain_library.add_domain(domain=domain)
# Create and load concepts
- concepts = self._load_concepts_from_blueprint(blueprint)
+ try:
+ concepts = self._load_concepts_from_blueprint(blueprint)
+ except ConceptDefinitionError as exc:
+ msg = f"Could not load concepts from PLX blueprint at '{blueprint.source}', domain code: '{blueprint.domain}': {exc}"
+ raise LibraryLoadingError(msg) from exc
self.concept_library.add_concepts(concepts=concepts)
# Create and load pipes
- pipes = self._load_pipes_from_blueprint(blueprint)
+ try:
+ pipes = self._load_pipes_from_blueprint(blueprint)
+ except PipeDefinitionError as exc:
+ msg = f"Could not load pipes from PLX blueprint at '{blueprint.source}', domain code: '{blueprint.domain}': {exc}"
+ raise LibraryLoadingError(msg) from exc
self.pipe_library.add_pipes(pipes=pipes)
return pipes
@@ -194,19 +199,23 @@ class LibraryManager(LibraryManagerAbstract):
)
def _load_concepts_from_blueprint(self, blueprint: PipelexBundleBlueprint) -> list[Concept]:
- concepts: list[Concept] = []
+ if blueprint.concept is None:
+ return []
- if blueprint.concept is not None:
- for concept_code, concept_blueprint_or_str in blueprint.concept.items():
- concept = ConceptFactory.make_from_blueprint(
- domain=blueprint.domain,
- concept_code=concept_code,
- concept_codes_from_the_same_domain=list(blueprint.concept.keys()),
- blueprint=ConceptBlueprint(definition=concept_blueprint_or_str)
- if isinstance(concept_blueprint_or_str, str)
- else concept_blueprint_or_str,
- )
- concepts.append(concept)
+ concepts: list[Concept] = []
+ for concept_code, concept_blueprint_or_str in blueprint.concept.items():
+ concept_blueprint: ConceptBlueprint
+ if isinstance(concept_blueprint_or_str, str):
+ concept_blueprint = ConceptBlueprint(definition=concept_blueprint_or_str)
+ else:
+ concept_blueprint = concept_blueprint_or_str
+ concept = ConceptFactory.make_from_blueprint(
+ domain=blueprint.domain,
+ concept_code=concept_code,
+ concept_codes_from_the_same_domain=list(blueprint.concept.keys()),
+ blueprint=concept_blueprint,
+ )
+ concepts.append(concept)
return concepts
def _load_pipes_from_blueprint(self, blueprint: PipelexBundleBlueprint) -> list[PipeAbstract]:
@@ -228,57 +237,69 @@ class LibraryManager(LibraryManagerAbstract):
library_dirs: list[Path] | None = None,
library_file_paths: list[Path] | None = None,
) -> None:
- dirs_to_use: list[Path] = self._get_pipeline_library_dirs()
- all_plx_paths: list[Path] = self._get_pipelex_plx_files_from_dirs(dirs_to_use)
-
- # Remove failing pipelines from the list
- failing_pipelines_file_paths = get_config().pipelex.library_config.failing_pipelines_file_paths
- failing_paths_set = {Path(fp) for fp in failing_pipelines_file_paths}
- all_plx_paths = [path for path in all_plx_paths if path not in failing_paths_set]
+ dirs_to_use = library_dirs or self._get_pipeline_library_dirs()
- if library_dirs is not None:
- dirs_to_use = library_dirs
+ valid_plx_paths: list[Path]
+ if library_file_paths:
+ valid_plx_paths = library_file_paths
+ else:
+ all_plx_paths: list[Path] = self._get_pipelex_plx_files_from_dirs(dirs_to_use)
+ # Remove failing pipelines from the list
+ failing_pipelines_file_paths = get_config().pipelex.library_config.failing_pipelines_file_paths
+ valid_plx_paths = [path for path in all_plx_paths if path not in failing_pipelines_file_paths]
# Register classes in the directories
for library_dir in dirs_to_use:
ClassRegistryUtils.register_classes_in_folder(folder_path=str(library_dir))
FuncRegistryUtils.register_funcs_in_folder(folder_path=str(library_dir))
- if library_file_paths is not None:
- all_plx_paths = library_file_paths
-
# Parse all blueprints first
blueprints: list[PipelexBundleBlueprint] = []
- for plx_file_path in all_plx_paths:
+ for plx_file_path in valid_plx_paths:
try:
blueprint = PipelexInterpreter(file_path=plx_file_path).make_pipelex_bundle_blueprint()
except FileNotFoundError as exc:
msg = f"Could not find PLX blueprint at '{plx_file_path}'"
- raise PipelexSetupError(msg) from exc
+ raise LibraryLoadingError(msg) from exc
except ValidationError as exc:
formatted_error_msg = format_pydantic_validation_error(exc)
msg = f"Could not load PLX blueprint from '{plx_file_path}' because of: {formatted_error_msg}"
- raise PipelexSetupError(msg) from exc
+ raise LibraryLoadingError(msg) from exc
+ except PipeDefinitionError as exc:
+ msg = f"Could not load PLX blueprint from '{plx_file_path}': {exc}"
+ raise LibraryLoadingError(msg) from exc
+ blueprint.source = str(plx_file_path)
blueprints.append(blueprint)
# Load all domains first
all_domains: list[Domain] = []
for blueprint in blueprints:
- domain = self._load_domain_from_blueprint(blueprint)
+ try:
+ domain = self._load_domain_from_blueprint(blueprint)
+ except DomainDefinitionError as exc:
+ msg = f"Could not load domain from PLX blueprint at '{blueprint.source}', domain code: '{blueprint.domain}': {exc}"
+ raise LibraryLoadingError(msg) from exc
all_domains.append(domain)
- for domain in all_domains:
- self.domain_library.add_domain(domain=domain)
+ self.domain_library.add_domains(domains=all_domains)
# Load all concepts second
all_concepts: list[Concept] = []
for blueprint in blueprints:
- concepts = self._load_concepts_from_blueprint(blueprint)
+ try:
+ concepts = self._load_concepts_from_blueprint(blueprint)
+ except ConceptDefinitionError as exc:
+ msg = f"Could not load concepts from PLX blueprint at '{blueprint.source}', domain code: '{blueprint.domain}': {exc}"
+ raise LibraryLoadingError(msg) from exc
all_concepts.extend(concepts)
self.concept_library.add_concepts(concepts=all_concepts)
# Load all pipes third
all_pipes: list[PipeAbstract] = []
for blueprint in blueprints:
- pipes = self._load_pipes_from_blueprint(blueprint)
+ try:
+ pipes = self._load_pipes_from_blueprint(blueprint)
+ except PipeDefinitionError as exc:
+ msg = f"Could not load pipes from PLX blueprint at '{blueprint.source}', domain code: '{blueprint.domain}': {exc}"
+ raise LibraryLoadingError(msg) from exc
all_pipes.extend(pipes)
self.pipe_library.add_pipes(pipes=all_pipes)
diff --git a/pipelex/libraries/library_manager_abstract.py b/pipelex/libraries/library_manager_abstract.py
index 3f93d87ed..9c23a9f09 100644
--- a/pipelex/libraries/library_manager_abstract.py
+++ b/pipelex/libraries/library_manager_abstract.py
@@ -26,10 +26,6 @@ class LibraryManagerAbstract(ABC):
def load_libraries(self, library_dirs: list[Path] | None = None, library_file_paths: list[Path] | None = None) -> None:
pass
- @abstractmethod
- def load_from_file(self, plx_path: Path) -> None:
- pass
-
@abstractmethod
def load_from_blueprint(self, blueprint: PipelexBundleBlueprint) -> list[PipeAbstract]:
pass
diff --git a/pipelex/libraries/pipelines/builder/builder.plx b/pipelex/libraries/pipelines/builder/builder.plx
index d610ba2bf..2289a482e 100644
--- a/pipelex/libraries/pipelines/builder/builder.plx
+++ b/pipelex/libraries/pipelines/builder/builder.plx
@@ -102,6 +102,7 @@ parallels = [
{ pipe = "draft_to_conceptspecs_text", result = "concept_spec_drafts_text" },
{ pipe = "draft_to_pipesignatures_text", result = "pipe_signatures_text" },
]
+add_each_output = true
[pipe.draft_to_conceptspecs_text]
type = "PipeLLM"
diff --git a/pipelex/libraries/pipelines/builder/pipe/pipe_parallel_spec.py b/pipelex/libraries/pipelines/builder/pipe/pipe_parallel_spec.py
index 2afdf8b6e..f3e481410 100644
--- a/pipelex/libraries/pipelines/builder/pipe/pipe_parallel_spec.py
+++ b/pipelex/libraries/pipelines/builder/pipe/pipe_parallel_spec.py
@@ -1,12 +1,14 @@
from typing import Literal
-from pydantic import Field, field_validator
+from pydantic import Field, field_validator, model_validator
from typing_extensions import override
+from pipelex.exceptions import PipeDefinitionError
from pipelex.libraries.pipelines.builder.concept.concept_spec import ConceptBlueprint
from pipelex.libraries.pipelines.builder.pipe.pipe_signature import PipeSpec
from pipelex.libraries.pipelines.builder.pipe.sub_pipe_spec import SubPipeSpec
from pipelex.pipe_controllers.parallel.pipe_parallel_blueprint import PipeParallelBlueprint
+from pipelex.types import Self
class PipeParallelSpec(PipeSpec):
@@ -38,7 +40,7 @@ class PipeParallelSpec(PipeSpec):
category: Literal["PipeController"] = "PipeController"
the_pipe_code: str = Field(description="Pipe code. Must be snake_case.")
parallels: list[SubPipeSpec]
- add_each_output: bool = True
+ add_each_output: bool = False
combined_output: str | None = None
@field_validator("combined_output", mode="before")
@@ -48,6 +50,16 @@ class PipeParallelSpec(PipeSpec):
ConceptBlueprint.validate_concept_string_or_code(concept_string_or_code=combined_output)
return combined_output
+ @model_validator(mode="after")
+ def validate_output_options(self) -> Self:
+ if not self.add_each_output and not self.combined_output:
+ msg = (
+ "PipeParallel requires either add_each_output to be True or combined_output to be set, "
+ "or both, otherwise the pipe won't output anything"
+ )
+ raise PipeDefinitionError(msg)
+ return self
+
@override
def to_blueprint(self) -> PipeParallelBlueprint:
base_blueprint = super().to_blueprint()
diff --git a/pipelex/libraries/pipelines/builder/pipe/sub_pipe_spec.py b/pipelex/libraries/pipelines/builder/pipe/sub_pipe_spec.py
index 42423bd3b..ce0cd7ab1 100644
--- a/pipelex/libraries/pipelines/builder/pipe/sub_pipe_spec.py
+++ b/pipelex/libraries/pipelines/builder/pipe/sub_pipe_spec.py
@@ -42,7 +42,7 @@ class SubPipeSpec(StructuredContent):
result: str | None = None
nb_output: int | None = None
multiple_output: bool | None = None
- batch_over: bool | str = False
+ batch_over: str | None = None
batch_as: str | None = None
@model_validator(mode="after")
@@ -54,14 +54,11 @@ class SubPipeSpec(StructuredContent):
@model_validator(mode="after")
def validate_batch_params(self) -> Self:
- batch_over_is_specified = self.batch_over is not False and self.batch_over != ""
- batch_as_is_specified = self.batch_as is not None and self.batch_as != ""
-
- if batch_over_is_specified and not batch_as_is_specified:
+ if self.batch_over and not self.batch_as:
msg = f"In pipe '{self.the_pipe_code}': When 'batch_over' is specified, 'batch_as' must also be provided"
raise PipeDefinitionError(msg)
- if batch_as_is_specified and not batch_over_is_specified:
+ if self.batch_as and not self.batch_over:
msg = f"In pipe '{self.the_pipe_code}': When 'batch_as' is specified, 'batch_over' must also be provided"
raise PipeDefinitionError(msg)
diff --git a/pipelex/pipe_controllers/batch/pipe_batch_factory.py b/pipelex/pipe_controllers/batch/pipe_batch_factory.py
index 3eeb04bb2..469bccaea 100644
--- a/pipelex/pipe_controllers/batch/pipe_batch_factory.py
+++ b/pipelex/pipe_controllers/batch/pipe_batch_factory.py
@@ -41,7 +41,7 @@ class PipeBatchFactory(PipeFactoryProtocol[PipeBatchBlueprint, PipeBatch]):
),
branch_pipe_code=blueprint.branch_pipe_code,
batch_params=BatchParams.make_optional_batch_params(
- input_list_name=blueprint.input_list_name or False,
+ input_list_name=blueprint.input_list_name,
input_item_name=blueprint.input_item_name,
),
)
diff --git a/pipelex/pipe_controllers/parallel/pipe_parallel_blueprint.py b/pipelex/pipe_controllers/parallel/pipe_parallel_blueprint.py
index f9f5599a0..9351f0814 100644
--- a/pipelex/pipe_controllers/parallel/pipe_parallel_blueprint.py
+++ b/pipelex/pipe_controllers/parallel/pipe_parallel_blueprint.py
@@ -1,17 +1,19 @@
from typing import Literal
-from pydantic import field_validator
+from pydantic import field_validator, model_validator
from pipelex.core.concepts.concept_blueprint import ConceptBlueprint
from pipelex.core.pipes.pipe_blueprint import PipeBlueprint
+from pipelex.exceptions import PipeDefinitionError
from pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprint
+from pipelex.types import Self
class PipeParallelBlueprint(PipeBlueprint):
type: Literal["PipeParallel"] = "PipeParallel"
category: Literal["PipeController"] = "PipeController"
parallels: list[SubPipeBlueprint]
- add_each_output: bool = True
+ add_each_output: bool = False
combined_output: str | None = None
@field_validator("combined_output", mode="before")
@@ -20,3 +22,13 @@ class PipeParallelBlueprint(PipeBlueprint):
if combined_output:
ConceptBlueprint.validate_concept_string_or_code(concept_string_or_code=combined_output)
return combined_output
+
+ @model_validator(mode="after")
+ def validate_output_options(self) -> Self:
+ if not self.add_each_output and not self.combined_output:
+ msg = (
+ "PipeParallel requires either add_each_output to be True or combined_output to be set, "
+ "or both, otherwise the pipe won't output anything"
+ )
+ raise PipeDefinitionError(message=msg, description=self.definition, source=self.source)
+ return self
diff --git a/pipelex/pipe_controllers/parallel/pipe_parallel_factory.py b/pipelex/pipe_controllers/parallel/pipe_parallel_factory.py
index af322d650..b43fa0423 100644
--- a/pipelex/pipe_controllers/parallel/pipe_parallel_factory.py
+++ b/pipelex/pipe_controllers/parallel/pipe_parallel_factory.py
@@ -6,7 +6,7 @@ from pipelex.core.concepts.concept_factory import ConceptFactory
from pipelex.core.pipes.pipe_factory import PipeFactoryProtocol
from pipelex.core.pipes.pipe_input_factory import PipeInputSpecFactory
from pipelex.exceptions import PipeDefinitionError
-from pipelex.hub import get_concept_provider
+from pipelex.hub import get_required_concept
from pipelex.pipe_controllers.parallel.pipe_parallel import PipeParallel
from pipelex.pipe_controllers.parallel.pipe_parallel_blueprint import PipeParallelBlueprint
from pipelex.pipe_controllers.sub_pipe_factory import SubPipeFactory
@@ -33,7 +33,10 @@ class PipeParallelFactory(PipeFactoryProtocol[PipeParallelBlueprint, PipeParalle
sub_pipe = SubPipeFactory.make_from_blueprint(sub_pipe_blueprint, concept_codes_from_the_same_domain=concept_codes_from_the_same_domain)
parallel_sub_pipes.append(sub_pipe)
if not blueprint.add_each_output and not blueprint.combined_output:
- msg = "PipeParallel requires either add_each_output or combined_output to be set"
+ msg = (
+ "Unexpected error:PipeParallel requires either add_each_output to be True or combined_output to be set, "
+ "or both, otherwise the pipe won't output anything"
+ )
raise PipeDefinitionError(msg)
if blueprint.combined_output:
@@ -42,7 +45,7 @@ class PipeParallelFactory(PipeFactoryProtocol[PipeParallelBlueprint, PipeParalle
concept_string_or_code=blueprint.output,
concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,
)
- combined_output = get_concept_provider().get_required_concept(
+ combined_output = get_required_concept(
concept_string=ConceptFactory.construct_concept_string_with_domain(
domain=combined_output_domain_and_code.domain,
concept_code=combined_output_domain_and_code.concept_code,
@@ -65,13 +68,13 @@ class PipeParallelFactory(PipeFactoryProtocol[PipeParallelBlueprint, PipeParalle
blueprint=blueprint.inputs or {},
concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,
),
- output=get_concept_provider().get_required_concept(
+ output=get_required_concept(
concept_string=ConceptFactory.construct_concept_string_with_domain(
domain=output_domain_and_code.domain,
concept_code=output_domain_and_code.concept_code,
),
),
parallel_sub_pipes=parallel_sub_pipes,
- add_each_output=blueprint.add_each_output,
+ add_each_output=blueprint.add_each_output or False,
combined_output=combined_output,
)
diff --git a/pipelex/pipe_controllers/sub_pipe_blueprint.py b/pipelex/pipe_controllers/sub_pipe_blueprint.py
index 73ee2d1ac..eb69fbff6 100644
--- a/pipelex/pipe_controllers/sub_pipe_blueprint.py
+++ b/pipelex/pipe_controllers/sub_pipe_blueprint.py
@@ -12,7 +12,7 @@ class SubPipeBlueprint(BaseModel):
result: str | None = None
nb_output: int | None = None
multiple_output: bool | None = None
- batch_over: bool | str = False
+ batch_over: str | None = None
batch_as: str | None = None
@model_validator(mode="after")
@@ -24,14 +24,11 @@ class SubPipeBlueprint(BaseModel):
@model_validator(mode="after")
def validate_batch_params(self) -> Self:
- batch_over_is_specified = self.batch_over is not False and self.batch_over != ""
- batch_as_is_specified = self.batch_as is not None and self.batch_as != ""
-
- if batch_over_is_specified and not batch_as_is_specified:
+ if self.batch_over and not self.batch_as:
msg = f"In pipe '{self.pipe}': When 'batch_over' is specified, 'batch_as' must also be provided"
raise PipeDefinitionError(msg)
- if batch_as_is_specified and not batch_over_is_specified:
+ if self.batch_as and not self.batch_over:
msg = f"In pipe '{self.pipe}': When 'batch_as' is specified, 'batch_over' must also be provided"
raise PipeDefinitionError(msg)
diff --git a/pipelex/pipelex.toml b/pipelex/pipelex.toml
index 3f8a28669..352b8b794 100644
--- a/pipelex/pipelex.toml
+++ b/pipelex/pipelex.toml
@@ -204,14 +204,34 @@ text_gen_truncate_length = 256
nb_list_items = 3
nb_ocr_pages = 4
allowed_to_fail_pipes = [
- "infinite_loop_1", # Loop but only for testing purposes
- "fix_failing_pipes_once", # Loop but its normal
+ "infinite_loop_1", # Loop but only for testing purposes
+ "fix_failing_pipes_once", # Loop but its normal
"validate_pipelex_bundle_spec", # Loop but its normal
- "handle_validation_result", # Complex case of a sub pipe needed a ListContent because its batching on, but the dry_run cannot know.
- "pipe_builder", # because of the 'fix_failing_pipes_once', 'validate_pipelex_bundle_spec', 'handle_validation_result'
+ "handle_validation_result", # Complex case of a sub pipe needed a ListContent because its batching on, but the dry_run cannot know.
+ "pipe_builder", # because of the 'fix_failing_pipes_once', 'validate_pipelex_bundle_spec', 'handle_validation_result'
]
image_urls = [
"https://storage.googleapis.com/public_test_files_7fa6_4277_9ab/fashion/fashion_photo_1.jpg",
"https://storage.googleapis.com/public_test_files_7fa6_4277_9ab/fashion/fashion_photo_2.png",
]
+####################################################################################################
+# PLX config
+####################################################################################################
+
+[pipelex.plx_config.inline_tables]
+spaces_inside_curly_braces = true
+
+[pipelex.plx_config.strings]
+prefer_literal = false
+force_multiline = false
+length_limit_to_multiline = 100
+ensure_trailing_newline = true
+ensure_leading_blank_line = true
+
+[pipelex.plx_config.concepts]
+structure_field_ordering = ["type", "definition", "choices", "required"]
+
+[pipelex.plx_config.pipes]
+field_ordering = ["type", "definition", "inputs", "output"]
+
diff --git a/pipelex/tools/config/manager.py b/pipelex/tools/config/manager.py
index 15376ec22..97aefde57 100644
--- a/pipelex/tools/config/manager.py
+++ b/pipelex/tools/config/manager.py
@@ -10,7 +10,7 @@ from pipelex.tools.config.config_root import (
CONFIG_BASE_OVERRIDES_BEFORE_ENV,
)
from pipelex.tools.misc.json_utils import deep_update
-from pipelex.tools.misc.toml_utils import failable_load_toml_from_path, load_toml_from_path
+from pipelex.tools.misc.toml_utils import load_toml_from_path, load_toml_from_path_if_exists
from pipelex.tools.runtime_manager import runtime_manager
CONFIG_DIR_NAME = ".pipelex"
@@ -74,8 +74,9 @@ class ConfigManager:
Dict[str, Any]: The configuration dictionary from the local pipelex.toml
"""
- config_path = os.path.join(self.local_root_dir, CONFIG_NAME)
- return failable_load_toml_from_path(config_path) or {}
+ config_path = os.path.join(self.pipelex_config_dir, CONFIG_NAME)
+ print(f"Getting local config from {config_path}")
+ return load_toml_from_path_if_exists(config_path) or {}
def load_inheritance_config(self, the_pipelex_config: dict[str, Any]):
"""Load the config by inheritance in a pyproject.toml file.
@@ -105,7 +106,7 @@ class ConfigManager:
if package_path:
config_path = os.path.join(package_path, "pipelex.toml")
if os.path.exists(config_path):
- config = failable_load_toml_from_path(config_path)
+ config = load_toml_from_path_if_exists(config_path)
if config:
deep_update(the_pipelex_config, config)
@@ -150,12 +151,12 @@ class ConfigManager:
override_path = os.path.join(self.local_root_dir, "tests", f"pipelex_{override}.toml")
else:
override_path = os.path.join(self.local_root_dir, "pipelex" if self.is_in_pipelex_config else "", f"pipelex_{override}.toml")
- if override_dict := failable_load_toml_from_path(override_path):
+ if override_dict := load_toml_from_path_if_exists(override_path):
deep_update(pipelex_config, override_dict)
#################### 5. Load specific config ####################
if specific_config_path:
- config = failable_load_toml_from_path(specific_config_path)
+ config = load_toml_from_path_if_exists(specific_config_path)
if config:
deep_update(pipelex_config, config)
else:
diff --git a/pipelex/tools/misc/file_utils.py b/pipelex/tools/misc/file_utils.py
index 2aa7e5a09..d5fe8e29a 100644
--- a/pipelex/tools/misc/file_utils.py
+++ b/pipelex/tools/misc/file_utils.py
@@ -251,6 +251,15 @@ def ensure_path(path: str) -> bool:
return True
+def ensure_directory_for_file_path(file_path: str) -> None:
+ """Ensures a directory exists for the specified file path.
+
+ Args:
+ file_path (str): The path to the file.
+ """
+ ensure_directory_exists(os.path.dirname(file_path))
+
+
def path_exists(path_str: str) -> bool:
"""Checks if a file or directory exists at the specified path.
diff --git a/pipelex/tools/misc/toml_utils.py b/pipelex/tools/misc/toml_utils.py
index d20b5bec8..0fc407277 100644
--- a/pipelex/tools/misc/toml_utils.py
+++ b/pipelex/tools/misc/toml_utils.py
@@ -1,17 +1,10 @@
from __future__ import annotations
-import re
-from collections.abc import Mapping
-from typing import Any, cast
+from typing import Any
import toml
-import tomlkit
-from pydantic import BaseModel
-from tomlkit import array, document, inline_table, table
-from tomlkit import string as tk_string
from pipelex.tools.misc.file_utils import path_exists
-from pipelex.tools.misc.json_utils import remove_none_values_from_dict
class TOMLValidationError(Exception):
@@ -122,267 +115,8 @@ def load_toml_from_path(path: str) -> dict[str, Any]:
raise toml.TomlDecodeError(msg, exc.doc, exc.pos) from exc
-def failable_load_toml_from_path(path: str) -> dict[str, Any] | None:
- """Load TOML from path with failure handling."""
+def load_toml_from_path_if_exists(path: str) -> dict[str, Any] | None:
+ """Load TOML from path if it exists."""
if not path_exists(path):
return None
- try:
- return load_toml_from_path(path)
- except (toml.TomlDecodeError, TOMLValidationError) as exc:
- print(f"Failed to parse TOML file '{path}': {exc}")
- return None
-
-
-def make_toml_string(
- text: str,
- prefer_literal: bool = False,
- force_multiline: bool = False,
- ensure_trailing_newline: bool = True,
- ensure_leading_blank_line_in_value: bool = False,
-):
- r"""Build a tomlkit string node.
- - If `force_multiline` or the text contains '\\n', we emit a triple-quoted multiline string.
- - When multiline, `ensure_trailing_newline` puts the closing quotes on their own line.
- - When multiline, `ensure_leading_blank_line_in_value` inserts a real blank line at the start of the value.
- """
- needs_multiline = force_multiline or ("\n" in text)
- normalized = text
-
- if needs_multiline:
- if ensure_leading_blank_line_in_value and not normalized.startswith("\n"):
- normalized = "\n" + normalized
- if ensure_trailing_newline and not normalized.endswith("\n"):
- normalized = normalized + "\n"
-
- use_literal = prefer_literal and ("'''" not in normalized)
- return tk_string(normalized, multiline=needs_multiline, literal=use_literal)
-
-
-def _convert_to_inline(value: Any) -> Any:
- """Recursively convert Python values; dicts -> inline tables; lists kept as arrays."""
- # Handle Pydantic models by converting them to dict first
- if isinstance(value, BaseModel):
- # For RootModel, use the root attribute; for regular models, use model_dump()
- value = value.root if hasattr(value, "root") else value.model_dump() # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownVariableType]
-
- if isinstance(value, Mapping):
- value = cast("Mapping[str, Any]", value)
- inline_table_obj = inline_table()
- for key, value_item in value.items():
- inline_table_obj[key] = _convert_to_inline(value_item)
-
- return inline_table_obj
-
- if isinstance(value, list):
- value = cast("list[Any]", value)
- array_obj = array()
- array_obj.multiline(True) # set to False for single-line arrays
- for element in value:
- if isinstance(element, Mapping):
- element = cast("Mapping[str, Any]", element)
- inline_element = inline_table()
- for inner_key, inner_value in element.items():
- inline_element[inner_key] = _convert_to_inline(inner_value)
- array_obj.append(inline_element) # pyright: ignore[reportUnknownMemberType]
- else:
- array_obj.append(_convert_to_inline(element)) # pyright: ignore[reportUnknownMemberType]
- return array_obj
-
- if isinstance(value, str):
- # For prompt templates and similar fields, escape newlines instead of using multiline format
- if any(keyword in str(value) for keyword in ["@", "$"]) and "\n" in value:
- # This looks like a prompt template - escape newlines for single-line format
- # Return the string directly and let tomlkit handle the escaping properly
- return value
- # Triple quotes if needed (or forced); closing quotes on their own line.
- return make_toml_string(
- value,
- prefer_literal=False, # flip to True for '''...'''
- force_multiline=False, # flip to True to force """...""" even without \n
- ensure_trailing_newline=True, # keep closing """ on its own line
- ensure_leading_blank_line_in_value=False, # flip to True to keep a blank first line
- )
- return value # pyright: ignore[reportUnknownVariableType]
-
-
-def _filter_empty_values(value: Any) -> Any:
- """Filter out empty lists and None values from data structures."""
- if isinstance(value, dict):
- filtered: dict[str, Any] = {}
- for k, v in cast("dict[str, Any]", value).items():
- filtered_v = _filter_empty_values(v)
- # Keep empty dicts but skip empty lists and None values
- if filtered_v is not None and (not isinstance(filtered_v, list) or filtered_v):
- filtered[k] = filtered_v
- return filtered
- if isinstance(value, list):
- return [_filter_empty_values(item) for item in cast("list[Any]", value) if _filter_empty_values(item) is not None]
- return value
-
-
-def _create_ordered_inline_table(data: Mapping[str, Any]) -> Any:
- """Create an inline table with fields in the expected order."""
- inline_table_obj = inline_table()
-
- # Define the preferred order for concept structure fields
- field_order = ["type", "definition", "required"]
-
- # Add fields in preferred order first
- for field in field_order:
- if field in data:
- value = data[field]
- # Skip empty lists
- if isinstance(value, list) and not value:
- continue
- inline_table_obj[field] = _convert_to_inline(value)
-
- # Add any remaining fields not in the preferred order
- for key, value in data.items():
- if key not in field_order:
- # Skip empty lists
- if isinstance(value, list) and not value:
- continue
- inline_table_obj[key] = _convert_to_inline(value)
-
- return inline_table_obj
-
-
-def dict_to_toml(data: Mapping[str, Any]) -> str:
- """Convert dictionary to TOML format matching Pipelex expectations."""
- data = remove_none_values_from_dict(data=data)
- data = _filter_empty_values(data)
- document_root = document()
-
- # Handle top-level fields first (domain, definition, system_prompt, etc.)
- for key, value in data.items():
- if not isinstance(value, Mapping):
- document_root.add(key, _convert_to_inline(value))
-
- # Handle sections (concepts, pipes)
- for section_key, section_value in data.items():
- if isinstance(section_value, Mapping):
- section_value = cast("Mapping[str, Any]", section_value)
-
- # Skip empty sections
- if not section_value:
- continue
-
- # Create the section table
- section_table = table()
-
- # Process each item in the section
- for item_key, item_value in section_value.items():
- if isinstance(item_value, str):
- # Simple string value (e.g., "SimpleData = 'Simple data concept'")
- section_table.add(item_key, _convert_to_inline(item_value))
- elif isinstance(item_value, Mapping):
- # Complex object that needs its own table
- item_value = cast("Mapping[str, Any]", item_value)
- item_table = table()
-
- # Handle the structure field specially
- for field_key, field_value in item_value.items():
- # THIS IS CUSTOM CODE FOR THE CONCEPT STRUCUTRE FIELD. It should be generic
- if field_key == "structure" and isinstance(field_value, Mapping):
- # Structure should be its own table [section.item.structure]
- structure_table = table()
- for struct_key, struct_value in cast("Mapping[str, Any]", field_value).items():
- if isinstance(struct_value, Mapping):
- structure_table.add(struct_key, _create_ordered_inline_table(cast("Mapping[str, Any]", struct_value)))
- else:
- structure_table.add(struct_key, _convert_to_inline(struct_value))
- item_table.add(field_key, structure_table)
- else:
- # Skip empty lists like refines = []
- if isinstance(field_value, list) and not field_value:
- continue
- item_table.add(field_key, _convert_to_inline(field_value))
-
- section_table.add(item_key, item_table)
-
- document_root.add(section_key, section_table)
-
- dumped_content = tomlkit.dumps(document_root) # pyright: ignore[reportUnknownMemberType]
-
- # Post-process to fix inline table spacing: {key = "value"} -> { key = "value" }
-
- # Fix spacing around inline table contents - ensure single space after { and before }
- # This regex matches TOML inline tables (containing = signs) but not Jinja2 templates
- # We need to handle nested braces properly by working from innermost to outermost
-
- def fix_inline_table_spacing(content: str) -> str:
- """Fix spacing in inline tables, handling nested structures, but avoid modifying braces inside strings."""
- # Split content into lines and process each line
- lines = content.split("\n")
- processed_lines: list[str] = []
-
- for line in lines:
- # Only process lines that contain TOML inline tables (have '=' and braces outside quotes)
- # Skip lines that are just string values with braces (like Jinja2 templates)
- if "=" in line and "{" in line and "}" in line:
- # Check if this line has a TOML key-value assignment with inline table
- # Pattern: key = { ... } or key = { nested = { ... } }
- if re.match(r"^\s*\w+\s*=\s*\{", line):
- # This is a TOML inline table assignment, apply spacing fixes
- processed_line = line
-
- # Apply multiple passes to handle nested structures properly
- for _ in range(10): # Prevent infinite loops
- old_line = processed_line
-
- # First: fix simple inline tables (no nested braces)
- processed_line = re.sub(r"\{\s*([^{}]*=[^{}]*?)\s*\}", r"{ \1 }", processed_line)
-
- # Second: fix nested inline tables - outer tables containing inner ones
- # Match pattern like { key = { inner = "value" } }
- processed_line = re.sub(r"\{\s*([^={}]*=\s*\{[^}]*\}[^{}]*?)\s*\}", r"{ \1 }", processed_line)
-
- # Third: fix complex nested cases with multiple inner tables
- # Handle cases like {key1 = {inner = "value"}, key2 = {inner2 = "value2"}}
- processed_line = re.sub(r"\{\s*([^{}]*=\s*\{[^}]*\}.*?)\s*\}", r"{ \1 }", processed_line)
-
- # Fourth: clean up any excessive whitespace that might remain
- # Remove multiple consecutive spaces inside tables
- processed_line = re.sub(r"\{\s*([^{}]*?)\s{2,}\}", r"{ \1 }", processed_line)
-
- # Fifth: ensure consistent single space formatting
- processed_line = re.sub(r"\{\s{2,}([^{}]*?)\s*\}", r"{ \1 }", processed_line)
-
- if old_line == processed_line:
- break
-
- # Final cleanup: fix any remaining spacing issues in inline tables
- # First: ensure there's a space between adjacent closing braces "}}"
- processed_line = re.sub(r"\}\}", "} }", processed_line)
-
- # Then: remove extra spaces (2 or more) before closing braces, but keep single spaces
- # This pattern matches cases like "} }" (2+ spaces) and replaces with "} }"
- processed_line = re.sub(r"\}\s{2,}\}", "} }", processed_line)
-
- processed_lines.append(processed_line)
- else:
- # This line has braces but is not a TOML inline table (e.g., string with Jinja2)
- # Leave it unchanged
- processed_lines.append(line)
- else:
- # No braces or no assignment, leave unchanged
- processed_lines.append(line)
-
- return "\n".join(processed_lines)
-
- return fix_inline_table_spacing(dumped_content)
-
-
-def save_toml_to_path(data: dict[str, Any], path: str) -> None:
- """Save dictionary as TOML to file path.
-
- Args:
- data: Dictionary to save as TOML
- path: Path where to save the TOML file
-
- """
- data_cleaned = data
- with open(path, "w", encoding="utf-8") as file:
- toml_content: str = dict_to_toml(data=data_cleaned)
- cleaned_content = clean_trailing_whitespace(toml_content)
- file.write(cleaned_content)
+ return load_toml_from_path(path)
diff --git a/pipelex/types.py b/pipelex/types.py
index 1f1c2116c..d1127172d 100644
--- a/pipelex/types.py
+++ b/pipelex/types.py
@@ -1,5 +1,3 @@
-# ruff: noqa: UP
-
try:
from typing import Self # Python 3.11+
except ImportError: # Python 3.10
diff --git a/pyproject.toml b/pyproject.toml
index 0cad96831..f4d38d4c5 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -115,7 +115,7 @@ module = ["filetype", "json2html", "pypdfium2", "pypdfium2.raw"]
[tool.pyright]
pythonVersion = "3.11"
include = ["pipelex", "tests"]
-exclude = ["**/__pycache__", ".venv", ".git", "build", "dist", "tests"]
+exclude = ["**/__pycache__", ".venv", ".git", "build", "dist"]
analyzeUnannotatedFunctions = true
deprecateTypingAliases = false
disableBytesTypePromotions = true
diff --git a/tests/integration/pipelex/language/test_plx_factory.py b/tests/integration/pipelex/language/test_plx_factory.py
new file mode 100644
index 000000000..a7cbbed86
--- /dev/null
+++ b/tests/integration/pipelex/language/test_plx_factory.py
@@ -0,0 +1,22 @@
+from pathlib import Path
+from typing import Type
+
+import pytest
+
+from pipelex import log, pretty_print
+from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint
+from pipelex.language.plx_factory import PlxFactory
+
+from tests.unit.pipelex.core.test_data import InterpreterTestCases
+
+
+class TestPlxFactoryIntegration:
+
+ @pytest.mark.parametrize("test_name,expected_plx_content,blueprint", InterpreterTestCases.VALID_TEST_CASES)
+ def test_make_plx_content(self, test_name: str, expected_plx_content: str, blueprint: PipelexBundleBlueprint):
+ """Test making blueprint from various valid PLX content."""
+ plx_content = PlxFactory.make_plx_content(blueprint=blueprint)
+ pretty_print(plx_content, title=f"Plx content {test_name}")
+ pretty_print(expected_plx_content, title=f"Expected PLX content {test_name}")
+ assert plx_content == expected_plx_content
+
diff --git a/tests/test_pipelines/discord_newsletter.py b/tests/test_pipelines/discord_newsletter.py
index 6de3cb373..6bd5799a7 100644
--- a/tests/test_pipelines/discord_newsletter.py
+++ b/tests/test_pipelines/discord_newsletter.py
@@ -2,6 +2,7 @@
from pydantic import Field
from pipelex.core.stuffs.stuff_content import StructuredContent
+from pipelex.tools.typing.pydantic_utils import CustomBaseModel, empty_list_factory_of
class Attachment(StructuredContent):
@@ -24,8 +25,8 @@ class DiscordMessage(StructuredContent):
author: str = Field(..., description="Author of the message")
content: str = Field(..., description="Content of the message")
- attachments: list[Attachment] = Field(default_factory=list, description="List of message attachments")
- embeds: list[Embed] = Field(default_factory=list, description="List of message embeds")
+ attachments: list[Attachment] = Field(default_factory=empty_list_factory_of(Attachment), description="List of message attachments")
+ embeds: list[Embed] = Field(default_factory=empty_list_factory_of(Embed), description="List of message embeds")
link: str = Field(..., description="Link to the message")
@@ -34,7 +35,7 @@ class DiscordChannelUpdate(StructuredContent):
name: str = Field(..., description="Name of the Discord channel")
position: int = Field(..., description="Position of the channel")
- messages: list[DiscordMessage] = Field(default_factory=list, description="List of messages in the channel")
+ messages: list[DiscordMessage] = Field(default_factory=empty_list_factory_of(DiscordMessage), description="List of messages in the channel")
class ChannelSummary(StructuredContent):
@@ -48,6 +49,6 @@ class Newsletter(StructuredContent):
"""Represents the final newsletter content"""
weekly_summary: str = Field(..., description="200 character summary of weekly Share channel content")
- new_members: list[str] = Field(default_factory=list, description="New member introductions in bullet points")
- channel_sections: list[ChannelSummary] = Field(default_factory=list, description="Ordered channel summaries")
- geographic_hubs: list[ChannelSummary] = Field(default_factory=list, description="Geographic hub channels grouped at end")
+ new_members: list[str] = Field(default_factory=empty_list_factory_of(str), description="New member introductions in bullet points")
+ channel_sections: list[ChannelSummary] = Field(default_factory=empty_list_factory_of(ChannelSummary), description="Ordered channel summaries")
+ geographic_hubs: list[ChannelSummary] = Field(default_factory=empty_list_factory_of(ChannelSummary), description="Geographic hub channels grouped at end")
diff --git a/tests/test_pipelines/test_structures_basic.py b/tests/test_pipelines/test_structures_basic.py
index cfc6fddfe..b7228f507 100644
--- a/tests/test_pipelines/test_structures_basic.py
+++ b/tests/test_pipelines/test_structures_basic.py
@@ -5,6 +5,7 @@ from datetime import datetime
from pydantic import Field
from pipelex.core.stuffs.stuff_content import StructuredContent
+from pipelex.tools.typing.pydantic_utils import empty_list_factory_of
class ConceptWithSimpleStructure(StructuredContent):
@@ -28,7 +29,7 @@ class ConceptWithLists(StructuredContent):
"""A structure with list fields."""
string_list: list[str] = Field(default_factory=list, description="A list of strings")
- number_list: list[int] = Field(default_factory=list, description="A list of numbers")
+ number_list: list[int] = Field(default_factory=empty_list_factory_of(int), description="A list of numbers")
optional_list: list[str] | None = Field(None, description="An optional list")
@@ -37,4 +38,4 @@ class ConceptWithNestedStructures(StructuredContent):
simple_nested: ConceptWithSimpleStructure = Field(..., description="A nested simple structure")
optional_nested: ConceptWithOptionals | None = Field(None, description="An optional nested structure")
- list_of_nested: list[ConceptWithSimpleStructure] = Field(default_factory=list, description="A list of nested structures")
+ list_of_nested: list[ConceptWithSimpleStructure] = Field(default_factory=empty_list_factory_of(ConceptWithSimpleStructure), description="A list of nested structures")
diff --git a/tests/test_pipelines/test_structures_complex.py b/tests/test_pipelines/test_structures_complex.py
index 7ba682a6d..0ecc88e1d 100644
--- a/tests/test_pipelines/test_structures_complex.py
+++ b/tests/test_pipelines/test_structures_complex.py
@@ -5,6 +5,7 @@ from pydantic import Field
from pipelex.core.stuffs.stuff_content import StructuredContent
from tests.test_pipelines.test_structures_basic import ConceptWithOptionals, ConceptWithSimpleStructure
+from pipelex.tools.typing.pydantic_utils import empty_list_factory_of
class ConceptWithDicts(StructuredContent):
@@ -20,7 +21,7 @@ class ConceptWithUnions(StructuredContent):
string_or_int: str | int = Field(..., description="A field that can be string or int")
optional_union: str | bool | None = Field(None, description="An optional union field")
- list_of_unions: list[str | int] = Field(default_factory=list, description="A list of union types")
+ list_of_unions: list[str | int] = Field(default_factory=lambda: [], description="A list of union types")
class ConceptWithComplexUnions(StructuredContent):
@@ -37,7 +38,7 @@ class ConceptWithNestedUnions(StructuredContent):
simple_or_complex: ConceptWithSimpleStructure | ConceptWithOptionals = Field(..., description="Simple or complex structure")
list_of_union_structures: list[ConceptWithSimpleStructure | ConceptWithUnions] = Field(
- default_factory=list, description="List of different structure types",
+ default_factory=lambda: [], description="List of different structure types",
)
optional_nested_union: ConceptWithUnions | ConceptWithComplexUnions | None = Field(
None, description="Optional union of union structures",
diff --git a/tests/unit/pipelex/core/concepts/concept_factory.py/data.py b/tests/unit/pipelex/core/concepts/concept_factory.py/data.py
deleted file mode 100644
index 5f1c22cc8..000000000
--- a/tests/unit/pipelex/core/concepts/concept_factory.py/data.py
+++ /dev/null
@@ -1,260 +0,0 @@
-from typing import ClassVar, Optional, List, Tuple
-
-from pipelex.core.concepts.concept import Concept
-from pipelex.core.concepts.concept_blueprint import (
- ConceptBlueprint,
- ConceptStructureBlueprint,
- ConceptStructureBlueprintFieldType,
-)
-from pipelex.core.concepts.concept_factory import DomainAndConceptCode
-from pipelex.core.concepts.concept_native import NATIVE_CONCEPTS_DATA, NativeConceptEnum
-from pipelex.core.domains.domain import SpecialDomain
-
-
-class TestCases:
- # Test cases for make_refines method - only native concepts can be refined
- MAKE_REFINES_TEST_CASES: ClassVar[list[tuple[str, ConceptBlueprint, str]]] = [
- (
- "native_concept_string",
- ConceptBlueprint(definition="A concept that refines a native text concept", refines=NativeConceptEnum.TEXT),
- f"{SpecialDomain.NATIVE}.{NativeConceptEnum.TEXT}",
- ),
- (
- "fully_qualified_native_string",
- ConceptBlueprint(
- definition="A concept that refines a fully qualified native concept",
- refines=f"{SpecialDomain.NATIVE}.{NativeConceptEnum.TEXT}",
- ),
- f"{SpecialDomain.NATIVE}.{NativeConceptEnum.TEXT}",
- ),
- ]
-
- # Test cases for make_domain_and_concept_code_from_concept_string_or_code method
- MAKE_DOMAIN_AND_CONCEPT_CODE_TEST_CASES: ClassVar[List[Tuple[str, str, List[str] | None, DomainAndConceptCode]]] = [
- # Test case 1: Concept string with dot notation
- ("my_domain", "other_domain.ConceptName", None, DomainAndConceptCode(domain="other_domain", concept_code="ConceptName")),
- # Test case 2: Concept string with dot notation (ignores same domain codes)
- ("my_domain", "other_domain.ConceptName", ["ConceptName"], DomainAndConceptCode(domain="other_domain", concept_code="ConceptName")),
- # Test case 3: Native concept code (Text)
- ("my_domain", "Text", None, DomainAndConceptCode(domain=SpecialDomain.NATIVE, concept_code="Text")),
- # Test case 4: Native concept code (Image)
- ("my_domain", "Image", None, DomainAndConceptCode(domain=SpecialDomain.NATIVE, concept_code="Image")),
- # Test case 5: Native concept code (PDF)
- ("my_domain", "PDF", None, DomainAndConceptCode(domain=SpecialDomain.NATIVE, concept_code="PDF")),
- # Test case 6: Native concept code with same domain codes provided (native takes precedence)
- ("my_domain", "Text", ["Text", "OtherConcept"], DomainAndConceptCode(domain=SpecialDomain.NATIVE, concept_code="Text")),
- # Test case 7: Concept code from same domain
- ("my_domain", "MyConcept", ["MyConcept", "OtherConcept"], DomainAndConceptCode(domain="my_domain", concept_code="MyConcept")),
- # Test case 8: Concept code from same domain (case sensitive)
- ("my_domain", "MyConcept", ["MyCon", "OtherConcept"], DomainAndConceptCode(domain=SpecialDomain.IMPLICIT, concept_code="MyConcept")),
- # Test case 9: Unknown concept code (no same domain codes)
- ("my_domain", "UnknownConcept", None, DomainAndConceptCode(domain=SpecialDomain.IMPLICIT, concept_code="UnknownConcept")),
- # Test case 10: Unknown concept code (not in same domain codes)
- (
- "my_domain",
- "UnknownConcept",
- ["KnownConcept", "OtherConcept"],
- DomainAndConceptCode(domain=SpecialDomain.IMPLICIT, concept_code="UnknownConcept"),
- ),
- # Test case 11: Empty same domain codes list
- ("my_domain", "SomeConcept", [], DomainAndConceptCode(domain=SpecialDomain.IMPLICIT, concept_code="SomeConcept")),
- # Test case 12: Different domain in concept string
- ("my_domain", "another_domain.SomeConcept", ["SomeConcept"], DomainAndConceptCode(domain="another_domain", concept_code="SomeConcept")),
- # Test case 13: All native concept codes
- ("my_domain", "Dynamic", None, DomainAndConceptCode(domain=SpecialDomain.NATIVE, concept_code="Dynamic")),
- ("my_domain", "TextAndImages", None, DomainAndConceptCode(domain=SpecialDomain.NATIVE, concept_code="TextAndImages")),
- ("my_domain", "Number", None, DomainAndConceptCode(domain=SpecialDomain.NATIVE, concept_code="Number")),
- ("my_domain", "LlmPrompt", None, DomainAndConceptCode(domain=SpecialDomain.NATIVE, concept_code="LlmPrompt")),
- ("my_domain", "Page", None, DomainAndConceptCode(domain=SpecialDomain.NATIVE, concept_code="Page")),
- ("my_domain", "Anything", None, DomainAndConceptCode(domain=SpecialDomain.NATIVE, concept_code="Anything")),
- ]
-
- # Test cases for make_from_blueprint method
- MAKE_FROM_BLUEPRINT_TEST_CASES: ClassVar[list[tuple[str, str, str, ConceptBlueprint, list[str] | None, Concept]]] = [
- # Test case 1: Simple blueprint with no structure or refines (goes to implicit)
- (
- "simple_blueprint",
- "my_domain",
- "SimpleConcept",
- ConceptBlueprint(definition="A simple concept"),
- None,
- Concept(
- domain=SpecialDomain.IMPLICIT,
- code="SimpleConcept",
- definition="A simple concept",
- structure_class_name=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT].content_class_name,
- refines=None,
- ),
- ),
- # Test case 2: Blueprint with string structure (existing class, goes to implicit)
- (
- "string_structure_existing_class",
- "my_domain",
- "ConceptWithStructure",
- ConceptBlueprint(definition="A concept with string structure", structure="TextContent"),
- None,
- Concept(
- domain=SpecialDomain.IMPLICIT,
- code="ConceptWithStructure",
- definition="A concept with string structure",
- structure_class_name=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT].content_class_name,
- refines=None,
- ),
- ),
- # Test case 3: Blueprint with refines (native concept, goes to implicit)
- (
- "blueprint_with_refines_text",
- "my_domain",
- "MyTextConcept",
- ConceptBlueprint(definition="A concept that refines Text", refines="Text"),
- None,
- Concept(
- domain=SpecialDomain.IMPLICIT,
- code="MyTextConcept",
- definition="A concept that refines Text",
- structure_class_name=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT].content_class_name,
- refines="native.Text",
- ),
- ),
- # Test case 4: Blueprint with refines (fully qualified native concept, goes to implicit)
- (
- "blueprint_with_refines_native_image",
- "my_domain",
- "MyImageConcept",
- ConceptBlueprint(definition="A concept that refines native.Image", refines="native.Image"),
- None,
- Concept(
- domain=SpecialDomain.IMPLICIT,
- code="MyImageConcept",
- definition="A concept that refines native.Image",
- structure_class_name=NATIVE_CONCEPTS_DATA[NativeConceptEnum.IMAGE].content_class_name,
- refines="native.Image",
- ),
- ),
- # Test case 5: Native concept code (should go to native domain)
- (
- "native_concept_code",
- "my_domain",
- "Text",
- ConceptBlueprint(definition="Native text concept"),
- None,
- Concept(
- domain=SpecialDomain.NATIVE,
- code="Text",
- definition="Native text concept",
- structure_class_name=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT].content_class_name,
- refines=None,
- ),
- ),
- # Test case 6: Concept code from same domain
- (
- "same_domain_concept",
- "my_domain",
- "DomainConcept",
- ConceptBlueprint(definition="A concept from same domain"),
- ["DomainConcept", "OtherConcept"],
- Concept(
- domain="my_domain",
- code="DomainConcept",
- definition="A concept from same domain",
- structure_class_name=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT].content_class_name,
- refines=None,
- ),
- ),
- # Test case 7: Unknown concept code (should go to implicit domain)
- (
- "implicit_concept",
- "my_domain",
- "UnknownConcept",
- ConceptBlueprint(definition="An unknown concept"),
- ["KnownConcept"],
- Concept(
- domain=SpecialDomain.IMPLICIT,
- code="UnknownConcept",
- definition="An unknown concept",
- structure_class_name=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT].content_class_name,
- refines=None,
- ),
- ),
- # Test case 8: Blueprint with dict structure (implicit domain)
- (
- "dict_structure_implicit",
- SpecialDomain.IMPLICIT,
- "PersonConcept",
- ConceptBlueprint(
- definition="A person with structured data",
- structure={
- "name": "The person's name",
- "age": ConceptStructureBlueprint(definition="The person's age", type=ConceptStructureBlueprintFieldType.NUMBER, required=True),
- "active": ConceptStructureBlueprint(
- definition="Whether the person is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True,
- ),
- },
- ),
- None,
- Concept(
- domain=SpecialDomain.IMPLICIT,
- code="PersonConcept",
- definition="A person with structured data",
- structure_class_name="PersonConcept",
- refines=None,
- ),
- ),
- # Test case 9: Blueprint with dict structure (same domain)
- (
- "dict_structure_same_domain",
- "my_domain",
- "PersonConcept",
- ConceptBlueprint(
- definition="A person with structured data",
- structure={
- "name": ConceptStructureBlueprint(definition="The person's name", type=ConceptStructureBlueprintFieldType.TEXT, required=True),
- "age": ConceptStructureBlueprint(definition="The person's age", type=ConceptStructureBlueprintFieldType.NUMBER, required=True),
- "active": ConceptStructureBlueprint(
- definition="Whether the person is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True,
- ),
- },
- ),
- ["PersonConcept", "OtherConcept"],
- Concept(
- domain="my_domain",
- code="PersonConcept",
- definition="A person with structured data",
- structure_class_name="PersonConcept",
- refines=None,
- ),
- ),
- # Test case 10: Blueprint with refines (implicit domain)
- (
- "refines_implicit_domain",
- "my_domain",
- "ConceptWithRefines",
- ConceptBlueprint(definition="A concept with refines", refines="Text"),
- None,
- Concept(
- domain=SpecialDomain.IMPLICIT,
- code="ConceptWithRefines",
- definition="A concept with refines",
- structure_class_name=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT].content_class_name,
- refines="native.Text",
- ),
- ),
- # Test case 11: Blueprint with refines (same domain)
- (
- "refines_same_domain",
- "my_domain",
- "ConceptWithRefines",
- ConceptBlueprint(definition="A concept with refines", refines="native.Image"),
- ["ConceptWithRefines", "OtherConcept"],
- Concept(
- domain="my_domain",
- code="ConceptWithRefines",
- definition="A concept with refines",
- structure_class_name=NATIVE_CONCEPTS_DATA[NativeConceptEnum.IMAGE].content_class_name,
- refines="native.Image",
- ),
- ),
- ]
-
- # Legacy alias for backward compatibility
- TEST_CASES = MAKE_REFINES_TEST_CASES
diff --git a/tests/unit/pipelex/core/concepts/concept_factory.py/test_concept_factory.py b/tests/unit/pipelex/core/concepts/concept_factory.py/test_concept_factory.py
deleted file mode 100644
index 8ea032084..000000000
--- a/tests/unit/pipelex/core/concepts/concept_factory.py/test_concept_factory.py
+++ /dev/null
@@ -1,123 +0,0 @@
-import pytest
-from typing import Optional, List
-
-from pipelex.core.concepts.concept import Concept
-from pipelex.core.concepts.concept_blueprint import (
- ConceptBlueprint,
- ConceptStructureBlueprint,
- ConceptStructureBlueprintFieldType,
-)
-from pipelex.core.concepts.concept_factory import ConceptFactory
-
-from .data import TestCases
-
-
-class TestConceptFactory:
- """Test ConceptFactory methods with various configurations."""
-
- @pytest.mark.parametrize(
- "test_name,blueprint,expected_result",
- TestCases.MAKE_REFINES_TEST_CASES,
- )
- def test_make_refines(
- self,
- test_name: str,
- blueprint: ConceptBlueprint,
- expected_result: str,
- ):
- """Test make_refines method with different blueprint configurations."""
- result = ConceptFactory.make_refines(blueprint=blueprint)
- assert result == expected_result, f"Failed for test case: {test_name}"
-
- def test_normalize_structure_blueprint(self):
- """Test that mixed structure blueprints are properly normalized."""
- mixed_structure_blueprint: dict[str, str | ConceptStructureBlueprint] = {
- "name": "The name of the person",
- "age": ConceptStructureBlueprint(definition="The age of the person", type=ConceptStructureBlueprintFieldType.NUMBER, required=True),
- "active": ConceptStructureBlueprint(
- definition="Whether the person is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True,
- ),
- }
-
- expected_structure: dict[str, ConceptStructureBlueprint] = {
- "name": ConceptStructureBlueprint(definition="The name of the person", type=ConceptStructureBlueprintFieldType.TEXT, required=True),
- "age": ConceptStructureBlueprint(definition="The age of the person", type=ConceptStructureBlueprintFieldType.NUMBER, required=True),
- "active": ConceptStructureBlueprint(
- definition="Whether the person is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True,
- ),
- }
-
- assert ConceptFactory.normalize_structure_blueprint(mixed_structure_blueprint) == expected_structure
-
- mixed_structure_blueprint2: dict[str, str | ConceptStructureBlueprint] = {
- "name": ConceptStructureBlueprint(definition="The name of the person", type=ConceptStructureBlueprintFieldType.TEXT, required=True),
- "age": ConceptStructureBlueprint(definition="The age of the person", type=ConceptStructureBlueprintFieldType.NUMBER, required=True),
- "active": ConceptStructureBlueprint(
- definition="Whether the person is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True,
- ),
- }
-
- expected_structure2: dict[str, ConceptStructureBlueprint] = {
- "name": ConceptStructureBlueprint(definition="The name of the person", type=ConceptStructureBlueprintFieldType.TEXT, required=True),
- "age": ConceptStructureBlueprint(definition="The age of the person", type=ConceptStructureBlueprintFieldType.NUMBER, required=True),
- "active": ConceptStructureBlueprint(
- definition="Whether the person is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True,
- ),
- }
-
- assert ConceptFactory.normalize_structure_blueprint(mixed_structure_blueprint2) == expected_structure2
-
- @pytest.mark.parametrize(
- "domain,concept_string_or_code,concept_codes_from_the_same_domain,expected_result",
- TestCases.MAKE_DOMAIN_AND_CONCEPT_CODE_TEST_CASES,
- )
- def test_make_domain_and_concept_code_from_concept_string_or_code(
- self,
- domain: str,
- concept_string_or_code: str,
- concept_codes_from_the_same_domain: List[str] | None,
- expected_result: List[str],
- ):
- """Test make_domain_and_concept_code_from_concept_string_or_code method with various inputs."""
- result = ConceptFactory.make_domain_and_concept_code_from_concept_string_or_code(
- domain=domain,
- concept_string_or_code=concept_string_or_code,
- concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,
- )
- assert result == expected_result
-
- @pytest.mark.parametrize(
- "test_name,domain,concept_code,blueprint,concept_codes_from_the_same_domain,expected_concept",
- TestCases.MAKE_FROM_BLUEPRINT_TEST_CASES,
- )
- def test_make_from_blueprint(
- self,
- test_name: str,
- domain: str,
- concept_code: str,
- blueprint: ConceptBlueprint,
- concept_codes_from_the_same_domain: list[str] | None,
- expected_concept: Concept,
- ):
- """Test make_from_blueprint method with various blueprint configurations."""
- result = ConceptFactory.make_from_blueprint(
- domain=domain,
- concept_code=concept_code,
- blueprint=blueprint,
- concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,
- )
-
- assert result == expected_concept, f"Concept mismatch for {test_name}"
-
- def test_make_from_blueprint_with_invalid_structure_class(self):
- """Test that make_from_blueprint raises StructureClassError for invalid structure class."""
- blueprint = ConceptBlueprint(definition="A concept with invalid structure", structure="NonExistentStructureClass")
-
- from pipelex.exceptions import StructureClassError
-
- with pytest.raises(StructureClassError, match="is not a registered subclass of StuffContent"):
- ConceptFactory.make_from_blueprint(
- domain="my_domain",
- concept_code="TestConcept",
- blueprint=blueprint,
- )
diff --git a/tests/unit/pipelex/core/concepts/concept_factory/data.py b/tests/unit/pipelex/core/concepts/concept_factory/data.py
index 930b8696a..2a635dc09 100644
--- a/tests/unit/pipelex/core/concepts/concept_factory/data.py
+++ b/tests/unit/pipelex/core/concepts/concept_factory/data.py
@@ -1,5 +1,3 @@
-"""Test data for ConceptFactory tests."""
-
from typing import ClassVar
from pipelex.core.concepts.concept import Concept
@@ -14,22 +12,20 @@ from pipelex.core.domains.domain import SpecialDomain
class TestCases:
- """Test cases for ConceptFactory methods."""
-
# Test cases for make_refines method - only native concepts can be refined
MAKE_REFINES_TEST_CASES: ClassVar[list[tuple[str, ConceptBlueprint, str]]] = [
(
"native_concept_string",
- ConceptBlueprint(definition="A concept that refines a native text concept", refines=NativeConceptEnum.TEXT.value),
- f"{SpecialDomain.NATIVE.value}.{NativeConceptEnum.TEXT.value}",
+ ConceptBlueprint(definition="A concept that refines a native text concept", refines=NativeConceptEnum.TEXT),
+ f"{SpecialDomain.NATIVE}.{NativeConceptEnum.TEXT}",
),
(
"fully_qualified_native_string",
ConceptBlueprint(
definition="A concept that refines a fully qualified native concept",
- refines=f"{SpecialDomain.NATIVE.value}.{NativeConceptEnum.TEXT.value}",
+ refines=f"{SpecialDomain.NATIVE}.{NativeConceptEnum.TEXT}",
),
- f"{SpecialDomain.NATIVE.value}.{NativeConceptEnum.TEXT.value}",
+ f"{SpecialDomain.NATIVE}.{NativeConceptEnum.TEXT}",
),
]
diff --git a/tests/unit/pipelex/core/concepts/concept_factory/test_concept_factory.py b/tests/unit/pipelex/core/concepts/concept_factory/test_concept_factory.py
index 195a43805..b8a47a9d1 100644
--- a/tests/unit/pipelex/core/concepts/concept_factory/test_concept_factory.py
+++ b/tests/unit/pipelex/core/concepts/concept_factory/test_concept_factory.py
@@ -1,6 +1,3 @@
-"""Tests for ConceptFactory methods."""
-
-
import pytest
from pipelex.core.concepts.concept import Concept
@@ -21,14 +18,16 @@ class TestConceptFactory:
"test_name,blueprint,expected_result",
TestCases.MAKE_REFINES_TEST_CASES,
)
- def test_make_refines(
+ def test_make_refine(
self,
test_name: str,
blueprint: ConceptBlueprint,
expected_result: str,
):
"""Test make_refines method with different blueprint configurations."""
- result = ConceptFactory.make_refines(blueprint=blueprint)
+ if not blueprint.refines:
+ pytest.skip("Test case has no refines")
+ result = ConceptFactory.make_refine(refine=blueprint.refines)
assert result == expected_result, f"Failed for test case: {test_name}"
def test_normalize_structure_blueprint(self):
@@ -73,7 +72,7 @@ class TestConceptFactory:
"domain,concept_string_or_code,concept_codes_from_the_same_domain,expected_result",
TestCases.MAKE_DOMAIN_AND_CONCEPT_CODE_TEST_CASES,
)
- def test_make_domain_and_concept_code_from_concept_string_orcode(
+ def test_make_domain_and_concept_code_from_concept_string_or_code(
self,
domain: str,
concept_string_or_code: str,
diff --git a/tests/unit/pipelex/create/test_structured_output_generator.py b/tests/unit/pipelex/core/concepts/test_structured_output_generator.py
similarity index 100%
rename from tests/unit/pipelex/create/test_structured_output_generator.py
rename to tests/unit/pipelex/core/concepts/test_structured_output_generator.py
diff --git a/tests/unit/pipelex/core/interpreter/test_interpreter.py b/tests/unit/pipelex/core/interpreter/test_interpreter.py
index 12bf546de..74bcade72 100644
--- a/tests/unit/pipelex/core/interpreter/test_interpreter.py
+++ b/tests/unit/pipelex/core/interpreter/test_interpreter.py
@@ -1,8 +1,9 @@
from pathlib import Path
+from typing import Type
import pytest
-from pipelex import pretty_print
+from pipelex import log, pretty_print
from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint
from pipelex.core.interpreter import PipelexInterpreter
from tests.unit.pipelex.core.test_data import InterpreterTestCases
@@ -10,6 +11,7 @@ from tests.unit.pipelex.core.test_data import InterpreterTestCases
class TestPipelexInterpreter:
def test_init_with_both_file_path_and_content(self, tmp_path: Path):
+ """Test initialization with both file_path and file_content."""
test_file = tmp_path / "test.plx"
test_file.write_text("domain = 'test'\n[concept]\n")
content = "domain = 'other'\n[concept]\n"
@@ -20,25 +22,19 @@ class TestPipelexInterpreter:
@pytest.mark.parametrize("test_name,plx_content,expected_blueprint", InterpreterTestCases.VALID_TEST_CASES)
def test_make_pipelex_bundle_blueprint(self, test_name: str, plx_content: str, expected_blueprint: PipelexBundleBlueprint):
+ """Test making blueprint from various valid PLX content."""
converter = PipelexInterpreter(file_content=plx_content)
-
blueprint = converter.make_pipelex_bundle_blueprint()
pretty_print(blueprint, title=f"Blueprint {test_name}")
pretty_print(expected_blueprint, title=f"Expected blueprint {test_name}")
assert blueprint == expected_blueprint
- @pytest.mark.parametrize("test_name,plx_content,expected_blueprint", InterpreterTestCases.VALID_TEST_CASES)
- def test_make_plx_content(self, test_name: str, plx_content: str, expected_blueprint: PipelexBundleBlueprint):
- get_plx_content = PipelexInterpreter.make_plx_content(blueprint=expected_blueprint)
-
- pretty_print(get_plx_content, title=f"PLx content {test_name}")
- pretty_print(plx_content, title=f"Expected PLX content {test_name}")
- assert get_plx_content == plx_content
-
@pytest.mark.parametrize("test_name,invalid_plx_content,expected_exception", InterpreterTestCases.ERROR_TEST_CASES)
- def test_invalid_plx_should_raise_exception(self, test_name: str, invalid_plx_content: str, expected_exception: type[Exception]):
+ def test_invalid_plx_should_raise_exception(self, test_name: str, invalid_plx_content: str, expected_exception: Type[Exception]):
+ """Test that invalid PLX content raises appropriate exceptions."""
converter = PipelexInterpreter(file_content=invalid_plx_content)
+ log.verbose(f"Testing invalid PLX content: {test_name}")
with pytest.raises(expected_exception):
converter.make_pipelex_bundle_blueprint()
diff --git a/tests/unit/pipelex/core/interpreter/test_interpreter_serialize.py b/tests/unit/pipelex/core/interpreter/test_interpreter_serialize.py
deleted file mode 100644
index 97145efd8..000000000
--- a/tests/unit/pipelex/core/interpreter/test_interpreter_serialize.py
+++ /dev/null
@@ -1,318 +0,0 @@
-from typing import Any
-
-from pipelex.core.concepts.concept_blueprint import ConceptBlueprint, ConceptStructureBlueprint, ConceptStructureBlueprintFieldType
-from pipelex.core.interpreter import PipelexInterpreter
-from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
-from pipelex.pipe_controllers.sequence.pipe_sequence_blueprint import PipeSequenceBlueprint
-from pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprint
-from pipelex.pipe_operators.llm.pipe_llm_blueprint import PipeLLMBlueprint
-from pipelex.pipe_operators.ocr.pipe_ocr_blueprint import PipeOcrBlueprint
-
-
-class TestSerializeInputRequirement:
- def test_serialize_input_requirement_basic(self):
- input_req = InputRequirementBlueprint(concept="Text")
- result = PipelexInterpreter.serialize_input_requirement(input_req)
-
- expected = {"concept": "Text"}
- assert result == expected
-
- def test_serialize_input_requirement_with_multiplicity_int(self):
- input_req = InputRequirementBlueprint(concept="Text", multiplicity=3)
- result = PipelexInterpreter.serialize_input_requirement(input_req)
-
- expected = {"concept": "Text", "multiplicity": 3}
- assert result == expected
-
- def test_serialize_input_requirement_with_multiplicity_bool(self):
- input_req = InputRequirementBlueprint(concept="Text", multiplicity=True)
- result = PipelexInterpreter.serialize_input_requirement(input_req)
-
- expected = {"concept": "Text", "multiplicity": True}
- assert result == expected
-
-
-class TestSerializeInputs:
- def test_serialize_inputs_mixed_types(self):
- inputs: dict[str, str | InputRequirementBlueprint] = {
- "text": "Text",
- "data": InputRequirementBlueprint(concept="Data", multiplicity=2),
- "query": InputRequirementBlueprint(concept="Text"),
- }
- result = PipelexInterpreter.serialize_inputs(inputs)
-
- expected = {"text": "Text", "data": {"concept": "Data", "multiplicity": 2}, "query": {"concept": "Text"}}
- assert result == expected
-
- def test_serialize_inputs_empty(self):
- inputs: dict[str, Any] = {}
- result = PipelexInterpreter.serialize_inputs(inputs)
-
- assert result == {}
-
-
-class TestSerializeConceptStructure:
-
- def test_serialize_concept_structure_field_string(self):
- field_value = "Simple description"
- result = PipelexInterpreter.serialize_concept_structure_field(field_value)
-
- assert result == "Simple description"
-
- def test_serialize_concept_structure_field_blueprint(self):
- field_value = ConceptStructureBlueprint(definition="A text field", type=ConceptStructureBlueprintFieldType.TEXT, required=True)
- result = PipelexInterpreter.serialize_concept_structure_field(field_value)
-
- expected = {"definition": "A text field", "required": True, "type": "text"}
- assert result == expected
-
- def test_serialize_concept_structure_field_blueprint_with_all_fields(self):
- field_value = ConceptStructureBlueprint(
- definition="A complex field",
- type=ConceptStructureBlueprintFieldType.DICT,
- key_type="string",
- value_type="integer",
- choices=["option1", "option2"],
- required=False,
- default_value={"default": 42},
- )
- result = PipelexInterpreter.serialize_concept_structure_field(field_value)
-
- expected = {
- "definition": "A complex field",
- "required": False,
- "type": "dict",
- "key_type": "string",
- "value_type": "integer",
- "choices": ["option1", "option2"],
- "default_value": {"default": 42},
- }
- assert result == expected
-
- def test_serialize_concept_structure_string(self):
- structure = "Simple structure"
- result = PipelexInterpreter.serialize_concept_structure(structure)
-
- assert result == "Simple structure"
-
- def test_serialize_concept_structure_dict(self):
- structure = {
- "name": "Person's name",
- "age": ConceptStructureBlueprint(definition="Person's age", type=ConceptStructureBlueprintFieldType.INTEGER, required=True),
- }
- result = PipelexInterpreter.serialize_concept_structure(structure)
-
- expected = {
- "name": "Person's name",
- "age": {"definition": "Person's age", "required": True, "type": "integer"},
- }
- assert result == expected
-
-
-class TestSerializeSingleConcept:
-
- def test_serialize_single_concept_string(self):
- concept = "Simple concept definition"
- result = PipelexInterpreter.serialize_single_concept(concept)
-
- assert result == "Simple concept definition"
-
- def test_serialize_single_concept_simple_definition(self):
- concept = ConceptBlueprint(definition="A simple concept")
- result = PipelexInterpreter.serialize_single_concept(concept)
-
- assert result == "A simple concept"
-
- def test_serialize_single_concept_with_refines(self):
- concept = ConceptBlueprint(definition="A specialized text", refines="Text")
- result = PipelexInterpreter.serialize_single_concept(concept)
-
- expected = {"definition": "A specialized text", "refines": "Text"}
- assert result == expected
-
- def test_serialize_single_concept_with_structure(self):
- concept = ConceptBlueprint(
- definition="A person",
- structure={
- "name": "Person's name",
- "age": ConceptStructureBlueprint(definition="Person's age", type=ConceptStructureBlueprintFieldType.INTEGER, required=True),
- },
- )
- result = PipelexInterpreter.serialize_single_concept(concept)
-
- expected = {
- "definition": "A person",
- "structure": {
- "name": "Person's name",
- "age": {"definition": "Person's age", "required": True, "type": "integer"},
- },
- }
- assert result == expected
-
-
-class TestSerializeConcepts:
- def test_serialize_concepts_empty(self):
- result = PipelexInterpreter.serialize_concepts(None)
- assert result == {}
-
- def test_serialize_concepts_mixed(self):
- concepts: dict[str, ConceptBlueprint | str] = {
- "SimpleText": "A simple text concept",
- "PersonInfo": ConceptBlueprint(definition="Information about a person", refines="Text"),
- "StructuredData": ConceptBlueprint(
- definition="Structured data",
- structure={
- "title": "Data title",
- "value": ConceptStructureBlueprint(definition="Data value", type=ConceptStructureBlueprintFieldType.NUMBER, required=True),
- },
- ),
- }
- result = PipelexInterpreter.serialize_concepts(concepts)
-
- expected = {
- "SimpleText": "A simple text concept",
- "PersonInfo": {"definition": "Information about a person", "refines": "Text"},
- "StructuredData": {
- "definition": "Structured data",
- "structure": {
- "title": "Data title",
- "value": {"definition": "Data value", "required": True, "type": "number"},
- },
- },
- }
- assert result == expected
-
-
-class TestSerializeSubPipe:
-
- def test_serialize_sub_pipe_basic(self):
- sub_pipe = SubPipeBlueprint(pipe="test_pipe", result="test_result")
- result = PipelexInterpreter.serialize_sub_pipe(sub_pipe)
-
- expected = {"pipe": "test_pipe", "result": "test_result"}
- assert result == expected
-
- def test_serialize_sub_pipe_with_nb_output_and_batch(self):
- sub_pipe = SubPipeBlueprint(pipe="test_pipe", result="test_result", nb_output=3, batch_over="input_list", batch_as="current_item")
- result = PipelexInterpreter.serialize_sub_pipe(sub_pipe)
-
- expected = {"pipe": "test_pipe", "result": "test_result", "nb_output": 3, "batch_over": "input_list", "batch_as": "current_item"}
- assert result == expected
-
- def test_serialize_sub_pipe_with_multiple_output_and_batch(self):
- sub_pipe = SubPipeBlueprint(pipe="test_pipe", result="test_result", multiple_output=True, batch_over="input_list", batch_as="current_item")
- result = PipelexInterpreter.serialize_sub_pipe(sub_pipe)
-
- expected = {"pipe": "test_pipe", "result": "test_result", "multiple_output": True, "batch_over": "input_list", "batch_as": "current_item"}
- assert result == expected
-
- def test_serialize_sub_pipe_skip_defaults(self):
- sub_pipe = SubPipeBlueprint(
- pipe="test_pipe",
- result="test_result",
- batch_over=False, # Default value
- batch_as=None, # Default value
- )
- result = PipelexInterpreter.serialize_sub_pipe(sub_pipe)
-
- expected = {"pipe": "test_pipe", "result": "test_result"}
- assert result == expected
-
-
-class TestSerializePipeBlueprints:
- def test_serialize_llm_pipe_basic(self):
- pipe = PipeLLMBlueprint(type="PipeLLM", definition="Generate text using LLM", output="Text", prompt_template="Generate a story")
- result = PipelexInterpreter.serialize_llm_pipe(pipe)
-
- expected = {"type": "PipeLLM", "definition": "Generate text using LLM", "output": "Text", "prompt_template": "Generate a story"}
- assert result == expected
-
- def test_serialize_llm_pipe_with_inputs_and_options(self):
- pipe = PipeLLMBlueprint(
- type="PipeLLM",
- definition="Extract information",
- inputs={"text": "Text", "topic": InputRequirementBlueprint(concept="Text", multiplicity=1)},
- output="ExtractedInfo",
- system_prompt="You are an expert extractor",
- prompt_template="Extract info about $topic from @text",
- nb_output=2,
- )
- result = PipelexInterpreter.serialize_llm_pipe(pipe)
-
- expected = {
- "type": "PipeLLM",
- "definition": "Extract information",
- "inputs": {"text": "Text", "topic": {"concept": "Text", "multiplicity": 1}},
- "output": "ExtractedInfo",
- "nb_output": 2,
- "system_prompt": "You are an expert extractor",
- "prompt_template": "Extract info about $topic from @text",
- }
- assert result == expected
-
- def test_serialize_ocr_pipe_basic(self):
- pipe = PipeOcrBlueprint(type="PipeOcr", definition="Extract text from PDF", output="Page", ocr="base_ocr_pypdfium2")
- result = PipelexInterpreter.serialize_ocr_pipe(pipe)
-
- expected = {"type": "PipeOcr", "definition": "Extract text from PDF", "output": "Page", "ocr": "base_ocr_pypdfium2"}
- assert result == expected
-
- def test_serialize_sequence_pipe_basic(self):
- pipe = PipeSequenceBlueprint(
- type="PipeSequence",
- definition="Process in sequence",
- output="Result",
- steps=[SubPipeBlueprint(pipe="step1", result="result1"), SubPipeBlueprint(pipe="step2", result="result2")],
- )
- result = PipelexInterpreter.serialize_sequence_pipe(pipe)
-
- expected = {
- "type": "PipeSequence",
- "definition": "Process in sequence",
- "output": "Result",
- "steps": [{"pipe": "step1", "result": "result1"}, {"pipe": "step2", "result": "result2"}],
- }
- assert result == expected
-
-
-class TestAddCommonPipeFields:
- def test_add_common_pipe_fields_with_inputs(self):
- result: dict[str, Any] = {"type": "TestPipe"}
-
- # Mock pipe with inputs
- class MockPipe:
- def __init__(self):
- self.inputs = {"text": "Text", "data": InputRequirementBlueprint(concept="Data")}
-
- pipe = MockPipe()
- PipelexInterpreter.add_common_pipe_fields(result, pipe)
-
- expected = {"type": "TestPipe", "inputs": {"text": "Text", "data": {"concept": "Data"}}}
- assert result == expected
-
- def test_add_common_pipe_fields_without_inputs(self):
- result: dict[str, Any] = {"type": "TestPipe"}
-
- # Mock pipe without inputs
- class MockPipe:
- def __init__(self):
- self.inputs = None
-
- pipe = MockPipe()
- PipelexInterpreter.add_common_pipe_fields(result, pipe)
-
- expected = {"type": "TestPipe"}
- assert result == expected
-
- def test_add_common_pipe_fields_no_inputs_attribute(self):
- result: dict[str, Any] = {"type": "TestPipe"}
-
- # Mock pipe without inputs attribute
- class MockPipe:
- pass
-
- pipe = MockPipe()
- PipelexInterpreter.add_common_pipe_fields(result, pipe)
-
- expected = {"type": "TestPipe"}
- assert result == expected
diff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_batch_plx.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_batch_plx.py
deleted file mode 100644
index 7e55a2efd..000000000
--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_batch_plx.py
+++ /dev/null
@@ -1,52 +0,0 @@
-import pytest
-
-from pipelex.core.interpreter import PipelexInterpreter
-from pipelex.pipe_controllers.batch.pipe_batch_blueprint import PipeBatchBlueprint
-
-
-class TestPipelexInterpreterBatchPLX:
- @pytest.mark.parametrize(
- "pipe_name,blueprint,expected_plx",
- [
- # Basic Batch pipe
- (
- "batch_process",
- PipeBatchBlueprint(
- type="PipeBatch",
- definition="Process items in batch",
- output="ProcessedData",
- branch_pipe_code="process_item",
- input_list_name="items",
- input_item_name="current_item",
- ),
- """[pipe.batch_process]
-type = "PipeBatch"
-definition = "Process items in batch"
-output = "ProcessedData"
-branch_pipe_code = "process_item"
-input_list_name = "items"
-input_item_name = "current_item\"""",
- ),
- # Batch pipe with inputs
- (
- "batch_with_inputs",
- PipeBatchBlueprint(
- type="PipeBatch",
- definition="Batch processing with inputs",
- inputs={"data_list": "Text", "config": "Text"},
- output="ProcessedData",
- branch_pipe_code="transform_item",
- ),
- """[pipe.batch_with_inputs]
-type = "PipeBatch"
-definition = "Batch processing with inputs"
-inputs = { data_list = "Text", config = "Text" }
-output = "ProcessedData"
-branch_pipe_code = "transform_item\"""",
- ),
- ],
- )
- def test_batch_pipe_to_plx_string(self, pipe_name: str, blueprint: PipeBatchBlueprint, expected_plx: str):
- """Test converting Batch pipe blueprint to PLX string."""
- result = PipelexInterpreter.batch_pipe_to_plx_string(pipe_name, blueprint)
- assert result == expected_plx
diff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_concept_plx.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_concept_plx.py
deleted file mode 100644
index 3e319da41..000000000
--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_concept_plx.py
+++ /dev/null
@@ -1,95 +0,0 @@
-import pytest
-
-from pipelex.core.concepts.concept_blueprint import ConceptBlueprint, ConceptStructureBlueprint, ConceptStructureBlueprintFieldType
-from pipelex.core.interpreter import PipelexInterpreter
-
-
-class TestPipelexInterpreterConceptPLX:
- @pytest.mark.parametrize(
- "concepts,expected_plx",
- [
- # Simple string concepts
- (
- {"ProcessedData": "Data that has been processed", "UserInfo": "Information about a user"},
- """[concept]
-ProcessedData = "Data that has been processed"
-UserInfo = "Information about a user\"""",
- ),
- # Single concept
- (
- {"Document": "A text document"},
- """[concept]
-Document = "A text document\"""",
- ),
- # ConceptBlueprint objects
- (
- {
- "ProcessedData": ConceptBlueprint(definition="Data that has been processed"),
- "UserInfo": ConceptBlueprint(definition="Information about a user"),
- },
- """[concept]
-ProcessedData = "Data that has been processed"
-UserInfo = "Information about a user\"""",
- ),
- # Complex ConceptBlueprint with structure
- (
- {
- "FeatureAnalysis": ConceptBlueprint(
- definition="Analysis of a photo's visual content and key features",
- structure={
- "dominant_feature": ConceptStructureBlueprint(
- definition="The most important or dominant feature in the image",
- type=ConceptStructureBlueprintFieldType.TEXT,
- required=True,
- ),
- "visual_elements": ConceptStructureBlueprint(
- definition="Description of key visual elements present",
- type=ConceptStructureBlueprintFieldType.TEXT,
- required=True,
- ),
- "composition": "Analysis of the image composition",
- "color_palette": "Description of the main colors in the image",
- "mood_atmosphere": "The overall mood or atmosphere of the image",
- },
- ),
- },
- """[concept.FeatureAnalysis]
-definition = "Analysis of a photo's visual content and key features"
-
-[concept.FeatureAnalysis.structure]
-dominant_feature = { type = "text", definition = "The most important or dominant feature in the image", required = true }
-visual_elements = { type = "text", definition = "Description of key visual elements present", required = true }
-composition = "Analysis of the image composition"
-color_palette = "Description of the main colors in the image"
-mood_atmosphere = "The overall mood or atmosphere of the image\"""",
- ),
- # ConceptBlueprint with refines
- (
- {
- "EnhancedText": ConceptBlueprint(
- definition="Enhanced text content with additional metadata",
- refines="Text",
- ),
- },
- """[concept.EnhancedText]
-definition = "Enhanced text content with additional metadata"
-refines = "Text\"""",
- ),
- # ConceptBlueprint with structure name
- (
- {
- "ProductInfo": ConceptBlueprint(
- definition="Detailed information about a product",
- structure="ProductStructure",
- ),
- },
- """[concept.ProductInfo]
-definition = "Detailed information about a product"
-structure = "ProductStructure\"""",
- ),
- ],
- )
- def test_concepts_to_plx_string(self, concepts: dict[str, ConceptBlueprint | str], expected_plx: str):
- """Test converting concepts dict to PLX string."""
- result = PipelexInterpreter.concepts_to_plx_string(concepts)
- assert result == expected_plx
diff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_condition_plx.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_condition_plx.py
deleted file mode 100644
index fd7098489..000000000
--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_condition_plx.py
+++ /dev/null
@@ -1,52 +0,0 @@
-import pytest
-
-from pipelex.core.interpreter import PipelexInterpreter
-from pipelex.pipe_controllers.condition.pipe_condition_blueprint import PipeConditionBlueprint, PipeConditionPipeMapBlueprint
-
-
-class TestPipelexInterpreterConditionPLX:
- @pytest.mark.parametrize(
- "pipe_name,blueprint,expected_plx",
- [
- # Basic Condition pipe with expression
- (
- "conditional_process",
- PipeConditionBlueprint(
- type="PipeCondition",
- definition="Process based on condition",
- output="ProcessedData",
- pipe_map=PipeConditionPipeMapBlueprint({"small": "process_small", "large": "process_large"}),
- expression="input_data.category",
- ),
- """[pipe.conditional_process]
-type = "PipeCondition"
-definition = "Process based on condition"
-output = "ProcessedData"
-pipe_map = { small = "process_small", large = "process_large" }
-expression = "input_data.category\"""",
- ),
- # Condition pipe with expression template and inputs
- (
- "complex_condition",
- PipeConditionBlueprint(
- type="PipeCondition",
- definition="Complex conditional processing",
- inputs={"data": "Text", "threshold": "Text"},
- output="ProcessedData",
- pipe_map=PipeConditionPipeMapBlueprint({"low": "process_low", "medium": "process_medium", "high": "process_high"}),
- expression_template="{{ data.size if data.size > threshold else 'low' }}",
- ),
- """[pipe.complex_condition]
-type = "PipeCondition"
-definition = "Complex conditional processing"
-inputs = { data = "Text", threshold = "Text" }
-output = "ProcessedData"
-pipe_map = { low = "process_low", medium = "process_medium", high = "process_high" }
-expression_template = "{{ data.size if data.size > threshold else 'low' }}\"""",
- ),
- ],
- )
- def test_condition_pipe_to_plx_string(self, pipe_name: str, blueprint: PipeConditionBlueprint, expected_plx: str):
- """Test converting Condition pipe blueprint to PLX string."""
- result = PipelexInterpreter.condition_pipe_to_plx_string(pipe_name, blueprint)
- assert result == expected_plx
diff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_func_plx.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_func_plx.py
deleted file mode 100644
index 80d6d53fb..000000000
--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_func_plx.py
+++ /dev/null
@@ -1,47 +0,0 @@
-import pytest
-
-from pipelex.core.interpreter import PipelexInterpreter
-from pipelex.pipe_operators.func.pipe_func_blueprint import PipeFuncBlueprint
-
-
-class TestPipelexInterpreterFuncPLX:
- @pytest.mark.parametrize(
- "pipe_name,blueprint,expected_plx",
- [
- # Basic Func pipe
- (
- "process_data",
- PipeFuncBlueprint(
- type="PipeFunc",
- definition="Process data with function",
- output="ProcessedData",
- function_name="process_data_function",
- ),
- """[pipe.process_data]
-type = "PipeFunc"
-definition = "Process data with function"
-output = "ProcessedData"
-function_name = "process_data_function\"""",
- ),
- # Func pipe with inputs
- (
- "transform_data",
- PipeFuncBlueprint(
- type="PipeFunc",
- definition="Transform input data",
- inputs={"data": "Text", "config": "Text"},
- output="TransformedData",
- function_name="transform_function",
- ),
- """[pipe.transform_data]
-type = "PipeFunc"
-definition = "Transform input data"
-inputs = { data = "Text", config = "Text" }
-output = "TransformedData"
-function_name = "transform_function\"""",
- ),
- ],
- )
- def test_func_pipe_to_plx_string(self, pipe_name: str, blueprint: PipeFuncBlueprint, expected_plx: str):
- result = PipelexInterpreter.func_pipe_to_plx_string(pipe_name, blueprint)
- assert result == expected_plx
diff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_img_gen_plx.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_img_gen_plx.py
deleted file mode 100644
index 5b02034f4..000000000
--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_img_gen_plx.py
+++ /dev/null
@@ -1,50 +0,0 @@
-import pytest
-
-from pipelex.cogt.img_gen.img_gen_job_components import AspectRatio
-from pipelex.core.interpreter import PipelexInterpreter
-from pipelex.pipe_operators.img_gen.pipe_img_gen_blueprint import PipeImgGenBlueprint
-
-
-class TestPipelexInterpreterImgGenPLX:
- @pytest.mark.parametrize(
- "pipe_name,blueprint,expected_plx",
- [
- # Basic ImgGen pipe
- (
- "generate_image",
- PipeImgGenBlueprint(
- type="PipeImgGen",
- definition="Generate an image from a prompt",
- output="Image",
- img_gen_prompt="A beautiful landscape",
- ),
- """[pipe.generate_image]
-type = "PipeImgGen"
-definition = "Generate an image from a prompt"
-output = "Image"
-img_gen_prompt = "A beautiful landscape\"""",
- ),
- # ImgGen pipe with some options
- (
- "generate_with_options",
- PipeImgGenBlueprint(
- type="PipeImgGen",
- definition="Generate image with options",
- output="Image",
- img_gen_prompt="A futuristic city",
- aspect_ratio=AspectRatio.LANDSCAPE_16_9,
- seed=12345,
- ),
- """[pipe.generate_with_options]
-type = "PipeImgGen"
-definition = "Generate image with options"
-output = "Image"
-img_gen_prompt = "A futuristic city"
-aspect_ratio = "landscape_16_9"
-seed = 12345""",
- ),
- ],
- )
- def test_img_gen_pipe_to_plx_string(self, pipe_name: str, blueprint: PipeImgGenBlueprint, expected_plx: str):
- result = PipelexInterpreter.img_gen_pipe_to_plx_string(pipe_name, blueprint)
- assert result == expected_plx
diff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_input_spec.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_input_spec.py
deleted file mode 100644
index ff135bbe1..000000000
--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_input_spec.py
+++ /dev/null
@@ -1,53 +0,0 @@
-from typing import Any
-
-import pytest
-
-from pipelex.core.interpreter import PipelexInterpreter
-from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
-
-
-class TestPipelexInterpreterInputSpec:
- def test_inputs_to_plx_string_simple_strings(self):
- # Simple string inputs (like "Text")
- simple_inputs = {"text": "Text", "topic": "Text"}
-
- result = PipelexInterpreter.inputs_to_plx_string(simple_inputs)
-
- expected = '{ text = "Text", topic = "Text" }'
- assert result == expected
-
- def test_inputs_to_plx_string_input_requirement_blueprints(self):
- # InputRequirementBlueprint inputs
- complex_inputs = {
- "text": InputRequirementBlueprint(concept="Text", multiplicity=1),
- "problem": InputRequirementBlueprint(concept="Text"), # multiplicity=None
- "documents": InputRequirementBlueprint(concept="Text", multiplicity=True),
- "query": InputRequirementBlueprint(concept="Text", multiplicity=False),
- }
-
- result = PipelexInterpreter.inputs_to_plx_string(complex_inputs)
-
- expected = (
- '{ text = { concept = "Text", multiplicity = 1 }, problem = { concept = "Text" }, '
- 'documents = { concept = "Text", multiplicity = true }, query = { concept = "Text", multiplicity = false } }'
- )
- assert result == expected
-
- @pytest.mark.parametrize(
- "input_req,expected",
- [
- # Test with just concept (multiplicity=None should not be included)
- (InputRequirementBlueprint(concept="Text"), {"concept": "Text"}),
- # Test with concept and multiplicity=1
- (InputRequirementBlueprint(concept="Text", multiplicity=1), {"concept": "Text", "multiplicity": 1}),
- # Test with concept and multiplicity=True
- (InputRequirementBlueprint(concept="Text", multiplicity=True), {"concept": "Text", "multiplicity": True}),
- # Test with concept and multiplicity=False
- (InputRequirementBlueprint(concept="Text", multiplicity=False), {"concept": "Text", "multiplicity": False}),
- # Test with different concept
- (InputRequirementBlueprint(concept="Image", multiplicity=5), {"concept": "Image", "multiplicity": 5}),
- ],
- )
- def test_serialize_input_requirement(self, input_req: InputRequirementBlueprint, expected: dict[str, Any]):
- result = PipelexInterpreter.serialize_input_requirement(input_req)
- assert result == expected
diff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_jinja2_plx.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_jinja2_plx.py
deleted file mode 100644
index 525a17a9c..000000000
--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_jinja2_plx.py
+++ /dev/null
@@ -1,47 +0,0 @@
-import pytest
-
-from pipelex.core.interpreter import PipelexInterpreter
-from pipelex.pipe_operators.jinja2.pipe_jinja2_blueprint import PipeJinja2Blueprint
-
-
-class TestPipelexInterpreterJinja2PLX:
- @pytest.mark.parametrize(
- "pipe_name,blueprint,expected_plx",
- [
- # Basic Jinja2 pipe
- (
- "process_template",
- PipeJinja2Blueprint(
- type="PipeJinja2",
- definition="Process a Jinja2 template",
- output="Text",
- jinja2="Hello {{ name }}!",
- ),
- """[pipe.process_template]
-type = "PipeJinja2"
-definition = "Process a Jinja2 template"
-output = "Text"
-jinja2 = "Hello {{ name }}!\"""",
- ),
- # Jinja2 pipe with inputs and named template
- (
- "format_data",
- PipeJinja2Blueprint(
- type="PipeJinja2",
- definition="Format data with template",
- inputs={"data": "Text", "format": "Text"},
- output="Text",
- jinja2_name="data_formatter",
- ),
- """[pipe.format_data]
-type = "PipeJinja2"
-definition = "Format data with template"
-inputs = { data = "Text", format = "Text" }
-output = "Text"
-jinja2_name = "data_formatter\"""",
- ),
- ],
- )
- def test_jinja2_pipe_to_plx_string(self, pipe_name: str, blueprint: PipeJinja2Blueprint, expected_plx: str):
- result = PipelexInterpreter.jinja2_pipe_to_plx_string(pipe_name, blueprint)
- assert result == expected_plx
diff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_jinja2_spacing.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_jinja2_spacing.py
deleted file mode 100644
index 41b8ec9f7..000000000
--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_jinja2_spacing.py
+++ /dev/null
@@ -1,77 +0,0 @@
-import pytest
-
-from pipelex.core.interpreter import PipelexInterpreter
-from pipelex.pipe_operators.jinja2.pipe_jinja2_blueprint import PipeJinja2Blueprint
-
-
-class TestPipelexInterpreterJinja2PipePLX:
- @pytest.mark.parametrize(
- "pipe_name,blueprint,expected_plx",
- [
- # Basic Jinja2 pipe
- (
- "greeting_pipe",
- PipeJinja2Blueprint(
- type="PipeJinja2",
- definition="Process a Jinja2 template",
- output="Text",
- jinja2="Hello {{ name }}!",
- ),
- """[pipe.greeting_pipe]
-type = "PipeJinja2"
-definition = "Process a Jinja2 template"
-output = "Text"
-jinja2 = "Hello {{ name }}!\"""",
- ),
- # Jinja2 pipe with inputs
- (
- "template_pipe",
- PipeJinja2Blueprint(
- type="PipeJinja2",
- definition="Process template with inputs",
- inputs={"name": "Text", "greeting": "Text"},
- output="Text",
- jinja2="{{ greeting }} {{ name }}!",
- ),
- """[pipe.template_pipe]
-type = "PipeJinja2"
-definition = "Process template with inputs"
-inputs = { name = "Text", greeting = "Text" }
-output = "Text"
-jinja2 = "{{ greeting }} {{ name }}!\"""",
- ),
- # Jinja2 pipe with complex template
- (
- "complex_pipe",
- PipeJinja2Blueprint(
- type="PipeJinja2",
- definition="Process complex template",
- output="Text",
- jinja2="{% for item in items %}{{ item.name }}{% endfor %}",
- ),
- """[pipe.complex_pipe]
-type = "PipeJinja2"
-definition = "Process complex template"
-output = "Text"
-jinja2 = "{% for item in items %}{{ item.name }}{% endfor %}\"""",
- ),
- # Jinja2 pipe with jinja2_name instead of inline jinja2
- (
- "named_template_pipe",
- PipeJinja2Blueprint(
- type="PipeJinja2",
- definition="Use named template",
- output="Text",
- jinja2_name="greeting_template",
- ),
- """[pipe.named_template_pipe]
-type = "PipeJinja2"
-definition = "Use named template"
-output = "Text"
-jinja2_name = "greeting_template\"""",
- ),
- ],
- )
- def test_jinja2_pipe_to_plx_string(self, pipe_name: str, blueprint: PipeJinja2Blueprint, expected_plx: str):
- result = PipelexInterpreter.jinja2_pipe_to_plx_string(pipe_name, blueprint)
- assert result == expected_plx
diff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_llm_plx.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_llm_plx.py
deleted file mode 100644
index 5b9b0e6f4..000000000
--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_llm_plx.py
+++ /dev/null
@@ -1,91 +0,0 @@
-import pytest
-
-from pipelex.core.interpreter import PipelexInterpreter
-from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
-from pipelex.pipe_operators.llm.pipe_llm_blueprint import PipeLLMBlueprint
-
-
-class TestPipelexInterpreterLLMPLX:
- @pytest.mark.parametrize(
- "pipe_name,blueprint,expected_plx",
- [
- # Simple LLM pipe with string inputs
- (
- "simple_llm",
- PipeLLMBlueprint(
- type="PipeLLM",
- definition="Simple text generation",
- inputs={"text": "Text", "topic": "Text"},
- output="Text",
- prompt_template="Generate text about $topic based on: @text",
- ),
- """[pipe.simple_llm]
-type = "PipeLLM"
-definition = "Simple text generation"
-inputs = { text = "Text", topic = "Text" }
-output = "Text"
-prompt_template = "Generate text about $topic based on: @text\"""",
- ),
- # LLM pipe with complex inputs (InputRequirementBlueprint)
- (
- "complex_llm",
- PipeLLMBlueprint(
- type="PipeLLM",
- definition="Extract structured information",
- inputs={
- "documents": InputRequirementBlueprint(concept="Text", multiplicity=True),
- "query": InputRequirementBlueprint(concept="Text", multiplicity=False),
- },
- output="DocumentSummary",
- prompt_template="Extract information from documents: @documents based on query: $query",
- ),
- """[pipe.complex_llm]
-type = "PipeLLM"
-definition = "Extract structured information"
-inputs = { documents = { concept = "Text", multiplicity = true }, query = { concept = "Text", multiplicity = false } }
-output = "DocumentSummary"
-prompt_template = "Extract information from documents: @documents based on query: $query\"""",
- ),
- # LLM pipe with system prompt
- (
- "system_prompt_llm",
- PipeLLMBlueprint(
- type="PipeLLM",
- definition="Analysis with system prompt",
- inputs={"content": "Text"},
- output="Text",
- system_prompt_template="You are an expert analyst with $expertise_level knowledge",
- prompt_template="Analyze this content: @content",
- ),
- """[pipe.system_prompt_llm]
-type = "PipeLLM"
-definition = "Analysis with system prompt"
-inputs = { content = "Text" }
-output = "Text"
-system_prompt_template = "You are an expert analyst with $expertise_level knowledge"
-prompt_template = "Analyze this content: @content\"""",
- ),
- # LLM pipe with nb_output
- (
- "multiple_output_llm",
- PipeLLMBlueprint(
- type="PipeLLM",
- definition="Generate multiple outputs",
- inputs={"topic": "Text"},
- output="Text",
- nb_output=3,
- prompt_template="Generate ideas about: $topic",
- ),
- """[pipe.multiple_output_llm]
-type = "PipeLLM"
-definition = "Generate multiple outputs"
-inputs = { topic = "Text" }
-output = "Text"
-nb_output = 3
-prompt_template = "Generate ideas about: $topic\"""",
- ),
- ],
- )
- def test_llm_pipe_to_plx_string(self, pipe_name: str, blueprint: PipeLLMBlueprint, expected_plx: str):
- result = PipelexInterpreter.llm_pipe_to_plx_string(pipe_name, blueprint)
- assert result == expected_plx
diff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_ocr_plx.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_ocr_plx.py
deleted file mode 100644
index c8f6b3fc9..000000000
--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_ocr_plx.py
+++ /dev/null
@@ -1,47 +0,0 @@
-import pytest
-
-from pipelex.core.interpreter import PipelexInterpreter
-from pipelex.pipe_operators.ocr.pipe_ocr_blueprint import PipeOcrBlueprint
-
-
-class TestPipelexInterpreterOcrPLX:
- @pytest.mark.parametrize(
- "pipe_name,blueprint,expected_plx",
- [
- # Basic OCR pipe
- (
- "extract_text",
- PipeOcrBlueprint(
- type="PipeOcr",
- definition="Extract text from document",
- output="Page",
- ocr="base_ocr_pypdfium2",
- ),
- """[pipe.extract_text]
-type = "PipeOcr"
-definition = "Extract text from document"
-output = "Page"
-ocr = "base_ocr_pypdfium2\"""",
- ),
- # OCR pipe with inputs
- (
- "extract_with_input",
- PipeOcrBlueprint(
- type="PipeOcr",
- definition="Extract text from PDF",
- inputs={"ocr_input": "PDF"},
- output="Page",
- ocr="base_ocr_pypdfium2",
- ),
- """[pipe.extract_with_input]
-type = "PipeOcr"
-definition = "Extract text from PDF"
-inputs = { ocr_input = "PDF" }
-output = "Page"
-ocr = "base_ocr_pypdfium2\"""",
- ),
- ],
- )
- def test_ocr_pipe_to_plx_string(self, pipe_name: str, blueprint: PipeOcrBlueprint, expected_plx: str):
- result = PipelexInterpreter.ocr_pipe_to_plx_string(pipe_name, blueprint)
- assert result == expected_plx
diff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_parallel_plx.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_parallel_plx.py
deleted file mode 100644
index 4ddbac40d..000000000
--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_parallel_plx.py
+++ /dev/null
@@ -1,66 +0,0 @@
-import pytest
-
-from pipelex.core.interpreter import PipelexInterpreter
-from pipelex.pipe_controllers.parallel.pipe_parallel_blueprint import PipeParallelBlueprint
-from pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprint
-
-
-class TestPipelexInterpreterParallelPLX:
- @pytest.mark.parametrize(
- "pipe_name,blueprint,expected_plx",
- [
- # Basic Parallel pipe
- (
- "parallel_process",
- PipeParallelBlueprint(
- type="PipeParallel",
- definition="Process data in parallel",
- output="ProcessedData",
- parallels=[
- SubPipeBlueprint(pipe="process_a", result="result_a"),
- SubPipeBlueprint(pipe="process_b", result="result_b"),
- ],
- ),
- """[pipe.parallel_process]
-type = "PipeParallel"
-definition = "Process data in parallel"
-output = "ProcessedData"
-parallels = [
- { pipe = "process_a", result = "result_a" },
- { pipe = "process_b", result = "result_b" },
-]""",
- ),
- # Parallel pipe with inputs and options
- (
- "complex_parallel",
- PipeParallelBlueprint(
- type="PipeParallel",
- definition="Complex parallel processing",
- inputs={"data": "Text"},
- output="ProcessedData",
- parallels=[
- SubPipeBlueprint(pipe="analyze", result="analysis"),
- SubPipeBlueprint(pipe="summarize", result="summary"),
- SubPipeBlueprint(pipe="extract", result="extraction"),
- ],
- add_each_output=False,
- combined_output="MergedResults",
- ),
- """[pipe.complex_parallel]
-type = "PipeParallel"
-definition = "Complex parallel processing"
-inputs = { data = "Text" }
-output = "ProcessedData"
-parallels = [
- { pipe = "analyze", result = "analysis" },
- { pipe = "summarize", result = "summary" },
- { pipe = "extract", result = "extraction" },
-]
-add_each_output = false
-combined_output = "MergedResults\"""",
- ),
- ],
- )
- def test_parallel_pipe_to_plx_string(self, pipe_name: str, blueprint: PipeParallelBlueprint, expected_plx: str):
- result = PipelexInterpreter.parallel_pipe_to_plx_string(pipe_name, blueprint)
- assert result == expected_plx
diff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_sequence_plx.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_sequence_plx.py
deleted file mode 100644
index 8c2e15b65..000000000
--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_sequence_plx.py
+++ /dev/null
@@ -1,84 +0,0 @@
-import pytest
-
-from pipelex.core.interpreter import PipelexInterpreter
-from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
-from pipelex.pipe_controllers.sequence.pipe_sequence_blueprint import PipeSequenceBlueprint
-from pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprint
-
-
-class TestPipelexInterpreterSequencePLX:
- @pytest.mark.parametrize(
- "pipe_name,blueprint,expected_plx",
- [
- # Simple sequence pipe with 2 steps
- (
- "simple_sequence",
- PipeSequenceBlueprint(
- type="PipeSequence",
- definition="Process data in sequence",
- output="ProcessedData",
- steps=[
- SubPipeBlueprint(pipe="extract_info", result="extracted"),
- SubPipeBlueprint(pipe="summarize", result="final_summary"),
- ],
- ),
- """[pipe.simple_sequence]
-type = "PipeSequence"
-definition = "Process data in sequence"
-output = "ProcessedData"
-steps = [
- { pipe = "extract_info", result = "extracted" },
- { pipe = "summarize", result = "final_summary" },
-]""",
- ),
- # Sequence pipe with inputs and steps
- (
- "sequence_with_inputs",
- PipeSequenceBlueprint(
- type="PipeSequence",
- definition="Process data with inputs",
- inputs={"text": InputRequirementBlueprint(concept="Text", multiplicity=1), "topic": "Text"},
- output="ProcessedData",
- steps=[
- SubPipeBlueprint(pipe="extract_info", result="extracted"),
- SubPipeBlueprint(pipe="summarize", result="final_summary"),
- ],
- ),
- """[pipe.sequence_with_inputs]
-type = "PipeSequence"
-definition = "Process data with inputs"
-inputs = { text = { concept = "Text", multiplicity = 1 }, topic = "Text" }
-output = "ProcessedData"
-steps = [
- { pipe = "extract_info", result = "extracted" },
- { pipe = "summarize", result = "final_summary" },
-]""",
- ),
- # Sequence pipe with complex sub pipe options
- (
- "complex_sequence",
- PipeSequenceBlueprint(
- type="PipeSequence",
- definition="Complex sequence processing",
- output="ProcessedData",
- steps=[
- SubPipeBlueprint(pipe="step1", result="result1", nb_output=2),
- SubPipeBlueprint(pipe="step2", result="result2", multiple_output=True),
- SubPipeBlueprint(pipe="step3", result="result3", batch_over="items", batch_as="item"),
- ],
- ),
- """[pipe.complex_sequence]
-type = "PipeSequence"
-definition = "Complex sequence processing"
-output = "ProcessedData"
-steps = [
- { pipe = "step1", result = "result1", nb_output = 2 },
- { pipe = "step2", result = "result2", multiple_output = true },
- { pipe = "step3", result = "result3", batch_over = "items", batch_as = "item" },
-]""",
- ),
- ],
- )
- def test_sequence_pipe_to_plx_string(self, pipe_name: str, blueprint: PipeSequenceBlueprint, expected_plx: str):
- result = PipelexInterpreter.sequence_pipe_to_plx_string(pipe_name, blueprint)
- assert result == expected_plx
diff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_serialize_sub_pipe.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_serialize_sub_pipe.py
deleted file mode 100644
index 95d72fa7c..000000000
--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_serialize_sub_pipe.py
+++ /dev/null
@@ -1,153 +0,0 @@
-from typing import Any
-
-import pytest
-
-from pipelex.core.interpreter import PipelexInterpreter
-from pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprint
-
-
-class TestPipelexInterpreterSerializeSubPipe:
- @pytest.mark.parametrize(
- "sub_pipe_blueprint,expected_result",
- [
- # Basic sub pipe (step in sequence)
- (
- SubPipeBlueprint(
- pipe="step1",
- result="intermediate1",
- ),
- {
- "pipe": "step1",
- "result": "intermediate1",
- },
- ),
- # Sub pipe with nb_output
- (
- SubPipeBlueprint(
- pipe="generate_ideas",
- result="ideas",
- nb_output=3,
- ),
- {
- "pipe": "generate_ideas",
- "result": "ideas",
- "nb_output": 3,
- },
- ),
- # Sub pipe with multiple_output
- (
- SubPipeBlueprint(
- pipe="brainstorm",
- result="solutions",
- multiple_output=True,
- ),
- {
- "pipe": "brainstorm",
- "result": "solutions",
- "multiple_output": True,
- },
- ),
- # Sub pipe with batch_over and batch_as
- (
- SubPipeBlueprint(
- pipe="process_items",
- result="processed_items",
- batch_over="input_list",
- batch_as="current_item",
- ),
- {
- "pipe": "process_items",
- "result": "processed_items",
- "batch_over": "input_list",
- "batch_as": "current_item",
- },
- ),
- # Sub pipe with nb_output and batch fields (no multiple_output)
- (
- SubPipeBlueprint(
- pipe="complex_step",
- result="complex_result",
- nb_output=2,
- batch_over="items",
- batch_as="item",
- ),
- {
- "pipe": "complex_step",
- "result": "complex_result",
- "nb_output": 2,
- "batch_over": "items",
- "batch_as": "item",
- },
- ),
- # Sub pipe with batch_over=False (default, should not be included)
- (
- SubPipeBlueprint(
- pipe="simple_step",
- result="simple_result",
- batch_over=False, # This is the default and should not appear in output
- ),
- {
- "pipe": "simple_step",
- "result": "simple_result",
- # batch_over=False should not be included since it's the default
- },
- ),
- ],
- )
- def test_serialize_sub_pipe(self, sub_pipe_blueprint: SubPipeBlueprint, expected_result: dict[str, Any]):
- result = PipelexInterpreter.serialize_sub_pipe(sub_pipe_blueprint)
- assert result == expected_result
-
- @pytest.mark.parametrize(
- "sub_pipe_blueprint,expected_plx_string",
- [
- # Basic sub pipe (step in sequence)
- (
- SubPipeBlueprint(
- pipe="step1",
- result="intermediate1",
- ),
- '{ pipe = "step1", result = "intermediate1" }',
- ),
- # Sub pipe with nb_output
- (
- SubPipeBlueprint(
- pipe="generate_ideas",
- result="ideas",
- nb_output=3,
- ),
- '{ pipe = "generate_ideas", result = "ideas", nb_output = 3 }',
- ),
- # Sub pipe with multiple_output
- (
- SubPipeBlueprint(
- pipe="brainstorm",
- result="solutions",
- multiple_output=True,
- ),
- '{ pipe = "brainstorm", result = "solutions", multiple_output = true }',
- ),
- # Sub pipe with batch_over and batch_as
- (
- SubPipeBlueprint(
- pipe="process_items",
- result="processed_items",
- batch_over="input_list",
- batch_as="current_item",
- ),
- '{ pipe = "process_items", result = "processed_items", batch_over = "input_list", batch_as = "current_item" }',
- ),
- # Sub pipe with batch_over=False (default, should not be included)
- (
- SubPipeBlueprint(
- pipe="simple_step",
- result="simple_result",
- batch_over=False, # This is the default and should not appear in output
- ),
- '{ pipe = "simple_step", result = "simple_result" }',
- ),
- ],
- )
- def test_sub_pipe_to_plx_string(self, sub_pipe_blueprint: SubPipeBlueprint, expected_plx_string: str):
- result = PipelexInterpreter.sub_pipe_to_plx_string(sub_pipe_blueprint)
- assert result == expected_plx_string
diff --git a/tests/unit/pipelex/core/test_data/__init__.py b/tests/unit/pipelex/core/test_data/__init__.py
index 6c1d15983..1df3bcca0 100644
--- a/tests/unit/pipelex/core/test_data/__init__.py
+++ b/tests/unit/pipelex/core/test_data/__init__.py
@@ -1,4 +1,4 @@
-from typing import ClassVar, List, Tuple, Type, Union
+from typing import ClassVar, Type, Union
from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint
@@ -23,7 +23,7 @@ class InterpreterTestCases:
"""Test cases for PipelexInterpreter with PLX content and expected blueprints."""
# Aggregate all valid test cases from organized modules
- VALID_TEST_CASES: ClassVar[list[tuple[str, str, PipelexBundleBlueprint]]] = [
+ VALID_TEST_CASES: ClassVar[list[tuple[str, str, PipelexBundleBlueprint]]] = [ # test_name,plx_content,blueprint
# Domain tests
*DOMAIN_TEST_CASES,
# Concept tests
@@ -36,7 +36,7 @@ class InterpreterTestCases:
*PIPE_FUNC_TEST_CASES,
*PIPE_IMG_GEN_TEST_CASES,
*PIPE_JINJA2_TEST_CASES,
- # # Pipe controller tests
+ # Pipe controller tests
*PIPE_SEQUENCE_TEST_CASES,
*PIPE_CONDITION_TEST_CASES,
*PIPE_PARALLEL_TEST_CASES,
diff --git a/tests/unit/pipelex/core/test_data/pipes/controllers/condition/pipe_condition.py b/tests/unit/pipelex/core/test_data/pipes/controllers/condition/pipe_condition.py
index d83deab06..d25ae5095 100644
--- a/tests/unit/pipelex/core/test_data/pipes/controllers/condition/pipe_condition.py
+++ b/tests/unit/pipelex/core/test_data/pipes/controllers/condition/pipe_condition.py
@@ -10,8 +10,8 @@ definition = "Domain with conditional pipe"
type = "PipeCondition"
definition = "Process based on condition"
output = "ProcessedData"
-pipe_map = { small = "process_small", large = "process_large" }
expression = "input_data.category"
+pipe_map = { small = "process_small", large = "process_large" }
""",
PipelexBundleBlueprint(
domain="test_pipes",
diff --git a/tests/unit/pipelex/core/test_data/pipes/controllers/parallel/pipe_parallel.py b/tests/unit/pipelex/core/test_data/pipes/controllers/parallel/pipe_parallel.py
index 1056a0abc..f184e46de 100644
--- a/tests/unit/pipelex/core/test_data/pipes/controllers/parallel/pipe_parallel.py
+++ b/tests/unit/pipelex/core/test_data/pipes/controllers/parallel/pipe_parallel.py
@@ -9,12 +9,13 @@ definition = "Domain with parallel pipe"
[pipe.parallel_process]
type = "PipeParallel"
-definition = "Process data in parallel"
+definition = "PipeParallel example in PIPE_PARALLEL_TEST_CASES"
output = "ProcessedData"
parallels = [
{ pipe = "process_a", result = "result_a" },
{ pipe = "process_b", result = "result_b" },
]
+add_each_output = true
""",
PipelexBundleBlueprint(
domain="test_pipes",
@@ -22,12 +23,13 @@ parallels = [
pipe={
"parallel_process": PipeParallelBlueprint(
type="PipeParallel",
- definition="Process data in parallel",
+ definition="PipeParallel example in PIPE_PARALLEL_TEST_CASES",
output="ProcessedData",
parallels=[
SubPipeBlueprint(pipe="process_a", result="result_a"),
SubPipeBlueprint(pipe="process_b", result="result_b"),
],
+ add_each_output=True,
),
},
),
diff --git a/tests/unit/pipelex/core/test_data/pipes/operators/jinja2/pipe_jinja2.py b/tests/unit/pipelex/core/test_data/pipes/operators/jinja2/pipe_jinja2.py
index 4781fe0e8..b020bdcd3 100644
--- a/tests/unit/pipelex/core/test_data/pipes/operators/jinja2/pipe_jinja2.py
+++ b/tests/unit/pipelex/core/test_data/pipes/operators/jinja2/pipe_jinja2.py
@@ -1,6 +1,7 @@
from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint
from pipelex.core.concepts.concept_native import NativeConceptEnum
from pipelex.pipe_operators.jinja2.pipe_jinja2_blueprint import PipeJinja2Blueprint
+from pipelex.tools.templating.jinja2_template_category import Jinja2TemplateCategory
PIPE_JINJA2 = (
"pipe_jinja2",
@@ -12,6 +13,7 @@ type = "PipeJinja2"
definition = "Process a Jinja2 template"
output = "Text"
jinja2 = "Hello {{ name }}!"
+template_category = "markdown"
""",
PipelexBundleBlueprint(
domain="test_pipes",
@@ -22,6 +24,7 @@ jinja2 = "Hello {{ name }}!"
definition="Process a Jinja2 template",
output=NativeConceptEnum.TEXT,
jinja2="Hello {{ name }}!",
+ template_category=Jinja2TemplateCategory.MARKDOWN,
),
},
),
diff --git a/tests/unit/pipelex/core/test_data/pipes/operators/llm/pipe_llm.py b/tests/unit/pipelex/core/test_data/pipes/operators/llm/pipe_llm.py
index 112a341ac..6fcedd3d3 100644
--- a/tests/unit/pipelex/core/test_data/pipes/operators/llm/pipe_llm.py
+++ b/tests/unit/pipelex/core/test_data/pipes/operators/llm/pipe_llm.py
@@ -32,7 +32,7 @@ prompt_template = "Generate a story about a programmer"
# PipeLLM with inputs and template variables
PIPE_LLM_WITH_INPUTS = (
"pipe_llm_with_inputs",
- """domain = "test_pipes"
+ '''domain = "test_pipes"
definition = "Domain with pipe definitions"
[pipe.extract_info]
@@ -40,8 +40,12 @@ type = "PipeLLM"
definition = "Extract information from text"
inputs = { text = "Text", topic = "Text" }
output = "Text"
-prompt_template = "Extract information about $topic from this text:\\n\\n@text"
-""",
+prompt_template = """
+Extract information about $topic from this text:
+
+@text
+"""
+''',
PipelexBundleBlueprint(
domain="test_pipes",
definition="Domain with pipe definitions",
@@ -51,7 +55,7 @@ prompt_template = "Extract information about $topic from this text:\\n\\n@text"
definition="Extract information from text",
inputs={"text": "Text", "topic": "Text"},
output=NativeConceptEnum.TEXT,
- prompt_template="Extract information about $topic from this text:\n\n@text",
+ prompt_template="Extract information about $topic from this text:\n\n@text\n",
),
},
),
@@ -85,6 +89,34 @@ prompt_template = "Analyze the following data and provide insights"
),
)
+# PipeLLM with long prompt
+PIPE_LLM_WITH_LONG_PROMPT = (
+ "pipe_llm_with_long_prompt",
+ '''domain = "test_pipes"
+definition = "Domain with pipe definitions"
+
+[pipe.expert_analysis]
+type = "PipeLLM"
+definition = "Expert analysis with system prompt"
+output = "Text"
+prompt_template = """
+Extract all articles/items from this invoice text: $extracted_text. For each item find: item name, quantity, unit price, total price, description, and product code if available. Return each article as separate structured data.
+"""
+''',
+ PipelexBundleBlueprint(
+ domain="test_pipes",
+ definition="Domain with pipe definitions",
+ pipe={
+ "expert_analysis": PipeLLMBlueprint(
+ type="PipeLLM",
+ definition="Expert analysis with system prompt",
+ output=NativeConceptEnum.TEXT,
+ prompt_template="Extract all articles/items from this invoice text: $extracted_text. For each item find: item name, quantity, unit price, total price, description, and product code if available. Return each article as separate structured data.\n",
+ ),
+ },
+ ),
+)
+
# PipeLLM with multiple outputs (nb_output)
PIPE_LLM_MULTIPLE_OUTPUTS = (
"pipe_llm_multiple_outputs",
@@ -95,8 +127,8 @@ definition = "Domain with pipe definitions"
type = "PipeLLM"
definition = "Generate multiple ideas"
output = "Text"
-nb_output = 3
prompt_template = "Generate creative ideas for a mobile app"
+nb_output = 3
""",
PipelexBundleBlueprint(
domain="test_pipes",
@@ -116,7 +148,7 @@ prompt_template = "Generate creative ideas for a mobile app"
# PipeLLM with dynamic multiple outputs
PIPE_LLM_DYNAMIC_MULTIPLE = (
"pipe_llm_dynamic_multiple",
- """domain = "test_pipes"
+ '''domain = "test_pipes"
definition = "Domain with pipe definitions"
[pipe.brainstorm_solutions]
@@ -124,9 +156,9 @@ type = "PipeLLM"
definition = "Brainstorm multiple solutions"
inputs = { problem = { concept = "Text" } }
output = "Text"
+prompt_template = "Brainstorm solutions for this problem: $problem"
multiple_output = true
-prompt_template = "Brainstorm solutions for this problem: @problem"
-""",
+''',
PipelexBundleBlueprint(
domain="test_pipes",
definition="Domain with pipe definitions",
@@ -137,7 +169,7 @@ prompt_template = "Brainstorm solutions for this problem: @problem"
inputs={"problem": InputRequirementBlueprint(concept="Text")},
output=NativeConceptEnum.TEXT,
multiple_output=True,
- prompt_template="Brainstorm solutions for this problem: @problem",
+ prompt_template="Brainstorm solutions for this problem: $problem",
),
},
),
@@ -290,7 +322,7 @@ prompt_template = "Extract person information from this text: @text"
# PipeLLM with boolean multiplicity values
PIPE_LLM_BOOLEAN_MULTIPLICITY = (
"pipe_llm_boolean_multiplicity",
- """domain = "test_pipes"
+ '''domain = "test_pipes"
definition = "Domain with pipe definitions"
[concept]
@@ -301,8 +333,12 @@ type = "PipeLLM"
definition = "Analyze multiple documents and single query"
inputs = { documents = { concept = "Text", multiplicity = true }, query = { concept = "Text", multiplicity = false } }
output = "DocumentSummary"
-prompt_template = "Analyze these documents based on the query: $query\\n\\nDocuments: @documents"
-""",
+prompt_template = """
+Analyze these documents based on the query: $query
+
+Documents: @documents
+"""
+''',
PipelexBundleBlueprint(
domain="test_pipes",
definition="Domain with pipe definitions",
@@ -316,7 +352,10 @@ prompt_template = "Analyze these documents based on the query: $query\\n\\nDocum
"query": InputRequirementBlueprint(concept="Text", multiplicity=False),
},
output="DocumentSummary",
- prompt_template="Analyze these documents based on the query: $query\n\nDocuments: @documents",
+ prompt_template="""Analyze these documents based on the query: $query
+
+Documents: @documents
+""",
),
},
),
@@ -327,6 +366,7 @@ PIPE_LLM_TEST_CASES = [
PIPE_LLM_BASIC,
PIPE_LLM_WITH_INPUTS,
PIPE_LLM_WITH_SYSTEM_PROMPT,
+ PIPE_LLM_WITH_LONG_PROMPT,
PIPE_LLM_MULTIPLE_OUTPUTS,
PIPE_LLM_DYNAMIC_MULTIPLE,
PIPE_LLM_VISION,
diff --git a/tests/unit/pipelex/create/__init__.py b/tests/unit/pipelex/language/__init__.py
similarity index 100%
rename from tests/unit/pipelex/create/__init__.py
rename to tests/unit/pipelex/language/__init__.py
diff --git a/tests/unit/pipelex/language/test_plx_factory.py b/tests/unit/pipelex/language/test_plx_factory.py
new file mode 100644
index 000000000..4b71c2687
--- /dev/null
+++ b/tests/unit/pipelex/language/test_plx_factory.py
@@ -0,0 +1,454 @@
+from pathlib import Path
+from typing import Any, Dict, List, Mapping
+
+import pytest
+import tomlkit
+from tomlkit import array, inline_table, table
+from pytest_mock import MockerFixture
+
+from pipelex import log, pretty_print
+from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint
+from pipelex.language.plx_config import PlxConfig, PlxConfigStrings, PlxConfigInlineTables, PlxConfigForConcepts, PlxConfigForPipes
+from pipelex.language.plx_factory import PlxFactory, SectionKey
+
+from tests.unit.pipelex.core.test_data import InterpreterTestCases
+
+
+class TestPlxFactoryUnit:
+ """Unit tests for PlxFactory methods."""
+
+ @pytest.fixture
+ def mock_plx_config(self) -> PlxConfig:
+ """Create a mock PLX configuration for testing."""
+ return PlxConfig(
+ strings=PlxConfigStrings(
+ prefer_literal=True,
+ force_multiline=False,
+ length_limit_to_multiline=50,
+ ensure_trailing_newline=True,
+ ensure_leading_blank_line=False,
+ ),
+ inline_tables=PlxConfigInlineTables(
+ spaces_inside_curly_braces=True,
+ ),
+ concepts=PlxConfigForConcepts(
+ structure_field_ordering=["type", "definition", "inputs", "output"],
+ ),
+ pipes=PlxConfigForPipes(
+ field_ordering=["type", "definition", "inputs", "output"],
+ ),
+ )
+
+ @pytest.fixture
+ def sample_mapping_data(self) -> Dict[str, Any]:
+ """Sample mapping data for testing."""
+ return {
+ "simple_field": "simple_value",
+ "nested_mapping": {
+ "inner_key": "inner_value",
+ "inner_number": 42,
+ },
+ "list_field": ["item1", "item2", "item3"],
+ "complex_list": [
+ {"name": "first", "value": 1},
+ {"name": "second", "value": 2},
+ ],
+ }
+
+ def test_format_tomlkit_string_simple(self, mocker: MockerFixture, mock_plx_config: PlxConfig):
+ """Test formatting simple strings."""
+ _mock_config = mocker.patch.object(PlxFactory, '_plx_config', return_value=mock_plx_config)
+
+ # Test simple string
+ result = PlxFactory.format_tomlkit_string("simple text")
+ assert isinstance(result, tomlkit.items.String) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
+ # The actual string value without quotes
+ assert result.value == "simple text"
+
+ def test_format_tomlkit_string_multiline(self, mocker: MockerFixture, mock_plx_config: PlxConfig):
+ """Test formatting multiline strings."""
+ _mock_config = mocker.patch.object(PlxFactory, '_plx_config', return_value=mock_plx_config)
+
+ # Test string with newlines
+ multiline_text = "line1\nline2\nline3"
+ result = PlxFactory.format_tomlkit_string(multiline_text)
+ assert isinstance(result, tomlkit.items.String) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
+ # Should be multiline with trailing newline
+ assert result.value == "line1\nline2\nline3\n"
+ # Check if it's a multiline string by checking if it has newlines in the value
+ assert "\n" in result.value
+
+ def test_format_tomlkit_string_force_multiline(self, mocker: MockerFixture, mock_plx_config: PlxConfig):
+ """Test force multiline configuration."""
+ mock_plx_config.strings.force_multiline = True
+ _mock_config = mocker.patch.object(PlxFactory, '_plx_config', return_value=mock_plx_config)
+
+ result = PlxFactory.format_tomlkit_string("short")
+ assert isinstance(result, tomlkit.items.String) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
+ # Should be multiline even for short text
+ assert result.value == "short\n"
+ # Check if it's a multiline string by checking if it has newlines in the value
+ assert "\n" in result.value
+
+ def test_format_tomlkit_string_length_limit(self, mocker: MockerFixture, mock_plx_config: PlxConfig):
+ """Test length limit for multiline conversion."""
+ mock_plx_config.strings.length_limit_to_multiline = 10
+ _mock_config = mocker.patch.object(PlxFactory, '_plx_config', return_value=mock_plx_config)
+
+ long_text = "this is a very long text that exceeds the limit"
+ result = PlxFactory.format_tomlkit_string(long_text)
+ assert isinstance(result, tomlkit.items.String) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
+ # Should be multiline due to length
+ assert result.value == "this is a very long text that exceeds the limit\n"
+ # Check if it's a multiline string by checking if it has newlines in the value
+ assert "\n" in result.value
+
+ def test_format_tomlkit_string_leading_blank_line(self, mocker: MockerFixture, mock_plx_config: PlxConfig):
+ """Test leading blank line configuration."""
+ mock_plx_config.strings.ensure_leading_blank_line = True
+ mock_plx_config.strings.force_multiline = True
+ _mock_config = mocker.patch.object(PlxFactory, '_plx_config', return_value=mock_plx_config)
+
+ result = PlxFactory.format_tomlkit_string("content")
+ assert isinstance(result, tomlkit.items.String) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
+ # Should have leading blank line
+ assert result.value == "\ncontent\n"
+ # Check if it's a multiline string by checking if it has newlines in the value
+ assert "\n" in result.value
+
+ def test_convert_dicts_to_inline_tables_simple_dict(self, mocker: MockerFixture, mock_plx_config: PlxConfig):
+ """Test converting simple dictionary to inline table."""
+ _mock_config = mocker.patch.object(PlxFactory, '_plx_config', return_value=mock_plx_config)
+
+ input_dict = {"key1": "value1", "key2": "value2"}
+ result = PlxFactory.convert_dicts_to_inline_tables(input_dict)
+
+ assert isinstance(result, tomlkit.items.InlineTable) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
+ assert result["key1"].value == "value1"
+ assert result["key2"].value == "value2"
+
+ def test_convert_dicts_to_inline_tables_with_field_ordering(self, mocker: MockerFixture, mock_plx_config: PlxConfig):
+ """Test converting dictionary with field ordering."""
+ _mock_config = mocker.patch.object(PlxFactory, '_plx_config', return_value=mock_plx_config)
+
+ input_dict = {"key2": "value2", "key1": "value1", "key3": "value3"}
+ field_ordering = ["key1", "key3"]
+ result = PlxFactory.convert_dicts_to_inline_tables(input_dict, field_ordering)
+
+ assert isinstance(result, tomlkit.items.InlineTable) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
+ # Check that ordered fields come first
+ keys = list(result.keys())
+ assert keys[0] == "key1"
+ assert keys[1] == "key3"
+ # Note: the implementation might not include all keys if not in ordering
+ if len(keys) > 2:
+ assert keys[2] == "key2"
+
+ def test_convert_dicts_to_inline_tables_nested_dict(self, mocker: MockerFixture, mock_plx_config: PlxConfig):
+ """Test converting nested dictionary."""
+ _mock_config = mocker.patch.object(PlxFactory, '_plx_config', return_value=mock_plx_config)
+
+ input_dict = {
+ "outer": {
+ "inner": "value"
+ }
+ }
+ result = PlxFactory.convert_dicts_to_inline_tables(input_dict)
+
+ assert isinstance(result, tomlkit.items.InlineTable) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
+ assert isinstance(result["outer"], tomlkit.items.InlineTable) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
+ assert result["outer"]["inner"].value == "value"
+
+ def test_convert_dicts_to_inline_tables_list_with_dicts(self, mocker: MockerFixture, mock_plx_config: PlxConfig):
+ """Test converting list containing dictionaries."""
+ _mock_config = mocker.patch.object(PlxFactory, '_plx_config', return_value=mock_plx_config)
+
+ input_list = [
+ {"name": "first", "value": 1},
+ {"name": "second", "value": 2}
+ ]
+ result = PlxFactory.convert_dicts_to_inline_tables(input_list)
+
+ assert isinstance(result, tomlkit.items.Array) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
+ assert len(result) == 2
+ assert isinstance(result[0], tomlkit.items.InlineTable) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
+ assert result[0]["name"].value == "first"
+ assert result[0]["value"] == 1
+
+ def test_convert_dicts_to_inline_tables_string_handling(self, mocker: MockerFixture, mock_plx_config: PlxConfig):
+ """Test string handling in conversion."""
+ _mock_config = mocker.patch.object(PlxFactory, '_plx_config', return_value=mock_plx_config)
+
+ # Test simple string
+ result = PlxFactory.convert_dicts_to_inline_tables("simple string")
+ assert isinstance(result, tomlkit.items.String) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
+
+ # Test other types pass through
+ assert PlxFactory.convert_dicts_to_inline_tables(42) == 42
+ assert PlxFactory.convert_dicts_to_inline_tables(True) is True
+
+ def test_convert_mapping_to_table(self, mocker: MockerFixture, mock_plx_config: PlxConfig, sample_mapping_data: Dict[str, Any]):
+ """Test converting mapping to table."""
+ _mock_config = mocker.patch.object(PlxFactory, '_plx_config', return_value=mock_plx_config)
+
+ result = PlxFactory.convert_mapping_to_table(sample_mapping_data)
+
+ assert isinstance(result, tomlkit.items.Table) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
+ assert "simple_field" in result
+ assert "nested_mapping" in result
+ assert "list_field" in result
+ assert "complex_list" in result
+
+ def test_convert_mapping_to_table_with_field_ordering(self, mocker: MockerFixture, mock_plx_config: PlxConfig):
+ """Test converting mapping with field ordering."""
+ _mock_config = mocker.patch.object(PlxFactory, '_plx_config', return_value=mock_plx_config)
+
+ mapping = {"field3": "value3", "field1": "value1", "field2": "value2"}
+ field_ordering = ["field1", "field2"]
+
+ result = PlxFactory.convert_mapping_to_table(mapping, field_ordering)
+
+ assert isinstance(result, tomlkit.items.Table) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
+ # Check ordering (note: tomlkit preserves insertion order)
+ keys = list(result.keys())
+ assert keys[0] == "field1"
+ assert keys[1] == "field2"
+ assert keys[2] == "field3"
+
+ def test_convert_mapping_to_table_skips_category(self, mocker: MockerFixture, mock_plx_config: PlxConfig):
+ """Test that category field is skipped."""
+ _mock_config = mocker.patch.object(PlxFactory, '_plx_config', return_value=mock_plx_config)
+
+ mapping = {"field1": "value1", "category": "should_be_skipped", "field2": "value2"}
+ result = PlxFactory.convert_mapping_to_table(mapping)
+
+ assert isinstance(result, tomlkit.items.Table) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
+ assert "field1" in result
+ assert "field2" in result
+ assert "category" not in result
+
+ def test_add_spaces_to_inline_tables_simple(self):
+ """Test adding spaces to simple inline tables."""
+ input_toml = "{key = value}"
+ result = PlxFactory.add_spaces_to_inline_tables(input_toml)
+ assert result == "{ key = value }"
+
+ def test_add_spaces_to_inline_tables_already_spaced(self):
+ """Test that already spaced tables are preserved."""
+ input_toml = "{ key = value }"
+ result = PlxFactory.add_spaces_to_inline_tables(input_toml)
+ assert result == "{ key = value }"
+
+ def test_add_spaces_to_inline_tables_nested(self):
+ """Test adding spaces to nested inline tables."""
+ input_toml = "{outer = {inner = value}}"
+ result = PlxFactory.add_spaces_to_inline_tables(input_toml)
+ assert result == "{ outer = { inner = value } }"
+
+ def test_add_spaces_to_inline_tables_with_jinja2(self):
+ """Test that Jinja2 templates are preserved."""
+ input_toml = "template = '{{ variable }}' and {key = value}"
+ result = PlxFactory.add_spaces_to_inline_tables(input_toml)
+ assert result == "template = '{{ variable }}' and { key = value }"
+
+ def test_add_spaces_to_inline_tables_complex(self):
+ """Test complex inline table spacing."""
+ input_toml = "config = {db = {host = 'localhost', port = 5432}, cache = {enabled = true}}"
+ result = PlxFactory.add_spaces_to_inline_tables(input_toml)
+ expected = "config = { db = { host = 'localhost', port = 5432 }, cache = { enabled = true } }"
+ assert result == expected
+
+ def test_add_spaces_to_inline_tables_partial_spacing(self):
+ """Test partial spacing scenarios."""
+ # Left space only
+ input_toml = "{ key = value}"
+ result = PlxFactory.add_spaces_to_inline_tables(input_toml)
+ assert result == "{ key = value }"
+
+ # Right space only
+ input_toml = "{key = value }"
+ result = PlxFactory.add_spaces_to_inline_tables(input_toml)
+ assert result == "{ key = value }"
+
+ def test_make_table_obj_for_pipe(self, mocker: MockerFixture, mock_plx_config: PlxConfig):
+ """Test making table object for pipe section."""
+ _mock_config = mocker.patch.object(PlxFactory, '_plx_config', return_value=mock_plx_config)
+
+ pipe_data = {
+ "type": "PipeLLM",
+ "definition": "Test pipe",
+ "inputs": {"input1": "Text"},
+ "output": "Text",
+ "nested_config": {
+ "param1": "value1",
+ "param2": 42
+ }
+ }
+
+ result = PlxFactory.make_table_obj_for_pipe(pipe_data)
+
+ assert isinstance(result, tomlkit.items.Table) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
+ assert "type" in result
+ assert "definition" in result
+ assert "inputs" in result
+ assert "output" in result
+ assert "nested_config" in result
+
+ def test_make_table_obj_for_concept_simple_string(self, mocker: MockerFixture, mock_plx_config: PlxConfig):
+ """Test making table object for concept with simple string definition."""
+ _mock_config = mocker.patch.object(PlxFactory, '_plx_config', return_value=mock_plx_config)
+
+ concept_data = {
+ "SimpleConcept": "A simple concept definition"
+ }
+
+ result = PlxFactory.make_table_obj_for_concept(concept_data)
+
+ assert isinstance(result, tomlkit.items.Table) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
+ assert "SimpleConcept" in result
+ assert result["SimpleConcept"] == "A simple concept definition"
+
+ def test_make_table_obj_for_concept_with_structure(self, mocker: MockerFixture, mock_plx_config: PlxConfig):
+ """Test making table object for concept with structure."""
+ _mock_config = mocker.patch.object(PlxFactory, '_plx_config', return_value=mock_plx_config)
+
+ concept_data = {
+ "ComplexConcept": {
+ "definition": "A complex concept",
+ "structure": {
+ "field1": "string",
+ "field2": "int"
+ }
+ }
+ }
+
+ result = PlxFactory.make_table_obj_for_concept(concept_data)
+
+ assert isinstance(result, tomlkit.items.Table) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
+ assert "ComplexConcept" in result
+ assert isinstance(result["ComplexConcept"], tomlkit.items.Table) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
+ assert "definition" in result["ComplexConcept"]
+ assert "structure" in result["ComplexConcept"]
+
+ def test_make_table_obj_for_concept_structure_string(self, mocker: MockerFixture, mock_plx_config: PlxConfig):
+ """Test concept with structure as string."""
+ _mock_config = mocker.patch.object(PlxFactory, '_plx_config', return_value=mock_plx_config)
+
+ concept_data = {
+ "ConceptWithStringStructure": {
+ "structure": "SomeClass"
+ }
+ }
+
+ result = PlxFactory.make_table_obj_for_concept(concept_data)
+
+ assert isinstance(result, tomlkit.items.Table) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
+ concept_table = result["ConceptWithStringStructure"]
+ assert isinstance(concept_table, tomlkit.items.Table) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
+ assert concept_table["structure"] == "SomeClass"
+
+ def test_make_table_obj_for_concept_invalid_structure(self, mocker: MockerFixture, mock_plx_config: PlxConfig):
+ """Test error handling for invalid structure types."""
+ _mock_config = mocker.patch.object(PlxFactory, '_plx_config', return_value=mock_plx_config)
+
+ concept_data = {
+ "InvalidConcept": {
+ "structure": 123 # Invalid type
+ }
+ }
+
+ with pytest.raises(TypeError, match="Structure field value is not a mapping"):
+ PlxFactory.make_table_obj_for_concept(concept_data)
+
+ def test_make_table_obj_for_concept_invalid_concept_value(self, mocker: MockerFixture, mock_plx_config: PlxConfig):
+ """Test error handling for invalid concept value types."""
+ _mock_config = mocker.patch.object(PlxFactory, '_plx_config', return_value=mock_plx_config)
+
+ concept_data = {
+ "InvalidConcept": 123 # Invalid type
+ }
+
+ with pytest.raises(TypeError, match="Concept field value is not a mapping"):
+ PlxFactory.make_table_obj_for_concept(concept_data)
+
+ def test_dict_to_plx_styled_toml_with_spacing(self, mocker: MockerFixture, mock_plx_config: PlxConfig):
+ """Test dict to PLX styled TOML with spacing enabled."""
+ mock_plx_config.inline_tables.spaces_inside_curly_braces = True
+ _mock_config = mocker.patch.object(PlxFactory, '_plx_config', return_value=mock_plx_config)
+ mock_add_spaces = mocker.patch.object(PlxFactory, 'add_spaces_to_inline_tables', return_value="spaced_output")
+
+ data = {"domain": "test", "definition": "test domain"}
+
+ result = PlxFactory.dict_to_plx_styled_toml(data)
+
+ assert result == "spaced_output"
+ mock_add_spaces.assert_called_once()
+
+ def test_dict_to_plx_styled_toml_without_spacing(self, mocker: MockerFixture, mock_plx_config: PlxConfig):
+ """Test dict to PLX styled TOML without spacing."""
+ mock_plx_config.inline_tables.spaces_inside_curly_braces = False
+ _mock_config = mocker.patch.object(PlxFactory, '_plx_config', return_value=mock_plx_config)
+ mock_add_spaces = mocker.patch.object(PlxFactory, 'add_spaces_to_inline_tables')
+
+ data = {"domain": "test", "definition": "test domain"}
+
+ result = PlxFactory.dict_to_plx_styled_toml(data)
+
+ # Should not call add_spaces_to_inline_tables
+ mock_add_spaces.assert_not_called()
+ assert isinstance(result, str)
+
+ def test_dict_to_plx_styled_toml_empty_sections(self, mocker: MockerFixture, mock_plx_config: PlxConfig):
+ """Test handling of empty sections."""
+ _mock_config = mocker.patch.object(PlxFactory, '_plx_config', return_value=mock_plx_config)
+
+ data: dict[str, Any] = {
+ "domain": "test",
+ "concept": {}, # Empty concept section
+ "pipe": {} # Empty pipe section
+ }
+
+ result = PlxFactory.dict_to_plx_styled_toml(data)
+
+ # Empty sections should be skipped
+ assert "concept" not in result
+ assert "pipe" not in result
+ assert "domain" in result
+
+ def test_dict_to_plx_styled_toml_with_pipe_section(self, mocker: MockerFixture, mock_plx_config: PlxConfig):
+ """Test dict to PLX styled TOML with pipe section."""
+ _mock_config = mocker.patch.object(PlxFactory, '_plx_config', return_value=mock_plx_config)
+
+ data = {
+ "domain": "test",
+ "pipe": {
+ "test_pipe": {
+ "type": "PipeLLM",
+ "definition": "Test pipe"
+ }
+ }
+ }
+
+ result = PlxFactory.dict_to_plx_styled_toml(data)
+
+ assert "domain" in result
+ assert "[pipe.test_pipe]" in result
+ assert "type" in result
+ assert "definition" in result
+
+ def test_dict_to_plx_styled_toml_with_concept_section(self, mocker: MockerFixture, mock_plx_config: PlxConfig):
+ """Test dict to PLX styled TOML with concept section."""
+ _mock_config = mocker.patch.object(PlxFactory, '_plx_config', return_value=mock_plx_config)
+
+ data = {
+ "domain": "test",
+ "concept": {
+ "TestConcept": "A test concept"
+ }
+ }
+
+ result = PlxFactory.dict_to_plx_styled_toml(data)
+
+ assert "domain" in result
+ assert "[concept]" in result
+ assert "TestConcept" in result
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_data.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_data.py
index 25023d6eb..63fbca416 100644
--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_data.py
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_data.py
@@ -9,8 +9,8 @@ from pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprint
class PipeParallelTestCases:
- SIMPLE_PARALLEL = (
- "simple_parallel",
+ PARALLEL_WITH_EACH_OUTPUT = (
+ "parallel_with_each_output",
PipeParallelSpec(
the_pipe_code="parallel_processor",
definition="Run pipes in parallel",
@@ -21,6 +21,7 @@ class PipeParallelTestCases:
SubPipeSpec(the_pipe_code="transform_data", result="transformed"),
SubPipeSpec(the_pipe_code="validate_data", result="validation"),
],
+ add_each_output=True,
),
PipeParallelBlueprint(
definition="Run pipes in parallel",
@@ -35,6 +36,7 @@ class PipeParallelTestCases:
],
add_each_output=True,
combined_output=None,
+ source="tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_data.py PipeParallelTestCases.PARALLEL_WITH_EACH_OUTPUT",
),
)
@@ -64,10 +66,42 @@ class PipeParallelTestCases:
],
add_each_output=False,
combined_output="MergedData",
+ source="tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_data.py PipeParallelTestCases.PARALLEL_WITH_COMBINED",
+ ),
+ )
+
+ PARALLEL_WITH_BOTH_EACH_OUTPUT_AND_COMBINED = (
+ "parallel_with_both_each_output_and_combined",
+ PipeParallelSpec(
+ the_pipe_code="combined_parallel",
+ definition="Parallel with combined output",
+ inputs={"input": InputRequirementSpec(concept="Input")},
+ output="CombinedResult",
+ parallels=[
+ SubPipeSpec(the_pipe_code="pipe1", result="result1"),
+ SubPipeSpec(the_pipe_code="pipe2", result="result2"),
+ ],
+ add_each_output=True,
+ combined_output="MergedData",
+ ),
+ PipeParallelBlueprint(
+ definition="Parallel with combined output",
+ inputs={"input": InputRequirementBlueprint(concept="Input")},
+ output="CombinedResult",
+ type="PipeParallel",
+ category="PipeController",
+ parallels=[
+ SubPipeBlueprint(pipe="pipe1", result="result1"),
+ SubPipeBlueprint(pipe="pipe2", result="result2"),
+ ],
+ add_each_output=True,
+ combined_output="MergedData",
+ source="tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_data.py PipeParallelTestCases.PARALLEL_WITH_BOTH_EACH_OUTPUT_AND_COMBINED",
),
)
TEST_CASES: ClassVar[list[tuple[str, PipeParallelSpec, PipeParallelBlueprint]]] = [
- SIMPLE_PARALLEL,
+ PARALLEL_WITH_EACH_OUTPUT,
PARALLEL_WITH_COMBINED,
+ PARALLEL_WITH_BOTH_EACH_OUTPUT_AND_COMBINED,
]
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_pipe_parallel.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_pipe_parallel.py
index 04d999a7f..e35dd1669 100644
--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_pipe_parallel.py
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_pipe_parallel.py
@@ -4,7 +4,7 @@ from pipelex.libraries.pipelines.builder.pipe.pipe_parallel_spec import PipePara
from pipelex.pipe_controllers.parallel.pipe_parallel_blueprint import PipeParallelBlueprint
from .test_data import PipeParallelTestCases
-
+from pipelex import log
class TestPipeParallelBlueprintConversion:
@pytest.mark.parametrize(
@@ -17,5 +17,7 @@ class TestPipeParallelBlueprintConversion:
pipe_spec: PipeParallelSpec,
expected_blueprint: PipeParallelBlueprint,
):
+ log.verbose(f"Testing {test_name}")
result = pipe_spec.to_blueprint()
+ expected_blueprint.source = None
assert result == expected_blueprint
diff --git a/tests/unit/pipelex/tools/misc/test_toml_utils.py b/tests/unit/pipelex/tools/misc/test_toml_utils.py
index a19a20c18..9980dea8e 100644
--- a/tests/unit/pipelex/tools/misc/test_toml_utils.py
+++ b/tests/unit/pipelex/tools/misc/test_toml_utils.py
@@ -4,7 +4,7 @@ import pytest
from pipelex.tools.misc.toml_utils import (
TOMLValidationError,
- failable_load_toml_from_path,
+ load_toml_from_path_if_exists,
load_toml_from_path,
validate_toml_file,
)
@@ -104,12 +104,12 @@ definition = "Test with trailing space"
assert isinstance(result, dict)
assert result["domain"] == "test"
- def test_failable_load_toml_from_path_nonexistent_file(self) -> None:
+ def test_load_toml_from_path_if_exists_nonexistent_file(self) -> None:
"""Test failable loading with non-existent file."""
- result = failable_load_toml_from_path("/nonexistent/path.toml")
+ result = load_toml_from_path_if_exists("/nonexistent/path.toml")
assert result is None
- def test_failable_load_toml_from_path_valid_file(self, tmp_path: Path) -> None:
+ def test_load_toml_from_path_if_exists_valid_file(self, tmp_path: Path) -> None:
"""Test failable loading with valid file."""
toml_content = """domain = "test"
definition = "Test definition"
@@ -117,12 +117,12 @@ definition = "Test definition"
toml_file = tmp_path / "valid.toml"
toml_file.write_text(toml_content)
- result = failable_load_toml_from_path(str(toml_file))
+ result = load_toml_from_path_if_exists(str(toml_file))
assert result is not None
assert result["domain"] == "test"
- def test_failable_load_toml_from_path_with_whitespace_works(self, tmp_path: Path) -> None:
+ def test_load_toml_from_path_if_exists_with_whitespace_works(self, tmp_path: Path) -> None:
"""Test failable loading works normally with whitespace."""
toml_content = """domain = "test"
"""
@@ -130,7 +130,7 @@ definition = "Test definition"
toml_file.write_text(toml_content)
# Should work fine - loading doesn't validate by default
- result = failable_load_toml_from_path(str(toml_file))
+ result = load_toml_from_path_if_exists(str(toml_file))
assert result is not None
assert result["domain"] == "test"