Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

Use proper tomlkit to serialize blueprints WIP : test cases need rewrite

Louis Choquel12f2ed897939
diff --git a/pipelex/cli/commands/build_cmd.py b/pipelex/cli/commands/build_cmd.py
index cdd4ea3bf..1aef5848d 100644
--- a/pipelex/cli/commands/build_cmd.py
+++ b/pipelex/cli/commands/build_cmd.py
@@ -5,11 +5,11 @@ from typing import Annotated, Optional
import typer
from pipelex import pretty_print
-from pipelex.core.interpreter import PipelexInterpreter
from pipelex.hub import get_report_delegate
from pipelex.libraries.pipelines.builder.builder import PipelexBundleSpec
from pipelex.pipelex import Pipelex
from pipelex.pipeline.execute import execute_pipeline
+from pipelex.tools.plx.plx_utils import make_plx_content
build_app = typer.Typer(help="Build artifacts like pipelines", no_args_is_help=True)
@@ -46,7 +46,7 @@ def build_pipe_cmd(
)
pretty_print(pipe_output, title="Pipe Output")
pipelex_bundle_spec = pipe_output.working_memory.get_stuff_as(name="pipelex_bundle_spec", content_type=PipelexBundleSpec)
- plx_content = PipelexInterpreter.make_plx_content(blueprint=pipelex_bundle_spec.to_blueprint())
+ plx_content = make_plx_content(blueprint=pipelex_bundle_spec.to_blueprint())
# Save to file unless explicitly disabled with empty string
if output_path and output_path != "":
diff --git a/pipelex/core/interpreter.py b/pipelex/core/interpreter.py
index 9413cd7a3..03c7b226e 100644
--- a/pipelex/core/interpreter.py
+++ b/pipelex/core/interpreter.py
@@ -140,878 +140,3 @@ class PipelexInterpreter(BaseModel):
file_content = self._read_and_reformat()
plx_data = self._parse_plx_content(file_content)
return PipelexBundleBlueprint.model_validate(plx_data)
-
- @staticmethod
- def make_plx_content(blueprint: PipelexBundleBlueprint) -> str:
- """Convert a PipelexBundleBlueprint to properly formatted PLX content."""
- plx_parts: list[str] = []
-
- # Domain-level fields
- domain_fields: list[str] = []
- domain_fields.append(f'domain = "{PipelexInterpreter.escape_plx_string(blueprint.domain)}"')
- if blueprint.definition:
- domain_fields.append(f'definition = "{PipelexInterpreter.escape_plx_string(blueprint.definition)}"')
- if blueprint.system_prompt:
- domain_fields.append(f'system_prompt = "{PipelexInterpreter.escape_plx_string(blueprint.system_prompt)}"')
- if blueprint.system_prompt_to_structure:
- domain_fields.append(f'system_prompt_to_structure = "{PipelexInterpreter.escape_plx_string(blueprint.system_prompt_to_structure)}"')
- if blueprint.prompt_template_to_structure:
- domain_fields.append(f'prompt_template_to_structure = "{PipelexInterpreter.escape_plx_string(blueprint.prompt_template_to_structure)}"')
-
- plx_parts.append("\n".join(domain_fields))
-
- # Concepts section
- if blueprint.concept:
- concept_plx = PipelexInterpreter.concepts_to_plx_string(blueprint.concept, blueprint.domain)
- if concept_plx: # Only add if not empty
- plx_parts.append(concept_plx)
-
- # Pipes section
- if blueprint.pipe:
- pipes_plx = PipelexInterpreter.pipes_to_plx_string(blueprint.pipe, blueprint.domain)
- if pipes_plx: # Only add if not empty
- plx_parts.append(pipes_plx)
-
- result = "\n\n".join(plx_parts)
- if result and not result.endswith("\n"):
- result += "\n"
- return result
-
- @staticmethod
- def concepts_to_plx_string(concepts: Dict[str, ConceptBlueprint | str], domain: str) -> str:
- """Convert concepts dict to PLX string."""
- if not concepts:
- return ""
-
- plx_parts: list[str] = []
- simple_concepts: list[str] = []
- complex_concepts: list[str] = []
-
- for concept_name, concept_blueprint in concepts.items():
- if isinstance(concept_blueprint, str):
- simple_concepts.append(f'{concept_name} = "{PipelexInterpreter.escape_plx_string(concept_blueprint)}"')
- else:
- # Handle ConceptBlueprint objects
- if concept_blueprint.structure is None and concept_blueprint.refines is None:
- # Simple concept with just definition
- simple_concepts.append(f'{concept_name} = "{PipelexInterpreter.escape_plx_string(concept_blueprint.definition)}"')
- else:
- # Complex concept needs its own section
- complex_concept_plx = PipelexInterpreter.complex_concept_to_plx_string(concept_name, concept_blueprint)
- complex_concepts.append(complex_concept_plx)
-
- # Add simple concepts section if we have any
- if simple_concepts:
- plx_parts.append("[concept]")
- plx_parts.extend(simple_concepts)
-
- # Add complex concepts with proper spacing
- if complex_concepts:
- # If we had simple concepts, add an empty line before complex ones
- if simple_concepts:
- plx_parts.append("")
- # Add each complex concept with empty lines between them
- for i, complex_concept in enumerate(complex_concepts):
- if i > 0: # Add empty line between complex concepts
- plx_parts.append("")
- plx_parts.append(complex_concept)
-
- return "\n".join(plx_parts)
-
- @staticmethod
- def complex_concept_to_plx_string(concept_name: str, concept: ConceptBlueprint) -> str:
- """Convert a complex ConceptBlueprint to PLX string."""
- lines: list[str] = [f"[concept.{concept_name}]"]
-
- if concept.definition:
- lines.append(f'definition = "{PipelexInterpreter.escape_plx_string(concept.definition)}"')
-
- if concept.refines:
- lines.append(f'refines = "{PipelexInterpreter.escape_plx_string(concept.refines)}"')
-
- if concept.structure:
- if isinstance(concept.structure, str):
- lines.append(f'structure = "{PipelexInterpreter.escape_plx_string(concept.structure)}"')
- else:
- # Complex structure with fields
- lines.append("")
- lines.append(f"[concept.{concept_name}.structure]")
- for field_name, field_def in concept.structure.items():
- field_plx = PipelexInterpreter.structure_field_to_plx_string(field_name, field_def)
- lines.append(field_plx)
-
- return "\n".join(lines)
-
- @staticmethod
- def structure_field_to_plx_string(field_name: str, field_def: Any) -> str:
- """Convert a structure field to PLX string."""
- if isinstance(field_def, str):
- return f'{field_name} = "{PipelexInterpreter.escape_plx_string(field_def)}"'
- else:
- # Complex field with type, definition, required, etc.
- field_parts: list[str] = []
-
- if hasattr(field_def, "type") and field_def.type:
- type_value = field_def.type.value if hasattr(field_def.type, "value") else field_def.type
- field_parts.append(f'type = "{PipelexInterpreter.escape_plx_string(str(type_value))}"')
-
- if hasattr(field_def, "definition") and field_def.definition:
- field_parts.append(f'definition = "{PipelexInterpreter.escape_plx_string(field_def.definition)}"')
-
- if hasattr(field_def, "required") and field_def.required is not None:
- field_parts.append(f"required = {str(field_def.required).lower()}")
-
- return f"{field_name} = {{ {', '.join(field_parts)} }}"
-
- @staticmethod
- def pipes_to_plx_string(pipes: Dict[str, Any], domain: str) -> str:
- """Convert pipes dict to PLX string."""
- plx_parts: list[str] = []
- for pipe_name, blueprint in pipes.items():
- pipe_plx = PipelexInterpreter.pipe_to_plx_string(pipe_name, blueprint, domain)
- plx_parts.append(pipe_plx)
- return "\n\n".join(plx_parts)
-
- @staticmethod
- def pipe_to_plx_string(pipe_name: str, blueprint: Any, domain: str) -> str:
- """Convert a single pipe blueprint to PLX string."""
- if isinstance(blueprint, PipeLLMBlueprint):
- return PipelexInterpreter.llm_pipe_to_plx_string(pipe_name, blueprint, domain)
- elif isinstance(blueprint, PipeSequenceBlueprint):
- return PipelexInterpreter.sequence_pipe_to_plx_string(pipe_name, blueprint, domain)
- elif isinstance(blueprint, PipeOcrBlueprint):
- return PipelexInterpreter.ocr_pipe_to_plx_string(pipe_name, blueprint, domain)
- elif isinstance(blueprint, PipeFuncBlueprint):
- return PipelexInterpreter.func_pipe_to_plx_string(pipe_name, blueprint, domain)
- elif isinstance(blueprint, PipeImgGenBlueprint):
- return PipelexInterpreter.img_gen_pipe_to_plx_string(pipe_name, blueprint, domain)
- elif isinstance(blueprint, PipeJinja2Blueprint):
- return PipelexInterpreter.jinja2_pipe_to_plx_string(pipe_name, blueprint, domain)
- elif isinstance(blueprint, PipeConditionBlueprint):
- return PipelexInterpreter.condition_pipe_to_plx_string(pipe_name, blueprint, domain)
- elif isinstance(blueprint, PipeParallelBlueprint):
- return PipelexInterpreter.parallel_pipe_to_plx_string(pipe_name, blueprint, domain)
- elif isinstance(blueprint, PipeBatchBlueprint):
- return PipelexInterpreter.batch_pipe_to_plx_string(pipe_name, blueprint, domain)
- else:
- # Fallback to old dict approach for unknown pipe types
- pipe_dict = PipelexInterpreter.serialize_pipe(blueprint, domain)
- return f"[pipe.{pipe_name}]\n" + "\n".join([f'{k} = "{v}"' if isinstance(v, str) else f"{k} = {v}" for k, v in pipe_dict.items()])
-
- @staticmethod
- def llm_pipe_to_plx_string(pipe_name: str, pipe: PipeLLMBlueprint, domain: str) -> str:
- """Convert a PipeLLM blueprint directly to PLX section string."""
- lines: list[str] = [
- f"[pipe.{pipe_name}]",
- f'type = "{PipelexInterpreter.escape_plx_string(pipe.type)}"',
- f'definition = "{PipelexInterpreter.escape_plx_string(pipe.definition)}"',
- ]
-
- # Add inputs first if they exist
- PipelexInterpreter.add_inputs_to_lines_if_exist(lines, pipe.inputs)
-
- # Add output after inputs (or immediately if no inputs)
- lines.append(f'output = "{PipelexInterpreter.escape_plx_string(pipe.output)}"')
-
- # Add optional fields
- if pipe.nb_output is not None:
- lines.append(f"nb_output = {pipe.nb_output}")
- if pipe.multiple_output is not None:
- lines.append(f"multiple_output = {str(pipe.multiple_output).lower()}")
- if pipe.system_prompt_template:
- lines.append(f'system_prompt_template = "{PipelexInterpreter.escape_plx_string(pipe.system_prompt_template)}"')
- if pipe.system_prompt:
- lines.append(f'system_prompt = "{PipelexInterpreter.escape_plx_string(pipe.system_prompt)}"')
- if pipe.prompt_template:
- lines.append(f'prompt_template = "{PipelexInterpreter.escape_plx_string(pipe.prompt_template)}"')
- if pipe.template_name:
- lines.append(f'template_name = "{PipelexInterpreter.escape_plx_string(pipe.template_name)}"')
- if pipe.prompt:
- lines.append(f'prompt = "{PipelexInterpreter.escape_plx_string(pipe.prompt)}"')
-
- return "\n".join(lines)
-
- @staticmethod
- def ocr_pipe_to_plx_string(pipe_name: str, pipe: PipeOcrBlueprint, domain: str) -> str:
- """Convert a PipeOcr blueprint directly to PLX section string."""
- lines: list[str] = [
- f"[pipe.{pipe_name}]",
- f'type = "{pipe.type}"',
- f'definition = "{pipe.definition}"',
- ]
-
- # Add inputs if they exist
- PipelexInterpreter.add_inputs_to_lines_if_exist(lines, pipe.inputs)
-
- lines.append(f'output = "{PipelexInterpreter.escape_plx_string(pipe.output)}"')
- lines.append(f'ocr = "{pipe.ocr}"')
-
- return "\n".join(lines)
-
- @staticmethod
- def func_pipe_to_plx_string(pipe_name: str, pipe: PipeFuncBlueprint, domain: str) -> str:
- """Convert a PipeFunc blueprint directly to PLX section string."""
- lines: list[str] = [
- f"[pipe.{pipe_name}]",
- f'type = "{pipe.type}"',
- f'definition = "{pipe.definition}"',
- ]
-
- # Add inputs if they exist
- PipelexInterpreter.add_inputs_to_lines_if_exist(lines, pipe.inputs)
-
- lines.append(f'output = "{PipelexInterpreter.escape_plx_string(pipe.output)}"')
- lines.append(f'function_name = "{PipelexInterpreter.escape_plx_string(pipe.function_name)}"')
-
- return "\n".join(lines)
-
- @staticmethod
- def img_gen_pipe_to_plx_string(pipe_name: str, pipe: PipeImgGenBlueprint, domain: str) -> str:
- """Convert a PipeImgGen blueprint directly to PLX section string."""
- lines: list[str] = [
- f"[pipe.{pipe_name}]",
- f'type = "{pipe.type}"',
- f'definition = "{pipe.definition}"',
- ]
-
- # Add inputs if they exist
- PipelexInterpreter.add_inputs_to_lines_if_exist(lines, pipe.inputs)
-
- lines.append(f'output = "{PipelexInterpreter.escape_plx_string(pipe.output)}"')
-
- # Add optional fields
- if pipe.img_gen_prompt:
- lines.append(f'img_gen_prompt = "{PipelexInterpreter.escape_plx_string(pipe.img_gen_prompt)}"')
- if pipe.aspect_ratio:
- lines.append(f'aspect_ratio = "{pipe.aspect_ratio}"')
- if pipe.is_raw is not None:
- lines.append(f"is_raw = {str(pipe.is_raw).lower()}")
- if pipe.seed:
- lines.append(f"seed = {pipe.seed}")
- if pipe.nb_output:
- lines.append(f"nb_output = {pipe.nb_output}")
-
- return "\n".join(lines)
-
- @staticmethod
- def jinja2_pipe_to_plx_string(pipe_name: str, pipe: PipeJinja2Blueprint, domain: str) -> str:
- """Convert a PipeJinja2 blueprint directly to PLX section string."""
- lines: list[str] = [
- f"[pipe.{pipe_name}]",
- f'type = "{pipe.type}"',
- f'definition = "{pipe.definition}"',
- ]
-
- # Add inputs if they exist
- PipelexInterpreter.add_inputs_to_lines_if_exist(lines, pipe.inputs)
-
- lines.append(f'output = "{PipelexInterpreter.escape_plx_string(pipe.output)}"')
-
- # Add jinja2 template
- if pipe.jinja2:
- lines.append(f'jinja2 = "{PipelexInterpreter.escape_plx_string(pipe.jinja2)}"')
- if pipe.jinja2_name:
- lines.append(f'jinja2_name = "{PipelexInterpreter.escape_plx_string(pipe.jinja2_name)}"')
-
- return "\n".join(lines)
-
- @staticmethod
- def condition_pipe_to_plx_string(pipe_name: str, pipe: PipeConditionBlueprint, domain: str) -> str:
- """Convert a PipeCondition blueprint directly to PLX section string."""
- lines: list[str] = [
- f"[pipe.{pipe_name}]",
- f'type = "{pipe.type}"',
- f'definition = "{pipe.definition}"',
- ]
-
- # Add inputs if they exist
- PipelexInterpreter.add_inputs_to_lines_if_exist(lines, pipe.inputs)
-
- lines.append(f'output = "{PipelexInterpreter.escape_plx_string(pipe.output)}"')
-
- # Add pipe_map
- if pipe.pipe_map:
- pipe_map_parts: list[str] = []
- pipe_map_dict = pipe.pipe_map.root
- for key, value in pipe_map_dict.items():
- pipe_map_parts.append(f'{key} = "{value}"')
- lines.append(f"pipe_map = {{ {', '.join(pipe_map_parts)} }}")
-
- # Add optional fields
- if pipe.expression_template:
- lines.append(f'expression_template = "{pipe.expression_template}"')
- if pipe.expression:
- lines.append(f'expression = "{pipe.expression}"')
- if pipe.default_pipe_code:
- lines.append(f'default_pipe_code = "{pipe.default_pipe_code}"')
- if pipe.add_alias_from_expression_to:
- lines.append(f'add_alias_from_expression_to = "{pipe.add_alias_from_expression_to}"')
-
- return "\n".join(lines)
-
- @staticmethod
- def parallel_pipe_to_plx_string(pipe_name: str, pipe: PipeParallelBlueprint, domain: str) -> str:
- """Convert a PipeParallel blueprint directly to PLX section string."""
- lines: list[str] = [
- f"[pipe.{pipe_name}]",
- f'type = "{pipe.type}"',
- f'definition = "{pipe.definition}"',
- ]
-
- # Add inputs if they exist
- PipelexInterpreter.add_inputs_to_lines_if_exist(lines, pipe.inputs)
-
- lines.append(f'output = "{PipelexInterpreter.escape_plx_string(pipe.output)}"')
-
- # Add parallels array
- if pipe.parallels:
- lines.append("parallels = [")
- for parallel in pipe.parallels:
- parallel_string = PipelexInterpreter.sub_pipe_to_plx_string(parallel)
- lines.append(f" {parallel_string},")
- lines.append("]")
-
- # Add optional fields
- if pipe.add_each_output is not True: # Only include if not default True
- lines.append(f"add_each_output = {str(pipe.add_each_output).lower()}")
- if pipe.combined_output:
- lines.append(f'combined_output = "{pipe.combined_output}"')
-
- return "\n".join(lines)
-
- @staticmethod
- def batch_pipe_to_plx_string(pipe_name: str, pipe: PipeBatchBlueprint, domain: str) -> str:
- """Convert a PipeBatch blueprint directly to PLX section string."""
- lines: list[str] = [
- f"[pipe.{pipe_name}]",
- f'type = "{pipe.type}"',
- f'definition = "{pipe.definition}"',
- ]
-
- # Add inputs if they exist
- PipelexInterpreter.add_inputs_to_lines_if_exist(lines, pipe.inputs)
-
- lines.append(f'output = "{PipelexInterpreter.escape_plx_string(pipe.output)}"')
- lines.append(f'branch_pipe_code = "{pipe.branch_pipe_code}"')
-
- # Add optional fields
- if pipe.input_list_name:
- lines.append(f'input_list_name = "{pipe.input_list_name}"')
- if pipe.input_item_name:
- lines.append(f'input_item_name = "{pipe.input_item_name}"')
-
- return "\n".join(lines)
-
- @staticmethod
- def serialize_concept_structure_field(field_value: Any) -> Any:
- """Serialize a single concept structure field."""
- from pipelex.core.concepts.concept_blueprint import ConceptStructureBlueprint
-
- if isinstance(field_value, str):
- return field_value
- elif isinstance(field_value, ConceptStructureBlueprint):
- field_data: Dict[str, Any] = {
- "definition": field_value.definition,
- "required": field_value.required,
- }
- if field_value.type is not None:
- field_data["type"] = field_value.type
- if field_value.item_type is not None:
- field_data["item_type"] = field_value.item_type
- if field_value.key_type is not None:
- field_data["key_type"] = field_value.key_type
- if field_value.value_type is not None:
- field_data["value_type"] = field_value.value_type
- if field_value.choices:
- field_data["choices"] = field_value.choices
- if field_value.default_value is not None:
- field_data["default_value"] = field_value.default_value
- return field_data
- else:
- return field_value
-
- @staticmethod
- def serialize_concept_structure(
- structure: Union[str, Dict[str, Any]],
- ) -> Union[str, Dict[str, Any]]:
- """Serialize concept structure field."""
- if isinstance(structure, str):
- return structure
- else:
- result: Dict[str, Any] = {}
- for field_name, field_value in structure.items():
- result[field_name] = PipelexInterpreter.serialize_concept_structure_field(field_value)
- return result
-
- @staticmethod
- def serialize_single_concept(
- concept_blueprint: Union[ConceptBlueprint, str],
- ) -> Union[str, Dict[str, Any]]:
- """Serialize a single concept blueprint."""
- if isinstance(concept_blueprint, str):
- return concept_blueprint
-
- # Handle ConceptBlueprint object
- if concept_blueprint.structure is not None:
- # Structured concept
- concept_data = {
- "definition": concept_blueprint.definition,
- "structure": PipelexInterpreter.serialize_concept_structure(concept_blueprint.structure),
- }
- return concept_data
- elif concept_blueprint.refines is not None:
- # Concept with refines
- concept_data = {
- "definition": concept_blueprint.definition,
- "refines": concept_blueprint.refines,
- }
- return concept_data
- else:
- # Simple concept with just definition
- return concept_blueprint.definition
-
- @staticmethod
- def serialize_concepts(concepts: Optional[Dict[str, ConceptBlueprint | str]], domain: str) -> Dict[str, Any]:
- """Serialize concepts section with domain context."""
- if concepts is None:
- return {}
-
- result: Dict[str, Any] = {}
- for concept_name, concept_blueprint in concepts.items():
- result[concept_name] = PipelexInterpreter.serialize_single_concept(concept_blueprint)
- return result
-
- @staticmethod
- def serialize_pipes(pipes: Dict[str, Any], domain: str) -> Dict[str, Any]:
- """Serialize pipes section with domain context."""
- result: Dict[str, Any] = {}
- for pipe_name, blueprint in pipes.items():
- result[pipe_name] = PipelexInterpreter.serialize_pipe(blueprint, domain)
- return result
-
- @staticmethod
- def serialize_pipe(blueprint: Any, domain: str) -> Dict[str, Any]:
- """Serialize a single pipe blueprint with domain context."""
-
- if isinstance(blueprint, PipeLLMBlueprint):
- return PipelexInterpreter.serialize_llm_pipe(blueprint, domain)
- elif isinstance(blueprint, PipeOcrBlueprint):
- return PipelexInterpreter.serialize_ocr_pipe(blueprint, domain)
- elif isinstance(blueprint, PipeFuncBlueprint):
- return PipelexInterpreter._serialize_func_pipe(blueprint, domain)
- elif isinstance(blueprint, PipeImgGenBlueprint):
- return PipelexInterpreter._serialize_img_gen_pipe(blueprint, domain)
- elif isinstance(blueprint, PipeJinja2Blueprint):
- return PipelexInterpreter.serialize_jinja2_pipe(blueprint, domain)
- elif isinstance(blueprint, PipeSequenceBlueprint):
- return PipelexInterpreter.serialize_sequence_pipe(blueprint, domain)
- elif isinstance(blueprint, PipeConditionBlueprint):
- return PipelexInterpreter._serialize_condition_pipe(blueprint, domain)
- elif isinstance(blueprint, PipeParallelBlueprint):
- return PipelexInterpreter._serialize_parallel_pipe(blueprint, domain)
- elif isinstance(blueprint, PipeBatchBlueprint):
- return PipelexInterpreter._serialize_batch_pipe(blueprint, domain)
- else:
- raise PipelexUnknownPipeError(f"Unknown pipe blueprint type: {type(blueprint)}")
-
- @staticmethod
- def serialize_llm_pipe(pipe: PipeLLMBlueprint, domain: str) -> Dict[str, Any]:
- """Serialize PipeLLM blueprint."""
- result: Dict[str, Any] = {
- "type": pipe.type,
- "definition": pipe.definition,
- }
-
- # Add inputs first if they exist
- if pipe.inputs:
- result["inputs"] = PipelexInterpreter.serialize_inputs(pipe.inputs)
-
- # Then output
- result["output"] = pipe.output
-
- # Add optional fields in expected order based on test cases
- if pipe.nb_output is not None:
- result["nb_output"] = pipe.nb_output
- if pipe.multiple_output is not None:
- result["multiple_output"] = pipe.multiple_output
-
- if pipe.system_prompt_template:
- result["system_prompt_template"] = pipe.system_prompt_template
- if pipe.system_prompt:
- result["system_prompt"] = pipe.system_prompt
-
- if pipe.prompt_template:
- result["prompt_template"] = pipe.prompt_template
- if pipe.template_name:
- result["template_name"] = pipe.template_name
- if pipe.prompt:
- result["prompt"] = pipe.prompt
-
- # Less common fields
- if pipe.system_prompt_template_name:
- result["system_prompt_template_name"] = pipe.system_prompt_template_name
- if pipe.system_prompt_name:
- result["system_prompt_name"] = pipe.system_prompt_name
- if pipe.prompt_name:
- result["prompt_name"] = pipe.prompt_name
- if pipe.llm:
- result["llm"] = pipe.llm
- if pipe.llm_to_structure:
- result["llm_to_structure"] = pipe.llm_to_structure
- if pipe.structuring_method:
- result["structuring_method"] = pipe.structuring_method
- if pipe.prompt_template_to_structure:
- result["prompt_template_to_structure"] = pipe.prompt_template_to_structure
- if pipe.system_prompt_to_structure:
- result["system_prompt_to_structure"] = pipe.system_prompt_to_structure
-
- return result
-
- @staticmethod
- def add_common_pipe_fields(result: Dict[str, Any], pipe: Any) -> None:
- """Add common pipe fields (inputs) to result dict if they exist."""
- if hasattr(pipe, "inputs") and pipe.inputs:
- result["inputs"] = PipelexInterpreter.serialize_inputs(pipe.inputs)
-
- @staticmethod
- def _serialize_ocr_pipe(pipe: PipeOcrBlueprint, domain: str) -> Dict[str, Any]:
- """Serialize PipeOcr blueprint."""
- result: Dict[str, Any] = {
- "type": pipe.type,
- "definition": pipe.definition,
- "output": pipe.output,
- }
-
- PipelexInterpreter.add_common_pipe_fields(result, pipe)
- return result
-
- @staticmethod
- def _serialize_func_pipe(pipe: PipeFuncBlueprint, domain: str) -> Dict[str, Any]:
- """Serialize PipeFunc blueprint."""
- result: Dict[str, Any] = {
- "type": pipe.type,
- "definition": pipe.definition,
- "output": pipe.output,
- "function_name": pipe.function_name,
- }
-
- PipelexInterpreter.add_common_pipe_fields(result, pipe)
- return result
-
- @staticmethod
- def _serialize_img_gen_pipe(pipe: PipeImgGenBlueprint, domain: str) -> Dict[str, Any]:
- """Serialize PipeImgGen blueprint."""
- result: Dict[str, Any] = {
- "type": pipe.type,
- "definition": pipe.definition,
- "output": pipe.output,
- }
-
- PipelexInterpreter.add_common_pipe_fields(result, pipe)
-
- # Add optional fields only if they have values
- if pipe.img_gen_prompt:
- result["img_gen_prompt"] = pipe.img_gen_prompt
- if pipe.aspect_ratio:
- result["aspect_ratio"] = pipe.aspect_ratio
- if pipe.is_raw is not None:
- result["is_raw"] = pipe.is_raw
- if pipe.seed:
- result["seed"] = pipe.seed
- if pipe.nb_output:
- result["nb_output"] = pipe.nb_output
-
- return result
-
- @staticmethod
- def serialize_ocr_pipe(pipe: PipeOcrBlueprint, domain: str) -> Dict[str, Any]:
- """Serialize a PipeOcr blueprint."""
- result: Dict[str, Any] = {
- "type": pipe.type,
- "definition": pipe.definition,
- }
-
- # Add common pipe fields
- PipelexInterpreter.add_common_pipe_fields(result, pipe)
-
- # Add output
- result["output"] = pipe.output
-
- # Add required fields
- result["ocr"] = pipe.ocr
-
- # Add optional fields
- if pipe.page_images:
- result["page_images"] = pipe.page_images
- if pipe.page_image_captions:
- result["page_image_captions"] = pipe.page_image_captions
- if pipe.page_views:
- result["page_views"] = pipe.page_views
- if pipe.page_views_dpi:
- result["page_views_dpi"] = pipe.page_views_dpi
-
- return result
-
- @staticmethod
- def serialize_sequence_pipe(pipe: PipeSequenceBlueprint, domain: str) -> Dict[str, Any]:
- """Serialize a PipeSequence blueprint."""
- result: Dict[str, Any] = {
- "type": pipe.type,
- "definition": pipe.definition,
- }
-
- # Add common pipe fields
- PipelexInterpreter.add_common_pipe_fields(result, pipe)
-
- # Add output
- result["output"] = pipe.output
-
- # Add steps
- if pipe.steps:
- result["steps"] = [PipelexInterpreter.serialize_sub_pipe(step) for step in pipe.steps]
-
- return result
-
- @staticmethod
- def serialize_jinja2_pipe(pipe: PipeJinja2Blueprint, domain: str) -> Dict[str, Any]:
- """Serialize PipeJinja2 blueprint."""
- result: Dict[str, Any] = {
- "type": pipe.type,
- "definition": pipe.definition,
- "output": pipe.output,
- }
-
- PipelexInterpreter.add_common_pipe_fields(result, pipe)
-
- # Add optional fields only if they have values
- if pipe.jinja2_name:
- result["jinja2_name"] = pipe.jinja2_name
- if pipe.jinja2:
- result["jinja2"] = pipe.jinja2
- if pipe.prompting_style:
- result["prompting_style"] = pipe.prompting_style
- result["template_category"] = pipe.template_category
-
- return result
-
- @staticmethod
- def serialize_sub_pipe(sub_pipe: Any) -> Dict[str, Any]:
- """Serialize a sub pipe (step or parallel) blueprint."""
- step_data: Dict[str, Any] = {
- "pipe": sub_pipe.pipe,
- "result": sub_pipe.result,
- }
- # Only include optional fields if they have non-default values
- if sub_pipe.nb_output is not None:
- step_data["nb_output"] = sub_pipe.nb_output
- if sub_pipe.multiple_output is not None:
- step_data["multiple_output"] = sub_pipe.multiple_output
- if sub_pipe.batch_over is not False: # Only include if not default False
- step_data["batch_over"] = sub_pipe.batch_over
- if sub_pipe.batch_as is not None:
- step_data["batch_as"] = sub_pipe.batch_as
- return step_data
-
- @staticmethod
- def sub_pipe_to_plx_string(sub_pipe: Any) -> str:
- """Convert a sub pipe blueprint directly to a PLX inline table string."""
- parts = [f'pipe = "{sub_pipe.pipe}"', f'result = "{sub_pipe.result}"']
-
- # Add optional fields if they have non-default values
- if sub_pipe.nb_output is not None:
- parts.append(f"nb_output = {sub_pipe.nb_output}")
- if sub_pipe.multiple_output is not None:
- parts.append(f"multiple_output = {str(sub_pipe.multiple_output).lower()}")
- if sub_pipe.batch_over is not False: # Only include if not default False
- if isinstance(sub_pipe.batch_over, str):
- parts.append(f'batch_over = "{sub_pipe.batch_over}"')
- else:
- parts.append(f"batch_over = {str(sub_pipe.batch_over).lower()}")
- if sub_pipe.batch_as is not None:
- parts.append(f'batch_as = "{sub_pipe.batch_as}"')
-
- return "{ " + ", ".join(parts) + " }"
-
- @staticmethod
- def _serialize_sequence_pipe(pipe: PipeSequenceBlueprint, domain: str) -> Dict[str, Any]:
- """Serialize PipeSequence blueprint."""
- result: Dict[str, Any] = {
- "type": pipe.type,
- "definition": pipe.definition,
- "output": pipe.output,
- }
-
- PipelexInterpreter.add_common_pipe_fields(result, pipe)
-
- if pipe.steps:
- result["steps"] = [PipelexInterpreter.serialize_sub_pipe(step) for step in pipe.steps]
-
- return result
-
- @staticmethod
- def sequence_pipe_to_plx_string(pipe_name: str, pipe: PipeSequenceBlueprint, domain: str) -> str:
- """Convert a PipeSequence blueprint directly to PLX section string."""
- lines: list[str] = [
- f"[pipe.{pipe_name}]",
- f'type = "{pipe.type}"',
- f'definition = "{pipe.definition}"',
- ]
-
- # Add inputs first if they exist
- if hasattr(pipe, "inputs"):
- PipelexInterpreter.add_inputs_to_lines_if_exist(lines, pipe.inputs)
-
- # Add output after inputs
- lines.append(f'output = "{pipe.output}"')
-
- # Add steps array with proper spacing
- if pipe.steps:
- lines.append("steps = [")
- for step in pipe.steps:
- step_string = PipelexInterpreter.sub_pipe_to_plx_string(step)
- lines.append(f" {step_string},")
- lines.append("]")
-
- return "\n".join(lines)
-
- @staticmethod
- def _serialize_condition_pipe(pipe: PipeConditionBlueprint, domain: str) -> Dict[str, Any]:
- """Serialize PipeCondition blueprint."""
- result: Dict[str, Any] = {
- "type": pipe.type,
- "definition": pipe.definition,
- "output": pipe.output,
- "pipe_map": pipe.pipe_map,
- }
-
- PipelexInterpreter.add_common_pipe_fields(result, pipe)
-
- # Add optional fields only if they have values
- if pipe.expression_template:
- result["expression_template"] = pipe.expression_template
- if pipe.expression:
- result["expression"] = pipe.expression
- if pipe.default_pipe_code:
- result["default_pipe_code"] = pipe.default_pipe_code
- if pipe.add_alias_from_expression_to:
- result["add_alias_from_expression_to"] = pipe.add_alias_from_expression_to
-
- return result
-
- @staticmethod
- def _serialize_parallel_pipe(pipe: PipeParallelBlueprint, domain: str) -> Dict[str, Any]:
- """Serialize PipeParallel blueprint."""
- result: Dict[str, Any] = {
- "type": pipe.type,
- "definition": pipe.definition,
- "output": pipe.output,
- }
-
- PipelexInterpreter.add_common_pipe_fields(result, pipe)
-
- if pipe.parallels:
- result["parallels"] = [PipelexInterpreter.serialize_sub_pipe(parallel) for parallel in pipe.parallels]
- if pipe.add_each_output is not True: # Only include if not default True
- result["add_each_output"] = pipe.add_each_output
- if pipe.combined_output:
- result["combined_output"] = pipe.combined_output
-
- return result
-
- @staticmethod
- def _serialize_batch_pipe(pipe: PipeBatchBlueprint, domain: str) -> Dict[str, Any]:
- """Serialize PipeBatch blueprint."""
- result: Dict[str, Any] = {
- "type": pipe.type,
- "definition": pipe.definition,
- "output": pipe.output,
- "branch_pipe_code": pipe.branch_pipe_code,
- }
-
- PipelexInterpreter.add_common_pipe_fields(result, pipe)
-
- if pipe.input_list_name:
- result["input_list_name"] = pipe.input_list_name
- if pipe.input_item_name:
- result["input_item_name"] = pipe.input_item_name
-
- return result
-
- @staticmethod
- def serialize_input_requirement(
- input_req: InputRequirementBlueprint,
- ) -> Dict[str, Any]:
- """Serialize a single InputRequirementBlueprint to PLX format."""
- result: Dict[str, Any] = {"concept": input_req.concept}
- if input_req.multiplicity is not None:
- result["multiplicity"] = input_req.multiplicity
- return result
-
- @staticmethod
- def serialize_inputs(
- inputs: Mapping[str, Union[str, InputRequirementBlueprint]],
- ) -> Dict[str, Any]:
- """Convert InputRequirementBlueprint objects to proper PLX format for serialization."""
- result: Dict[str, Any] = {}
- for key, value in inputs.items():
- if isinstance(value, InputRequirementBlueprint):
- result[key] = PipelexInterpreter.serialize_input_requirement(value)
- else:
- # Already a string - simple case
- result[key] = str(value)
- return result
-
- @staticmethod
- def inputs_to_plx_string(
- inputs: Mapping[str, Union[str, InputRequirementBlueprint]],
- ) -> str:
- """Convert inputs dictionary to PLX string format."""
- inputs_dict = PipelexInterpreter.serialize_inputs(inputs)
- inputs_parts: list[str] = []
-
- for key, value in inputs_dict.items():
- if isinstance(value, dict):
- # Nested dict (like InputRequirementBlueprint)
- nested_parts: list[str] = []
- nested_dict: Dict[str, Any] = cast(Dict[str, Any], value)
- for nested_key, nested_value in nested_dict.items():
- if isinstance(nested_value, str):
- nested_parts.append(f'{nested_key} = "{PipelexInterpreter.escape_plx_string(nested_value)}"')
- else:
- nested_parts.append(f"{nested_key} = {str(nested_value).lower()}")
- inputs_parts.append(f"{key} = {{ {', '.join(nested_parts)} }}")
- else:
- # Simple string value
- if isinstance(value, str):
- inputs_parts.append(f'{key} = "{PipelexInterpreter.escape_plx_string(value)}"')
- else:
- # Fallback for any other type
- inputs_parts.append(f'{key} = "{PipelexInterpreter.escape_plx_string(str(value))}"')
-
- return f"{{ {', '.join(inputs_parts)} }}"
-
- @staticmethod
- def add_inputs_to_lines_if_exist(
- lines: list[str],
- pipe_inputs: Optional[Mapping[str, Union[str, InputRequirementBlueprint]]],
- ) -> None:
- """Add inputs line to PLX lines if inputs exist."""
- if pipe_inputs:
- inputs_dict = PipelexInterpreter.serialize_inputs(pipe_inputs)
- inputs_parts: list[str] = []
- for key, value in inputs_dict.items():
- if isinstance(value, dict):
- nested_parts: list[str] = []
- nested_dict: Dict[str, Any] = cast(Dict[str, Any], value)
- for nested_key, nested_value in nested_dict.items():
- if isinstance(nested_value, str):
- nested_parts.append(f'{nested_key} = "{PipelexInterpreter.escape_plx_string(nested_value)}"')
- else:
- nested_parts.append(f"{nested_key} = {str(nested_value).lower()}")
- inputs_parts.append(f"{key} = {{ {', '.join(nested_parts)} }}")
- else:
- if isinstance(value, str):
- inputs_parts.append(f'{key} = "{PipelexInterpreter.escape_plx_string(value)}"')
- else:
- inputs_parts.append(f'{key} = "{PipelexInterpreter.escape_plx_string(str(value))}"')
- lines.append(f"inputs = {{ {', '.join(inputs_parts)} }}")
diff --git a/pipelex/tools/misc/toml_utils.py b/pipelex/tools/misc/toml_utils.py
index c14e5c286..1afa2f93e 100644
--- a/pipelex/tools/misc/toml_utils.py
+++ b/pipelex/tools/misc/toml_utils.py
@@ -130,267 +130,3 @@ def failable_load_toml_from_path(path: str) -> Optional[Dict[str, Any]]:
except (toml.TomlDecodeError, TOMLValidationError) as exc:
print(f"Failed to parse TOML file '{path}': {exc}")
return None
-
-
-def make_toml_string(
- text: str,
- prefer_literal: bool = False,
- force_multiline: bool = False,
- ensure_trailing_newline: bool = True,
- ensure_leading_blank_line_in_value: bool = False,
-):
- """
- Build a tomlkit string node.
- - If `force_multiline` or the text contains '\\n', we emit a triple-quoted multiline string.
- - When multiline, `ensure_trailing_newline` puts the closing quotes on their own line.
- - When multiline, `ensure_leading_blank_line_in_value` inserts a real blank line at the start of the value.
- """
- needs_multiline = force_multiline or ("\n" in text)
- normalized = text
-
- if needs_multiline:
- if ensure_leading_blank_line_in_value and not normalized.startswith("\n"):
- normalized = "\n" + normalized
- if ensure_trailing_newline and not normalized.endswith("\n"):
- normalized = normalized + "\n"
-
- use_literal = prefer_literal and ("'''" not in normalized)
- return tk_string(normalized, multiline=needs_multiline, literal=use_literal)
-
-
-def _convert_to_inline(value: Any) -> Any:
- """Recursively convert Python values; dicts -> inline tables; lists kept as arrays."""
- # Handle Pydantic models by converting them to dict first
- if isinstance(value, BaseModel):
- # For RootModel, use the root attribute; for regular models, use model_dump()
- if hasattr(value, "root"):
- # This is a RootModel, use its root value
- value = getattr(value, "root")
- else:
- # This is a regular BaseModel, convert to dict
- value = value.model_dump()
-
- if isinstance(value, Mapping):
- value = cast(Mapping[str, Any], value)
- inline_table_obj = inline_table()
- for key, value_item in value.items():
- inline_table_obj[key] = _convert_to_inline(value_item)
-
- return inline_table_obj
-
- if isinstance(value, list):
- value = cast(List[Any], value)
- array_obj = array()
- array_obj.multiline(True) # set to False for single-line arrays
- for element in value:
- if isinstance(element, Mapping):
- element = cast(Mapping[str, Any], element)
- inline_element = inline_table()
- for inner_key, inner_value in element.items():
- inline_element[inner_key] = _convert_to_inline(inner_value)
- array_obj.append(inline_element) # pyright: ignore[reportUnknownMemberType]
- else:
- array_obj.append(_convert_to_inline(element)) # pyright: ignore[reportUnknownMemberType]
- return array_obj
-
- if isinstance(value, str):
- # For prompt templates and similar fields, escape newlines instead of using multiline format
- if any(keyword in str(value) for keyword in ["@", "$"]) and "\n" in value:
- # This looks like a prompt template - escape newlines for single-line format
- # Return the string directly and let tomlkit handle the escaping properly
- return value
- else:
- # Triple quotes if needed (or forced); closing quotes on their own line.
- return make_toml_string(
- value,
- prefer_literal=False, # flip to True for '''...'''
- force_multiline=False, # flip to True to force """...""" even without \n
- ensure_trailing_newline=True, # keep closing """ on its own line
- ensure_leading_blank_line_in_value=False, # flip to True to keep a blank first line
- )
- return value
-
-
-def _filter_empty_values(value: Any) -> Any:
- """Filter out empty lists and None values from data structures."""
- if isinstance(value, dict):
- filtered: Dict[str, Any] = {}
- for k, v in cast(Dict[str, Any], value).items():
- filtered_v = _filter_empty_values(v)
- # Keep empty dicts but skip empty lists and None values
- if filtered_v is not None and (not isinstance(filtered_v, list) or filtered_v):
- filtered[k] = filtered_v
- return filtered
- elif isinstance(value, list):
- return [_filter_empty_values(item) for item in cast(List[Any], value) if _filter_empty_values(item) is not None]
- else:
- return value
-
-
-def _create_ordered_inline_table(data: Mapping[str, Any]) -> Any:
- """Create an inline table with fields in the expected order."""
- inline_table_obj = inline_table()
-
- # Define the preferred order for concept structure fields
- field_order = ["type", "definition", "required"]
-
- # Add fields in preferred order first
- for field in field_order:
- if field in data:
- value = data[field]
- # Skip empty lists
- if isinstance(value, list) and not value:
- continue
- inline_table_obj[field] = _convert_to_inline(value)
-
- # Add any remaining fields not in the preferred order
- for key, value in data.items():
- if key not in field_order:
- # Skip empty lists
- if isinstance(value, list) and not value:
- continue
- inline_table_obj[key] = _convert_to_inline(value)
-
- return inline_table_obj
-
-
-def dict_to_toml(data: Mapping[str, Any]) -> str:
- """Convert dictionary to TOML format matching Pipelex expectations."""
- data = remove_none_values_from_dict(data=data)
- data = _filter_empty_values(data)
- document_root = document()
-
- # Handle top-level fields first (domain, definition, system_prompt, etc.)
- for key, value in data.items():
- if not isinstance(value, Mapping):
- document_root.add(key, _convert_to_inline(value))
-
- # Handle sections (concepts, pipes)
- for section_key, section_value in data.items():
- if isinstance(section_value, Mapping):
- section_value = cast(Mapping[str, Any], section_value)
-
- # Skip empty sections
- if not section_value:
- continue
-
- # Create the section table
- section_table = table()
-
- # Process each item in the section
- for item_key, item_value in section_value.items():
- if isinstance(item_value, str):
- # Simple string value (e.g., "SimpleData = 'Simple data concept'")
- section_table.add(item_key, _convert_to_inline(item_value))
- elif isinstance(item_value, Mapping):
- # Complex object that needs its own table
- item_value = cast(Mapping[str, Any], item_value)
- item_table = table()
-
- # Handle the structure field specially
- for field_key, field_value in item_value.items():
- # THIS IS CUSTOM CODE FOR THE CONCEPT STRUCUTRE FIELD. It should be generic
- if field_key == "structure" and isinstance(field_value, Mapping):
- # Structure should be its own table [section.item.structure]
- structure_table = table()
- for struct_key, struct_value in cast(Mapping[str, Any], field_value).items():
- if isinstance(struct_value, Mapping):
- structure_table.add(struct_key, _create_ordered_inline_table(cast(Mapping[str, Any], struct_value)))
- else:
- structure_table.add(struct_key, _convert_to_inline(struct_value))
- item_table.add(field_key, structure_table)
- else:
- # Skip empty lists like refines = []
- if isinstance(field_value, list) and not field_value:
- continue
- item_table.add(field_key, _convert_to_inline(field_value))
-
- section_table.add(item_key, item_table)
-
- document_root.add(section_key, section_table)
-
- dumped_content = tomlkit.dumps(document_root) # pyright: ignore[reportUnknownMemberType]
-
- # Post-process to fix inline table spacing: {key = "value"} -> { key = "value" }
-
- # Fix spacing around inline table contents - ensure single space after { and before }
- # This regex matches TOML inline tables (containing = signs) but not Jinja2 templates
- # We need to handle nested braces properly by working from innermost to outermost
-
- def fix_inline_table_spacing(content: str) -> str:
- """Fix spacing in inline tables, handling nested structures, but avoid modifying braces inside strings."""
- # Split content into lines and process each line
- lines = content.split("\n")
- processed_lines: list[str] = []
-
- for line in lines:
- # Only process lines that contain TOML inline tables (have '=' and braces outside quotes)
- # Skip lines that are just string values with braces (like Jinja2 templates)
- if "=" in line and "{" in line and "}" in line:
- # Check if this line has a TOML key-value assignment with inline table
- # Pattern: key = { ... } or key = { nested = { ... } }
- if re.match(r"^\s*\w+\s*=\s*\{", line):
- # This is a TOML inline table assignment, apply spacing fixes
- processed_line = line
-
- # Apply multiple passes to handle nested structures properly
- for _ in range(10): # Prevent infinite loops
- old_line = processed_line
-
- # First: fix simple inline tables (no nested braces)
- processed_line = re.sub(r"\{\s*([^{}]*=[^{}]*?)\s*\}", r"{ \1 }", processed_line)
-
- # Second: fix nested inline tables - outer tables containing inner ones
- # Match pattern like { key = { inner = "value" } }
- processed_line = re.sub(r"\{\s*([^={}]*=\s*\{[^}]*\}[^{}]*?)\s*\}", r"{ \1 }", processed_line)
-
- # Third: fix complex nested cases with multiple inner tables
- # Handle cases like {key1 = {inner = "value"}, key2 = {inner2 = "value2"}}
- processed_line = re.sub(r"\{\s*([^{}]*=\s*\{[^}]*\}.*?)\s*\}", r"{ \1 }", processed_line)
-
- # Fourth: clean up any excessive whitespace that might remain
- # Remove multiple consecutive spaces inside tables
- processed_line = re.sub(r"\{\s*([^{}]*?)\s{2,}\}", r"{ \1 }", processed_line)
-
- # Fifth: ensure consistent single space formatting
- processed_line = re.sub(r"\{\s{2,}([^{}]*?)\s*\}", r"{ \1 }", processed_line)
-
- if old_line == processed_line:
- break
-
- # Final cleanup: fix any remaining spacing issues in inline tables
- # First: ensure there's a space between adjacent closing braces "}}"
- processed_line = re.sub(r"\}\}", "} }", processed_line)
-
- # Then: remove extra spaces (2 or more) before closing braces, but keep single spaces
- # This pattern matches cases like "} }" (2+ spaces) and replaces with "} }"
- processed_line = re.sub(r"\}\s{2,}\}", "} }", processed_line)
-
- processed_lines.append(processed_line)
- else:
- # This line has braces but is not a TOML inline table (e.g., string with Jinja2)
- # Leave it unchanged
- processed_lines.append(line)
- else:
- # No braces or no assignment, leave unchanged
- processed_lines.append(line)
-
- return "\n".join(processed_lines)
-
- dumped_content = fix_inline_table_spacing(dumped_content)
-
- return dumped_content
-
-
-def save_toml_to_path(data: Dict[str, Any], path: str) -> None:
- """Save dictionary as TOML to file path.
-
- Args:
- data: Dictionary to save as TOML
- path: Path where to save the TOML file
- """
- data_cleaned = data
- with open(path, "w", encoding="utf-8") as file:
- toml_content: str = dict_to_toml(data=data_cleaned)
- cleaned_content = clean_trailing_whitespace(toml_content)
- file.write(cleaned_content)
diff --git a/pipelex/tools/plx/__init__.py b/pipelex/tools/plx/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/pipelex/tools/plx/plx_utils.py b/pipelex/tools/plx/plx_utils.py
new file mode 100644
index 000000000..018fef9fd
--- /dev/null
+++ b/pipelex/tools/plx/plx_utils.py
@@ -0,0 +1,116 @@
+from __future__ import annotations
+
+from typing import Any, List, Mapping, cast
+
+import tomlkit
+from tomlkit import array, document, inline_table, table
+from tomlkit import string as tomlkit_string
+
+from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint
+from pipelex.tools.misc.file_utils import save_text_to_path
+from pipelex.tools.misc.json_utils import remove_none_values_from_dict
+
+
+def _format_tomlkit_string(
+ text: str,
+ prefer_literal: bool = True,
+ force_multiline: bool = False,
+ ensure_trailing_newline: bool = True,
+ ensure_leading_blank_line: bool = True,
+) -> Any: # Can't type this because of tomlkit
+ """
+ Build a tomlkit string node.
+ - If `force_multiline` or the text contains '\\n', we emit a triple-quoted multiline string.
+ - When multiline, `ensure_trailing_newline` puts the closing quotes on their own line.
+ - When multiline, `ensure_leading_blank_line` inserts a real blank line at the start of the string.
+ """
+ needs_multiline = force_multiline or ("\n" in text)
+ normalized = text
+
+ if needs_multiline:
+ if ensure_leading_blank_line and not normalized.startswith("\n"):
+ normalized = "\n" + normalized
+ if ensure_trailing_newline and not normalized.endswith("\n"):
+ normalized = normalized + "\n"
+
+ use_literal = prefer_literal and ("'''" not in normalized)
+ return tomlkit_string(normalized, multiline=needs_multiline, literal=use_literal)
+
+
+def _convert_dicts_to_inline_tables(value: Any) -> Any: # Can't type this because of tomlkit
+ """Recursively convert Python values; dicts -> inline tables; lists kept as arrays."""
+ if isinstance(value, Mapping):
+ value = cast(Mapping[str, Any], value)
+ inline_table_obj = inline_table()
+ for key, value_item in value.items():
+ inline_table_obj[key] = _convert_dicts_to_inline_tables(value_item)
+ return inline_table_obj
+
+ if isinstance(value, list):
+ value = cast(List[Any], value)
+ array_obj = array()
+ array_obj.multiline(True) # set to False for single-line arrays
+ for element in value:
+ if isinstance(element, Mapping):
+ element = cast(Mapping[str, Any], element)
+ inline_element = inline_table()
+ for inner_key, inner_value in element.items():
+ inline_element[inner_key] = _convert_dicts_to_inline_tables(inner_value)
+ array_obj.append(inline_element) # pyright: ignore[reportUnknownMemberType]
+ else:
+ array_obj.append(_convert_dicts_to_inline_tables(element)) # pyright: ignore[reportUnknownMemberType]
+ return array_obj
+
+ if isinstance(value, str):
+ return _format_tomlkit_string(value)
+ return value
+
+
+def _convert_mapping_to_table(mapping: Mapping[str, Any]) -> Any: # Can't type this because of tomlkit
+ """
+ Convert a mapping into a TOML Table where any nested mappings (third level+)
+ are converted to inline tables.
+
+ This creates a second-level standard table, and only uses inline tables for
+ third level and deeper structures.
+ """
+ tbl = table()
+ for field_key, field_value in mapping.items():
+ if isinstance(field_value, Mapping):
+ # Third-level mapping -> inline table
+ tbl.add(field_key, _convert_dicts_to_inline_tables(field_value))
+ else:
+ tbl.add(field_key, _convert_dicts_to_inline_tables(field_value))
+ return tbl
+
+
+def dict_to_plx_styled_toml(data: Mapping[str, Any]) -> str:
+ """Top-level keys become tables; second-level mappings become tables; inline tables start at third level."""
+ data = remove_none_values_from_dict(data=data)
+ document_root = document()
+ for section_key, section_value in data.items():
+ if isinstance(section_value, Mapping):
+ section_value = cast(Mapping[str, Any], section_value)
+ table_obj = table()
+ for field_key, field_value in section_value.items():
+ if isinstance(field_value, Mapping):
+ # Second level mapping -> standard table [section.field]
+ sub_table = _convert_mapping_to_table(cast(Mapping[str, Any], field_value))
+ table_obj.add(field_key, sub_table)
+ else:
+ table_obj.add(field_key, _convert_dicts_to_inline_tables(field_value))
+ document_root.add(section_key, table_obj) # `[section_key]` section
+ else:
+ document_root.add(section_key, _convert_dicts_to_inline_tables(section_value))
+ dumped_content = tomlkit.dumps(document_root) # pyright: ignore[reportUnknownMemberType]
+ return dumped_content
+
+
+def make_plx_content(blueprint: PipelexBundleBlueprint) -> str:
+ blueprint_dict = blueprint.model_dump(serialize_as_any=True)
+ return dict_to_plx_styled_toml(data=blueprint_dict)
+
+
+# def save_bundle_blueprint_toml_to_path(blueprint: PipelexBundleBlueprint, path: str) -> None:
+# bundle_blueprint_toml = make_plx_content(blueprint=blueprint)
+# save_text_to_path(text=bundle_blueprint_toml, path=path)
diff --git a/tests/unit/pipelex/core/interpreter/test_interpreter.py b/tests/unit/pipelex/core/interpreter/test_interpreter.py
index fb2c2be72..023730c59 100644
--- a/tests/unit/pipelex/core/interpreter/test_interpreter.py
+++ b/tests/unit/pipelex/core/interpreter/test_interpreter.py
@@ -3,8 +3,10 @@ from typing import Type
import pytest
+from pipelex import pretty_print
from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint
from pipelex.core.interpreter import PipelexInterpreter
+from pipelex.tools.plx.plx_utils import make_plx_content
from tests.unit.pipelex.core.test_data import InterpreterTestCases
@@ -23,22 +25,19 @@ class TestPipelexInterpreter:
def test_make_pipelex_bundle_blueprint(self, test_name: str, plx_content: str, expected_blueprint: PipelexBundleBlueprint):
"""Test making blueprint from various valid PLX content."""
converter = PipelexInterpreter(file_content=plx_content)
- from pipelex import pretty_print
blueprint = converter.make_pipelex_bundle_blueprint()
pretty_print(blueprint, title=f"Blueprint {test_name}")
pretty_print(expected_blueprint, title=f"Expected blueprint {test_name}")
assert blueprint == expected_blueprint
- @pytest.mark.parametrize("test_name,plx_content,expected_blueprint", InterpreterTestCases.VALID_TEST_CASES)
- def test_make_plx_content(self, test_name: str, plx_content: str, expected_blueprint: PipelexBundleBlueprint):
+ @pytest.mark.parametrize("test_name,expected_plx_content,blueprint", InterpreterTestCases.VALID_TEST_CASES)
+ def test_make_plx_content(self, test_name: str, expected_plx_content: str, blueprint: PipelexBundleBlueprint):
"""Test making blueprint from various valid PLX content."""
- get_plx_content = PipelexInterpreter.make_plx_content(blueprint=expected_blueprint)
- from pipelex import pretty_print
-
- pretty_print(get_plx_content, title=f"PLx content {test_name}")
- pretty_print(plx_content, title=f"Expected PLX content {test_name}")
- assert get_plx_content == plx_content
+ plx_content = make_plx_content(blueprint=blueprint)
+ pretty_print(plx_content, title=f"PLx content {test_name}")
+ pretty_print(expected_plx_content, title=f"Expected PLX content {test_name}")
+ assert plx_content == expected_plx_content
@pytest.mark.parametrize("test_name,invalid_plx_content,expected_exception", InterpreterTestCases.ERROR_TEST_CASES)
def test_invalid_plx_should_raise_exception(self, test_name: str, invalid_plx_content: str, expected_exception: Type[Exception]):
diff --git a/tests/unit/pipelex/core/interpreter/test_interpreter_serialize.py b/tests/unit/pipelex/core/interpreter/test_interpreter_serialize.py
deleted file mode 100644
index 923b80aa1..000000000
--- a/tests/unit/pipelex/core/interpreter/test_interpreter_serialize.py
+++ /dev/null
@@ -1,360 +0,0 @@
-"""Unit tests for PipelexInterpreter serialize functions."""
-
-from typing import Any, Dict
-
-from pipelex.core.concepts.concept_blueprint import ConceptBlueprint, ConceptStructureBlueprint, ConceptStructureBlueprintFieldType
-from pipelex.core.interpreter import PipelexInterpreter
-from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
-from pipelex.pipe_controllers.sequence.pipe_sequence_blueprint import PipeSequenceBlueprint
-from pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprint
-from pipelex.pipe_operators.llm.pipe_llm_blueprint import PipeLLMBlueprint
-from pipelex.pipe_operators.ocr.pipe_ocr_blueprint import PipeOcrBlueprint
-
-
-class TestSerializeInputRequirement:
- """Test _serialize_input_requirement function."""
-
- def test_serialize_input_requirement_basic(self):
- """Test serializing basic InputRequirementBlueprint."""
- input_req = InputRequirementBlueprint(concept="Text")
- result = PipelexInterpreter.serialize_input_requirement(input_req)
-
- expected = {"concept": "Text"}
- assert result == expected
-
- def test_serialize_input_requirement_with_multiplicity_int(self):
- """Test serializing InputRequirementBlueprint with integer multiplicity."""
- input_req = InputRequirementBlueprint(concept="Text", multiplicity=3)
- result = PipelexInterpreter.serialize_input_requirement(input_req)
-
- expected = {"concept": "Text", "multiplicity": 3}
- assert result == expected
-
- def test_serialize_input_requirement_with_multiplicity_bool(self):
- """Test serializing InputRequirementBlueprint with boolean multiplicity."""
- input_req = InputRequirementBlueprint(concept="Text", multiplicity=True)
- result = PipelexInterpreter.serialize_input_requirement(input_req)
-
- expected = {"concept": "Text", "multiplicity": True}
- assert result == expected
-
-
-class TestSerializeInputs:
- """Test _serialize_inputs function."""
-
- def test_serialize_inputs_mixed_types(self):
- """Test serializing mixed input types."""
- inputs: Dict[str, str | InputRequirementBlueprint] = {
- "text": "Text",
- "data": InputRequirementBlueprint(concept="Data", multiplicity=2),
- "query": InputRequirementBlueprint(concept="Text"),
- }
- result = PipelexInterpreter.serialize_inputs(inputs)
-
- expected = {"text": "Text", "data": {"concept": "Data", "multiplicity": 2}, "query": {"concept": "Text"}}
- assert result == expected
-
- def test_serialize_inputs_empty(self):
- """Test serializing empty inputs."""
- inputs: Dict[str, Any] = {}
- result = PipelexInterpreter.serialize_inputs(inputs)
-
- assert result == {}
-
-
-class TestSerializeConceptStructure:
- """Test concept structure serialization functions."""
-
- def test_serialize_concept_structure_field_string(self):
- """Test serializing string concept structure field."""
- field_value = "Simple description"
- result = PipelexInterpreter.serialize_concept_structure_field(field_value)
-
- assert result == "Simple description"
-
- def test_serialize_concept_structure_field_blueprint(self):
- """Test serializing ConceptStructureBlueprint field."""
- field_value = ConceptStructureBlueprint(definition="A text field", type=ConceptStructureBlueprintFieldType.TEXT, required=True)
- result = PipelexInterpreter.serialize_concept_structure_field(field_value)
-
- expected = {"definition": "A text field", "required": True, "type": "text"}
- assert result == expected
-
- def test_serialize_concept_structure_field_blueprint_with_all_fields(self):
- """Test serializing ConceptStructureBlueprint with all optional fields."""
- field_value = ConceptStructureBlueprint(
- definition="A complex field",
- type=ConceptStructureBlueprintFieldType.DICT,
- key_type="string",
- value_type="integer",
- choices=["option1", "option2"],
- required=False,
- default_value={"default": 42},
- )
- result = PipelexInterpreter.serialize_concept_structure_field(field_value)
-
- expected = {
- "definition": "A complex field",
- "required": False,
- "type": "dict",
- "key_type": "string",
- "value_type": "integer",
- "choices": ["option1", "option2"],
- "default_value": {"default": 42},
- }
- assert result == expected
-
- def test_serialize_concept_structure_string(self):
- """Test serializing string concept structure."""
- structure = "Simple structure"
- result = PipelexInterpreter.serialize_concept_structure(structure)
-
- assert result == "Simple structure"
-
- def test_serialize_concept_structure_dict(self):
- """Test serializing dict concept structure."""
- structure = {
- "name": "Person's name",
- "age": ConceptStructureBlueprint(definition="Person's age", type=ConceptStructureBlueprintFieldType.INTEGER, required=True),
- }
- result = PipelexInterpreter.serialize_concept_structure(structure)
-
- expected = {
- "name": "Person's name",
- "age": {"definition": "Person's age", "required": True, "type": "integer"},
- }
- assert result == expected
-
-
-class TestSerializeSingleConcept:
- """Test _serialize_single_concept function."""
-
- def test_serialize_single_concept_string(self):
- """Test serializing string concept."""
- concept = "Simple concept definition"
- result = PipelexInterpreter.serialize_single_concept(concept)
-
- assert result == "Simple concept definition"
-
- def test_serialize_single_concept_simple_definition(self):
- """Test serializing ConceptBlueprint with just definition."""
- concept = ConceptBlueprint(definition="A simple concept")
- result = PipelexInterpreter.serialize_single_concept(concept)
-
- assert result == "A simple concept"
-
- def test_serialize_single_concept_with_refines(self):
- """Test serializing ConceptBlueprint with refines."""
- concept = ConceptBlueprint(definition="A specialized text", refines="Text")
- result = PipelexInterpreter.serialize_single_concept(concept)
-
- expected = {"definition": "A specialized text", "refines": "Text"}
- assert result == expected
-
- def test_serialize_single_concept_with_structure(self):
- """Test serializing ConceptBlueprint with structure."""
- concept = ConceptBlueprint(
- definition="A person",
- structure={
- "name": "Person's name",
- "age": ConceptStructureBlueprint(definition="Person's age", type=ConceptStructureBlueprintFieldType.INTEGER, required=True),
- },
- )
- result = PipelexInterpreter.serialize_single_concept(concept)
-
- expected = {
- "definition": "A person",
- "structure": {
- "name": "Person's name",
- "age": {"definition": "Person's age", "required": True, "type": "integer"},
- },
- }
- assert result == expected
-
-
-class TestSerializeConcepts:
- """Test _serialize_concepts function."""
-
- def test_serialize_concepts_empty(self):
- """Test serializing empty concepts."""
- result = PipelexInterpreter.serialize_concepts(None, "test_domain")
- assert result == {}
-
- def test_serialize_concepts_mixed(self):
- """Test serializing mixed concept types."""
- concepts: Dict[str, ConceptBlueprint | str] = {
- "SimpleText": "A simple text concept",
- "PersonInfo": ConceptBlueprint(definition="Information about a person", refines="Text"),
- "StructuredData": ConceptBlueprint(
- definition="Structured data",
- structure={
- "title": "Data title",
- "value": ConceptStructureBlueprint(definition="Data value", type=ConceptStructureBlueprintFieldType.NUMBER, required=True),
- },
- ),
- }
- result = PipelexInterpreter.serialize_concepts(concepts, "test_domain")
-
- expected = {
- "SimpleText": "A simple text concept",
- "PersonInfo": {"definition": "Information about a person", "refines": "Text"},
- "StructuredData": {
- "definition": "Structured data",
- "structure": {
- "title": "Data title",
- "value": {"definition": "Data value", "required": True, "type": "number"},
- },
- },
- }
- assert result == expected
-
-
-class TestSerializeSubPipe:
- """Test _serialize_sub_pipe function."""
-
- def test_serialize_sub_pipe_basic(self):
- """Test serializing basic sub pipe."""
- sub_pipe = SubPipeBlueprint(pipe="test_pipe", result="test_result")
- result = PipelexInterpreter.serialize_sub_pipe(sub_pipe)
-
- expected = {"pipe": "test_pipe", "result": "test_result"}
- assert result == expected
-
- def test_serialize_sub_pipe_with_nb_output_and_batch(self):
- """Test serializing sub pipe with nb_output and batch fields."""
- sub_pipe = SubPipeBlueprint(pipe="test_pipe", result="test_result", nb_output=3, batch_over="input_list", batch_as="current_item")
- result = PipelexInterpreter.serialize_sub_pipe(sub_pipe)
-
- expected = {"pipe": "test_pipe", "result": "test_result", "nb_output": 3, "batch_over": "input_list", "batch_as": "current_item"}
- assert result == expected
-
- def test_serialize_sub_pipe_with_multiple_output_and_batch(self):
- """Test serializing sub pipe with multiple_output and batch fields."""
- sub_pipe = SubPipeBlueprint(pipe="test_pipe", result="test_result", multiple_output=True, batch_over="input_list", batch_as="current_item")
- result = PipelexInterpreter.serialize_sub_pipe(sub_pipe)
-
- expected = {"pipe": "test_pipe", "result": "test_result", "multiple_output": True, "batch_over": "input_list", "batch_as": "current_item"}
- assert result == expected
-
- def test_serialize_sub_pipe_skip_defaults(self):
- """Test that default values are not included in serialization."""
- sub_pipe = SubPipeBlueprint(
- pipe="test_pipe",
- result="test_result",
- batch_over=False, # Default value
- batch_as=None, # Default value
- )
- result = PipelexInterpreter.serialize_sub_pipe(sub_pipe)
-
- expected = {"pipe": "test_pipe", "result": "test_result"}
- assert result == expected
-
-
-class TestSerializePipeBlueprints:
- """Test individual pipe blueprint serialization functions."""
-
- def test_serialize_llm_pipe_basic(self):
- """Test serializing basic PipeLLM blueprint."""
- pipe = PipeLLMBlueprint(type="PipeLLM", definition="Generate text using LLM", output="Text", prompt_template="Generate a story")
- result = PipelexInterpreter.serialize_llm_pipe(pipe, "test_domain")
-
- expected = {"type": "PipeLLM", "definition": "Generate text using LLM", "output": "Text", "prompt_template": "Generate a story"}
- assert result == expected
-
- def test_serialize_llm_pipe_with_inputs_and_options(self):
- """Test serializing PipeLLM blueprint with inputs and optional fields."""
- pipe = PipeLLMBlueprint(
- type="PipeLLM",
- definition="Extract information",
- inputs={"text": "Text", "topic": InputRequirementBlueprint(concept="Text", multiplicity=1)},
- output="ExtractedInfo",
- system_prompt="You are an expert extractor",
- prompt_template="Extract info about $topic from @text",
- nb_output=2,
- )
- result = PipelexInterpreter.serialize_llm_pipe(pipe, "test_domain")
-
- expected = {
- "type": "PipeLLM",
- "definition": "Extract information",
- "inputs": {"text": "Text", "topic": {"concept": "Text", "multiplicity": 1}},
- "output": "ExtractedInfo",
- "nb_output": 2,
- "system_prompt": "You are an expert extractor",
- "prompt_template": "Extract info about $topic from @text",
- }
- assert result == expected
-
- def test_serialize_ocr_pipe_basic(self):
- """Test serializing basic PipeOcr blueprint."""
- pipe = PipeOcrBlueprint(type="PipeOcr", definition="Extract text from PDF", output="Page", ocr="base_ocr_pypdfium2")
- result = PipelexInterpreter.serialize_ocr_pipe(pipe, "test_domain")
-
- expected = {"type": "PipeOcr", "definition": "Extract text from PDF", "output": "Page", "ocr": "base_ocr_pypdfium2"}
- assert result == expected
-
- def test_serialize_sequence_pipe_basic(self):
- """Test serializing basic PipeSequence blueprint."""
- pipe = PipeSequenceBlueprint(
- type="PipeSequence",
- definition="Process in sequence",
- output="Result",
- steps=[SubPipeBlueprint(pipe="step1", result="result1"), SubPipeBlueprint(pipe="step2", result="result2")],
- )
- result = PipelexInterpreter.serialize_sequence_pipe(pipe, "test_domain")
-
- expected = {
- "type": "PipeSequence",
- "definition": "Process in sequence",
- "output": "Result",
- "steps": [{"pipe": "step1", "result": "result1"}, {"pipe": "step2", "result": "result2"}],
- }
- assert result == expected
-
-
-class TestAddCommonPipeFields:
- """Test _add_common_pipe_fields helper function."""
-
- def test_add_common_pipe_fields_with_inputs(self):
- """Test adding common pipe fields when inputs exist."""
- result: Dict[str, Any] = {"type": "TestPipe"}
-
- # Mock pipe with inputs
- class MockPipe:
- def __init__(self):
- self.inputs = {"text": "Text", "data": InputRequirementBlueprint(concept="Data")}
-
- pipe = MockPipe()
- PipelexInterpreter.add_common_pipe_fields(result, pipe)
-
- expected = {"type": "TestPipe", "inputs": {"text": "Text", "data": {"concept": "Data"}}}
- assert result == expected
-
- def test_add_common_pipe_fields_without_inputs(self):
- """Test adding common pipe fields when no inputs exist."""
- result: Dict[str, Any] = {"type": "TestPipe"}
-
- # Mock pipe without inputs
- class MockPipe:
- def __init__(self):
- self.inputs = None
-
- pipe = MockPipe()
- PipelexInterpreter.add_common_pipe_fields(result, pipe)
-
- expected = {"type": "TestPipe"}
- assert result == expected
-
- def test_add_common_pipe_fields_no_inputs_attribute(self):
- """Test adding common pipe fields when pipe has no inputs attribute."""
- result: Dict[str, Any] = {"type": "TestPipe"}
-
- # Mock pipe without inputs attribute
- class MockPipe:
- pass
-
- pipe = MockPipe()
- PipelexInterpreter.add_common_pipe_fields(result, pipe)
-
- expected = {"type": "TestPipe"}
- assert result == expected
diff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_batch_plx.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_batch_plx.py
deleted file mode 100644
index f8c27a00c..000000000
--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_batch_plx.py
+++ /dev/null
@@ -1,54 +0,0 @@
-import pytest
-
-from pipelex.core.interpreter import PipelexInterpreter
-from pipelex.pipe_controllers.batch.pipe_batch_blueprint import PipeBatchBlueprint
-
-
-class TestPipelexInterpreterBatchPLX:
- """Test Batch pipe to PLX string conversion."""
-
- @pytest.mark.parametrize(
- "pipe_name,blueprint,expected_plx",
- [
- # Basic Batch pipe
- (
- "batch_process",
- PipeBatchBlueprint(
- type="PipeBatch",
- definition="Process items in batch",
- output="ProcessedData",
- branch_pipe_code="process_item",
- input_list_name="items",
- input_item_name="current_item",
- ),
- """[pipe.batch_process]
-type = "PipeBatch"
-definition = "Process items in batch"
-output = "ProcessedData"
-branch_pipe_code = "process_item"
-input_list_name = "items"
-input_item_name = "current_item\"""",
- ),
- # Batch pipe with inputs
- (
- "batch_with_inputs",
- PipeBatchBlueprint(
- type="PipeBatch",
- definition="Batch processing with inputs",
- inputs={"data_list": "Text", "config": "Text"},
- output="ProcessedData",
- branch_pipe_code="transform_item",
- ),
- """[pipe.batch_with_inputs]
-type = "PipeBatch"
-definition = "Batch processing with inputs"
-inputs = { data_list = "Text", config = "Text" }
-output = "ProcessedData"
-branch_pipe_code = "transform_item\"""",
- ),
- ],
- )
- def test_batch_pipe_to_plx_string(self, pipe_name: str, blueprint: PipeBatchBlueprint, expected_plx: str):
- """Test converting Batch pipe blueprint to PLX string."""
- result = PipelexInterpreter.batch_pipe_to_plx_string(pipe_name, blueprint, "test_domain")
- assert result == expected_plx
diff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_concept_plx.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_concept_plx.py
deleted file mode 100644
index 5aec6a2a3..000000000
--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_concept_plx.py
+++ /dev/null
@@ -1,97 +0,0 @@
-import pytest
-
-from pipelex.core.concepts.concept_blueprint import ConceptBlueprint, ConceptStructureBlueprint, ConceptStructureBlueprintFieldType
-from pipelex.core.interpreter import PipelexInterpreter
-
-
-class TestPipelexInterpreterConceptPLX:
- """Test concept to PLX string conversion."""
-
- @pytest.mark.parametrize(
- "concepts,expected_plx",
- [
- # Simple string concepts
- (
- {"ProcessedData": "Data that has been processed", "UserInfo": "Information about a user"},
- """[concept]
-ProcessedData = "Data that has been processed"
-UserInfo = "Information about a user\"""",
- ),
- # Single concept
- (
- {"Document": "A text document"},
- """[concept]
-Document = "A text document\"""",
- ),
- # ConceptBlueprint objects
- (
- {
- "ProcessedData": ConceptBlueprint(definition="Data that has been processed"),
- "UserInfo": ConceptBlueprint(definition="Information about a user"),
- },
- """[concept]
-ProcessedData = "Data that has been processed"
-UserInfo = "Information about a user\"""",
- ),
- # Complex ConceptBlueprint with structure
- (
- {
- "FeatureAnalysis": ConceptBlueprint(
- definition="Analysis of a photo's visual content and key features",
- structure={
- "dominant_feature": ConceptStructureBlueprint(
- definition="The most important or dominant feature in the image",
- type=ConceptStructureBlueprintFieldType.TEXT,
- required=True,
- ),
- "visual_elements": ConceptStructureBlueprint(
- definition="Description of key visual elements present",
- type=ConceptStructureBlueprintFieldType.TEXT,
- required=True,
- ),
- "composition": "Analysis of the image composition",
- "color_palette": "Description of the main colors in the image",
- "mood_atmosphere": "The overall mood or atmosphere of the image",
- },
- ),
- },
- """[concept.FeatureAnalysis]
-definition = "Analysis of a photo's visual content and key features"
-
-[concept.FeatureAnalysis.structure]
-dominant_feature = { type = "text", definition = "The most important or dominant feature in the image", required = true }
-visual_elements = { type = "text", definition = "Description of key visual elements present", required = true }
-composition = "Analysis of the image composition"
-color_palette = "Description of the main colors in the image"
-mood_atmosphere = "The overall mood or atmosphere of the image\"""",
- ),
- # ConceptBlueprint with refines
- (
- {
- "EnhancedText": ConceptBlueprint(
- definition="Enhanced text content with additional metadata",
- refines="Text",
- ),
- },
- """[concept.EnhancedText]
-definition = "Enhanced text content with additional metadata"
-refines = "Text\"""",
- ),
- # ConceptBlueprint with structure name
- (
- {
- "ProductInfo": ConceptBlueprint(
- definition="Detailed information about a product",
- structure="ProductStructure",
- ),
- },
- """[concept.ProductInfo]
-definition = "Detailed information about a product"
-structure = "ProductStructure\"""",
- ),
- ],
- )
- def test_concepts_to_plx_string(self, concepts: dict[str, ConceptBlueprint | str], expected_plx: str):
- """Test converting concepts dict to PLX string."""
- result = PipelexInterpreter.concepts_to_plx_string(concepts, "test_domain")
- assert result == expected_plx
diff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_condition_plx.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_condition_plx.py
deleted file mode 100644
index 216a8de5c..000000000
--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_condition_plx.py
+++ /dev/null
@@ -1,54 +0,0 @@
-import pytest
-
-from pipelex.core.interpreter import PipelexInterpreter
-from pipelex.pipe_controllers.condition.pipe_condition_blueprint import PipeConditionBlueprint, PipeConditionPipeMapBlueprint
-
-
-class TestPipelexInterpreterConditionPLX:
- """Test Condition pipe to PLX string conversion."""
-
- @pytest.mark.parametrize(
- "pipe_name,blueprint,expected_plx",
- [
- # Basic Condition pipe with expression
- (
- "conditional_process",
- PipeConditionBlueprint(
- type="PipeCondition",
- definition="Process based on condition",
- output="ProcessedData",
- pipe_map=PipeConditionPipeMapBlueprint({"small": "process_small", "large": "process_large"}),
- expression="input_data.category",
- ),
- """[pipe.conditional_process]
-type = "PipeCondition"
-definition = "Process based on condition"
-output = "ProcessedData"
-pipe_map = { small = "process_small", large = "process_large" }
-expression = "input_data.category\"""",
- ),
- # Condition pipe with expression template and inputs
- (
- "complex_condition",
- PipeConditionBlueprint(
- type="PipeCondition",
- definition="Complex conditional processing",
- inputs={"data": "Text", "threshold": "Text"},
- output="ProcessedData",
- pipe_map=PipeConditionPipeMapBlueprint({"low": "process_low", "medium": "process_medium", "high": "process_high"}),
- expression_template="{{ data.size if data.size > threshold else 'low' }}",
- ),
- """[pipe.complex_condition]
-type = "PipeCondition"
-definition = "Complex conditional processing"
-inputs = { data = "Text", threshold = "Text" }
-output = "ProcessedData"
-pipe_map = { low = "process_low", medium = "process_medium", high = "process_high" }
-expression_template = "{{ data.size if data.size > threshold else 'low' }}\"""",
- ),
- ],
- )
- def test_condition_pipe_to_plx_string(self, pipe_name: str, blueprint: PipeConditionBlueprint, expected_plx: str):
- """Test converting Condition pipe blueprint to PLX string."""
- result = PipelexInterpreter.condition_pipe_to_plx_string(pipe_name, blueprint, "test_domain")
- assert result == expected_plx
diff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_func_plx.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_func_plx.py
deleted file mode 100644
index 38e2cdb8f..000000000
--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_func_plx.py
+++ /dev/null
@@ -1,50 +0,0 @@
-import pytest
-
-from pipelex.core.interpreter import PipelexInterpreter
-from pipelex.pipe_operators.func.pipe_func_blueprint import PipeFuncBlueprint
-
-
-class TestPipelexInterpreterFuncPLX:
- """Test Func pipe to PLX string conversion."""
-
- @pytest.mark.parametrize(
- "pipe_name,blueprint,expected_plx",
- [
- # Basic Func pipe
- (
- "process_data",
- PipeFuncBlueprint(
- type="PipeFunc",
- definition="Process data with function",
- output="ProcessedData",
- function_name="process_data_function",
- ),
- """[pipe.process_data]
-type = "PipeFunc"
-definition = "Process data with function"
-output = "ProcessedData"
-function_name = "process_data_function\"""",
- ),
- # Func pipe with inputs
- (
- "transform_data",
- PipeFuncBlueprint(
- type="PipeFunc",
- definition="Transform input data",
- inputs={"data": "Text", "config": "Text"},
- output="TransformedData",
- function_name="transform_function",
- ),
- """[pipe.transform_data]
-type = "PipeFunc"
-definition = "Transform input data"
-inputs = { data = "Text", config = "Text" }
-output = "TransformedData"
-function_name = "transform_function\"""",
- ),
- ],
- )
- def test_func_pipe_to_plx_string(self, pipe_name: str, blueprint: PipeFuncBlueprint, expected_plx: str):
- """Test converting Func pipe blueprint to PLX string."""
- result = PipelexInterpreter.func_pipe_to_plx_string(pipe_name, blueprint, "test_domain")
- assert result == expected_plx
diff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_img_gen_plx.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_img_gen_plx.py
deleted file mode 100644
index 663f44187..000000000
--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_img_gen_plx.py
+++ /dev/null
@@ -1,53 +0,0 @@
-import pytest
-
-from pipelex.cogt.img_gen.img_gen_job_components import AspectRatio
-from pipelex.core.interpreter import PipelexInterpreter
-from pipelex.pipe_operators.img_gen.pipe_img_gen_blueprint import PipeImgGenBlueprint
-
-
-class TestPipelexInterpreterImgGenPLX:
- """Test ImgGen pipe to PLX string conversion."""
-
- @pytest.mark.parametrize(
- "pipe_name,blueprint,expected_plx",
- [
- # Basic ImgGen pipe
- (
- "generate_image",
- PipeImgGenBlueprint(
- type="PipeImgGen",
- definition="Generate an image from a prompt",
- output="Image",
- img_gen_prompt="A beautiful landscape",
- ),
- """[pipe.generate_image]
-type = "PipeImgGen"
-definition = "Generate an image from a prompt"
-output = "Image"
-img_gen_prompt = "A beautiful landscape\"""",
- ),
- # ImgGen pipe with some options
- (
- "generate_with_options",
- PipeImgGenBlueprint(
- type="PipeImgGen",
- definition="Generate image with options",
- output="Image",
- img_gen_prompt="A futuristic city",
- aspect_ratio=AspectRatio.LANDSCAPE_16_9,
- seed=12345,
- ),
- """[pipe.generate_with_options]
-type = "PipeImgGen"
-definition = "Generate image with options"
-output = "Image"
-img_gen_prompt = "A futuristic city"
-aspect_ratio = "landscape_16_9"
-seed = 12345""",
- ),
- ],
- )
- def test_img_gen_pipe_to_plx_string(self, pipe_name: str, blueprint: PipeImgGenBlueprint, expected_plx: str):
- """Test converting ImgGen pipe blueprint to PLX string."""
- result = PipelexInterpreter.img_gen_pipe_to_plx_string(pipe_name, blueprint, "test_domain")
- assert result == expected_plx
diff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_input_spec.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_input_spec.py
deleted file mode 100644
index 4a9f30432..000000000
--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_input_spec.py
+++ /dev/null
@@ -1,58 +0,0 @@
-from typing import Any
-
-import pytest
-
-from pipelex.core.interpreter import PipelexInterpreter
-from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
-
-
-class TestPipelexInterpreterInputSpec:
- """Test input serialization for both simple strings and InputRequirementBlueprint objects."""
-
- def test_inputs_to_plx_string_simple_strings(self):
- """Test converting simple string inputs to PLX format like { text = "Text", topic = "Text" }."""
- # Simple string inputs (like "Text")
- simple_inputs = {"text": "Text", "topic": "Text"}
-
- result = PipelexInterpreter.inputs_to_plx_string(simple_inputs)
-
- expected = '{ text = "Text", topic = "Text" }'
- assert result == expected
-
- def test_inputs_to_plx_string_input_requirement_blueprints(self):
- """Test converting InputRequirementBlueprint inputs to PLX format like { text = { concept = "Text", multiplicity = 1 } }."""
- # InputRequirementBlueprint inputs
- complex_inputs = {
- "text": InputRequirementBlueprint(concept="Text", multiplicity=1),
- "problem": InputRequirementBlueprint(concept="Text"), # multiplicity=None
- "documents": InputRequirementBlueprint(concept="Text", multiplicity=True),
- "query": InputRequirementBlueprint(concept="Text", multiplicity=False),
- }
-
- result = PipelexInterpreter.inputs_to_plx_string(complex_inputs)
-
- expected = (
- '{ text = { concept = "Text", multiplicity = 1 }, problem = { concept = "Text" }, '
- 'documents = { concept = "Text", multiplicity = true }, query = { concept = "Text", multiplicity = false } }'
- )
- assert result == expected
-
- @pytest.mark.parametrize(
- "input_req,expected",
- [
- # Test with just concept (multiplicity=None should not be included)
- (InputRequirementBlueprint(concept="Text"), {"concept": "Text"}),
- # Test with concept and multiplicity=1
- (InputRequirementBlueprint(concept="Text", multiplicity=1), {"concept": "Text", "multiplicity": 1}),
- # Test with concept and multiplicity=True
- (InputRequirementBlueprint(concept="Text", multiplicity=True), {"concept": "Text", "multiplicity": True}),
- # Test with concept and multiplicity=False
- (InputRequirementBlueprint(concept="Text", multiplicity=False), {"concept": "Text", "multiplicity": False}),
- # Test with different concept
- (InputRequirementBlueprint(concept="Image", multiplicity=5), {"concept": "Image", "multiplicity": 5}),
- ],
- )
- def test_serialize_input_requirement(self, input_req: InputRequirementBlueprint, expected: dict[str, Any]):
- """Test serializing a single InputRequirementBlueprint with various multiplicity values."""
- result = PipelexInterpreter.serialize_input_requirement(input_req)
- assert result == expected
diff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_jinja2_plx.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_jinja2_plx.py
deleted file mode 100644
index 290efeb5f..000000000
--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_jinja2_plx.py
+++ /dev/null
@@ -1,50 +0,0 @@
-import pytest
-
-from pipelex.core.interpreter import PipelexInterpreter
-from pipelex.pipe_operators.jinja2.pipe_jinja2_blueprint import PipeJinja2Blueprint
-
-
-class TestPipelexInterpreterJinja2PLX:
- """Test Jinja2 pipe to PLX string conversion."""
-
- @pytest.mark.parametrize(
- "pipe_name,blueprint,expected_plx",
- [
- # Basic Jinja2 pipe
- (
- "process_template",
- PipeJinja2Blueprint(
- type="PipeJinja2",
- definition="Process a Jinja2 template",
- output="Text",
- jinja2="Hello {{ name }}!",
- ),
- """[pipe.process_template]
-type = "PipeJinja2"
-definition = "Process a Jinja2 template"
-output = "Text"
-jinja2 = "Hello {{ name }}!\"""",
- ),
- # Jinja2 pipe with inputs and named template
- (
- "format_data",
- PipeJinja2Blueprint(
- type="PipeJinja2",
- definition="Format data with template",
- inputs={"data": "Text", "format": "Text"},
- output="Text",
- jinja2_name="data_formatter",
- ),
- """[pipe.format_data]
-type = "PipeJinja2"
-definition = "Format data with template"
-inputs = { data = "Text", format = "Text" }
-output = "Text"
-jinja2_name = "data_formatter\"""",
- ),
- ],
- )
- def test_jinja2_pipe_to_plx_string(self, pipe_name: str, blueprint: PipeJinja2Blueprint, expected_plx: str):
- """Test converting Jinja2 pipe blueprint to PLX string."""
- result = PipelexInterpreter.jinja2_pipe_to_plx_string(pipe_name, blueprint, "test_domain")
- assert result == expected_plx
diff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_jinja2_spacing.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_jinja2_spacing.py
deleted file mode 100644
index 15e1b9d8c..000000000
--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_jinja2_spacing.py
+++ /dev/null
@@ -1,80 +0,0 @@
-import pytest
-
-from pipelex.core.interpreter import PipelexInterpreter
-from pipelex.pipe_operators.jinja2.pipe_jinja2_blueprint import PipeJinja2Blueprint
-
-
-class TestPipelexInterpreterJinja2PipePLX:
- """Test the jinja2_pipe_to_plx_string function directly."""
-
- @pytest.mark.parametrize(
- "pipe_name,blueprint,expected_plx",
- [
- # Basic Jinja2 pipe
- (
- "greeting_pipe",
- PipeJinja2Blueprint(
- type="PipeJinja2",
- definition="Process a Jinja2 template",
- output="Text",
- jinja2="Hello {{ name }}!",
- ),
- """[pipe.greeting_pipe]
-type = "PipeJinja2"
-definition = "Process a Jinja2 template"
-output = "Text"
-jinja2 = "Hello {{ name }}!\"""",
- ),
- # Jinja2 pipe with inputs
- (
- "template_pipe",
- PipeJinja2Blueprint(
- type="PipeJinja2",
- definition="Process template with inputs",
- inputs={"name": "Text", "greeting": "Text"},
- output="Text",
- jinja2="{{ greeting }} {{ name }}!",
- ),
- """[pipe.template_pipe]
-type = "PipeJinja2"
-definition = "Process template with inputs"
-inputs = { name = "Text", greeting = "Text" }
-output = "Text"
-jinja2 = "{{ greeting }} {{ name }}!\"""",
- ),
- # Jinja2 pipe with complex template
- (
- "complex_pipe",
- PipeJinja2Blueprint(
- type="PipeJinja2",
- definition="Process complex template",
- output="Text",
- jinja2="{% for item in items %}{{ item.name }}{% endfor %}",
- ),
- """[pipe.complex_pipe]
-type = "PipeJinja2"
-definition = "Process complex template"
-output = "Text"
-jinja2 = "{% for item in items %}{{ item.name }}{% endfor %}\"""",
- ),
- # Jinja2 pipe with jinja2_name instead of inline jinja2
- (
- "named_template_pipe",
- PipeJinja2Blueprint(
- type="PipeJinja2",
- definition="Use named template",
- output="Text",
- jinja2_name="greeting_template",
- ),
- """[pipe.named_template_pipe]
-type = "PipeJinja2"
-definition = "Use named template"
-output = "Text"
-jinja2_name = "greeting_template\"""",
- ),
- ],
- )
- def test_jinja2_pipe_to_plx_string(self, pipe_name: str, blueprint: PipeJinja2Blueprint, expected_plx: str):
- """Test converting PipeJinja2Blueprint to PLX string with various configurations."""
- result = PipelexInterpreter.jinja2_pipe_to_plx_string(pipe_name, blueprint, "test_domain")
- assert result == expected_plx
diff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_llm_plx.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_llm_plx.py
deleted file mode 100644
index 08d6641a7..000000000
--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_llm_plx.py
+++ /dev/null
@@ -1,94 +0,0 @@
-import pytest
-
-from pipelex.core.interpreter import PipelexInterpreter
-from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
-from pipelex.pipe_operators.llm.pipe_llm_blueprint import PipeLLMBlueprint
-
-
-class TestPipelexInterpreterLLMPLX:
- """Test LLM pipe to PLX string conversion."""
-
- @pytest.mark.parametrize(
- "pipe_name,blueprint,expected_plx",
- [
- # Simple LLM pipe with string inputs
- (
- "simple_llm",
- PipeLLMBlueprint(
- type="PipeLLM",
- definition="Simple text generation",
- inputs={"text": "Text", "topic": "Text"},
- output="Text",
- prompt_template="Generate text about $topic based on: @text",
- ),
- """[pipe.simple_llm]
-type = "PipeLLM"
-definition = "Simple text generation"
-inputs = { text = "Text", topic = "Text" }
-output = "Text"
-prompt_template = "Generate text about $topic based on: @text\"""",
- ),
- # LLM pipe with complex inputs (InputRequirementBlueprint)
- (
- "complex_llm",
- PipeLLMBlueprint(
- type="PipeLLM",
- definition="Extract structured information",
- inputs={
- "documents": InputRequirementBlueprint(concept="Text", multiplicity=True),
- "query": InputRequirementBlueprint(concept="Text", multiplicity=False),
- },
- output="DocumentSummary",
- prompt_template="Extract information from documents: @documents based on query: $query",
- ),
- """[pipe.complex_llm]
-type = "PipeLLM"
-definition = "Extract structured information"
-inputs = { documents = { concept = "Text", multiplicity = true }, query = { concept = "Text", multiplicity = false } }
-output = "DocumentSummary"
-prompt_template = "Extract information from documents: @documents based on query: $query\"""",
- ),
- # LLM pipe with system prompt
- (
- "system_prompt_llm",
- PipeLLMBlueprint(
- type="PipeLLM",
- definition="Analysis with system prompt",
- inputs={"content": "Text"},
- output="Text",
- system_prompt_template="You are an expert analyst with $expertise_level knowledge",
- prompt_template="Analyze this content: @content",
- ),
- """[pipe.system_prompt_llm]
-type = "PipeLLM"
-definition = "Analysis with system prompt"
-inputs = { content = "Text" }
-output = "Text"
-system_prompt_template = "You are an expert analyst with $expertise_level knowledge"
-prompt_template = "Analyze this content: @content\"""",
- ),
- # LLM pipe with nb_output
- (
- "multiple_output_llm",
- PipeLLMBlueprint(
- type="PipeLLM",
- definition="Generate multiple outputs",
- inputs={"topic": "Text"},
- output="Text",
- nb_output=3,
- prompt_template="Generate ideas about: $topic",
- ),
- """[pipe.multiple_output_llm]
-type = "PipeLLM"
-definition = "Generate multiple outputs"
-inputs = { topic = "Text" }
-output = "Text"
-nb_output = 3
-prompt_template = "Generate ideas about: $topic\"""",
- ),
- ],
- )
- def test_llm_pipe_to_plx_string(self, pipe_name: str, blueprint: PipeLLMBlueprint, expected_plx: str):
- """Test converting LLM pipe blueprint to PLX string."""
- result = PipelexInterpreter.llm_pipe_to_plx_string(pipe_name, blueprint, "test_domain")
- assert result == expected_plx
diff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_ocr_plx.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_ocr_plx.py
deleted file mode 100644
index 54d83bf8d..000000000
--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_ocr_plx.py
+++ /dev/null
@@ -1,50 +0,0 @@
-import pytest
-
-from pipelex.core.interpreter import PipelexInterpreter
-from pipelex.pipe_operators.ocr.pipe_ocr_blueprint import PipeOcrBlueprint
-
-
-class TestPipelexInterpreterOcrPLX:
- """Test OCR pipe to PLX string conversion."""
-
- @pytest.mark.parametrize(
- "pipe_name,blueprint,expected_plx",
- [
- # Basic OCR pipe
- (
- "extract_text",
- PipeOcrBlueprint(
- type="PipeOcr",
- definition="Extract text from document",
- output="Page",
- ocr="base_ocr_pypdfium2",
- ),
- """[pipe.extract_text]
-type = "PipeOcr"
-definition = "Extract text from document"
-output = "Page"
-ocr = "base_ocr_pypdfium2\"""",
- ),
- # OCR pipe with inputs
- (
- "extract_with_input",
- PipeOcrBlueprint(
- type="PipeOcr",
- definition="Extract text from PDF",
- inputs={"ocr_input": "PDF"},
- output="Page",
- ocr="base_ocr_pypdfium2",
- ),
- """[pipe.extract_with_input]
-type = "PipeOcr"
-definition = "Extract text from PDF"
-inputs = { ocr_input = "PDF" }
-output = "Page"
-ocr = "base_ocr_pypdfium2\"""",
- ),
- ],
- )
- def test_ocr_pipe_to_plx_string(self, pipe_name: str, blueprint: PipeOcrBlueprint, expected_plx: str):
- """Test converting OCR pipe blueprint to PLX string."""
- result = PipelexInterpreter.ocr_pipe_to_plx_string(pipe_name, blueprint, "test_domain")
- assert result == expected_plx
diff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_parallel_plx.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_parallel_plx.py
deleted file mode 100644
index e2a086d3c..000000000
--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_parallel_plx.py
+++ /dev/null
@@ -1,69 +0,0 @@
-import pytest
-
-from pipelex.core.interpreter import PipelexInterpreter
-from pipelex.pipe_controllers.parallel.pipe_parallel_blueprint import PipeParallelBlueprint
-from pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprint
-
-
-class TestPipelexInterpreterParallelPLX:
- """Test Parallel pipe to PLX string conversion."""
-
- @pytest.mark.parametrize(
- "pipe_name,blueprint,expected_plx",
- [
- # Basic Parallel pipe
- (
- "parallel_process",
- PipeParallelBlueprint(
- type="PipeParallel",
- definition="Process data in parallel",
- output="ProcessedData",
- parallels=[
- SubPipeBlueprint(pipe="process_a", result="result_a"),
- SubPipeBlueprint(pipe="process_b", result="result_b"),
- ],
- ),
- """[pipe.parallel_process]
-type = "PipeParallel"
-definition = "Process data in parallel"
-output = "ProcessedData"
-parallels = [
- { pipe = "process_a", result = "result_a" },
- { pipe = "process_b", result = "result_b" },
-]""",
- ),
- # Parallel pipe with inputs and options
- (
- "complex_parallel",
- PipeParallelBlueprint(
- type="PipeParallel",
- definition="Complex parallel processing",
- inputs={"data": "Text"},
- output="ProcessedData",
- parallels=[
- SubPipeBlueprint(pipe="analyze", result="analysis"),
- SubPipeBlueprint(pipe="summarize", result="summary"),
- SubPipeBlueprint(pipe="extract", result="extraction"),
- ],
- add_each_output=False,
- combined_output="MergedResults",
- ),
- """[pipe.complex_parallel]
-type = "PipeParallel"
-definition = "Complex parallel processing"
-inputs = { data = "Text" }
-output = "ProcessedData"
-parallels = [
- { pipe = "analyze", result = "analysis" },
- { pipe = "summarize", result = "summary" },
- { pipe = "extract", result = "extraction" },
-]
-add_each_output = false
-combined_output = "MergedResults\"""",
- ),
- ],
- )
- def test_parallel_pipe_to_plx_string(self, pipe_name: str, blueprint: PipeParallelBlueprint, expected_plx: str):
- """Test converting Parallel pipe blueprint to PLX string."""
- result = PipelexInterpreter.parallel_pipe_to_plx_string(pipe_name, blueprint, "test_domain")
- assert result == expected_plx
diff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_sequence_plx.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_sequence_plx.py
deleted file mode 100644
index 1c0f95bdf..000000000
--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_sequence_plx.py
+++ /dev/null
@@ -1,89 +0,0 @@
-"""Test PipelexInterpreter Sequence pipe to PLX string conversion."""
-
-import pytest
-
-from pipelex.core.interpreter import PipelexInterpreter
-from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
-from pipelex.pipe_controllers.sequence.pipe_sequence_blueprint import PipeSequenceBlueprint
-from pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprint
-
-
-class TestPipelexInterpreterSequencePLX:
- """Test Sequence pipe to PLX string conversion."""
-
- @pytest.mark.parametrize(
- "pipe_name,blueprint,expected_plx",
- [
- # Simple sequence pipe with 2 steps
- (
- "simple_sequence",
- PipeSequenceBlueprint(
- type="PipeSequence",
- definition="Process data in sequence",
- output="ProcessedData",
- steps=[
- SubPipeBlueprint(pipe="extract_info", result="extracted"),
- SubPipeBlueprint(pipe="summarize", result="final_summary"),
- ],
- ),
- """[pipe.simple_sequence]
-type = "PipeSequence"
-definition = "Process data in sequence"
-output = "ProcessedData"
-steps = [
- { pipe = "extract_info", result = "extracted" },
- { pipe = "summarize", result = "final_summary" },
-]""",
- ),
- # Sequence pipe with inputs and steps
- (
- "sequence_with_inputs",
- PipeSequenceBlueprint(
- type="PipeSequence",
- definition="Process data with inputs",
- inputs={"text": InputRequirementBlueprint(concept="Text", multiplicity=1), "topic": "Text"},
- output="ProcessedData",
- steps=[
- SubPipeBlueprint(pipe="extract_info", result="extracted"),
- SubPipeBlueprint(pipe="summarize", result="final_summary"),
- ],
- ),
- """[pipe.sequence_with_inputs]
-type = "PipeSequence"
-definition = "Process data with inputs"
-inputs = { text = { concept = "Text", multiplicity = 1 }, topic = "Text" }
-output = "ProcessedData"
-steps = [
- { pipe = "extract_info", result = "extracted" },
- { pipe = "summarize", result = "final_summary" },
-]""",
- ),
- # Sequence pipe with complex sub pipe options
- (
- "complex_sequence",
- PipeSequenceBlueprint(
- type="PipeSequence",
- definition="Complex sequence processing",
- output="ProcessedData",
- steps=[
- SubPipeBlueprint(pipe="step1", result="result1", nb_output=2),
- SubPipeBlueprint(pipe="step2", result="result2", multiple_output=True),
- SubPipeBlueprint(pipe="step3", result="result3", batch_over="items", batch_as="item"),
- ],
- ),
- """[pipe.complex_sequence]
-type = "PipeSequence"
-definition = "Complex sequence processing"
-output = "ProcessedData"
-steps = [
- { pipe = "step1", result = "result1", nb_output = 2 },
- { pipe = "step2", result = "result2", multiple_output = true },
- { pipe = "step3", result = "result3", batch_over = "items", batch_as = "item" },
-]""",
- ),
- ],
- )
- def test_sequence_pipe_to_plx_string(self, pipe_name: str, blueprint: PipeSequenceBlueprint, expected_plx: str):
- """Test converting Sequence pipe blueprint to PLX string."""
- result = PipelexInterpreter.sequence_pipe_to_plx_string(pipe_name, blueprint, "test_domain")
- assert result == expected_plx
diff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_serialize_sub_pipe.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_serialize_sub_pipe.py
deleted file mode 100644
index dac9c022e..000000000
--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_serialize_sub_pipe.py
+++ /dev/null
@@ -1,159 +0,0 @@
-"""Test PipelexInterpreter serialize_sub_pipe function."""
-
-from typing import Any, Dict
-
-import pytest
-
-from pipelex.core.interpreter import PipelexInterpreter
-from pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprint
-
-
-class TestPipelexInterpreterSerializeSubPipe:
- """Test the serialize_sub_pipe function directly."""
-
- @pytest.mark.parametrize(
- "sub_pipe_blueprint,expected_result",
- [
- # Basic sub pipe (step in sequence)
- (
- SubPipeBlueprint(
- pipe="step1",
- result="intermediate1",
- ),
- {
- "pipe": "step1",
- "result": "intermediate1",
- },
- ),
- # Sub pipe with nb_output
- (
- SubPipeBlueprint(
- pipe="generate_ideas",
- result="ideas",
- nb_output=3,
- ),
- {
- "pipe": "generate_ideas",
- "result": "ideas",
- "nb_output": 3,
- },
- ),
- # Sub pipe with multiple_output
- (
- SubPipeBlueprint(
- pipe="brainstorm",
- result="solutions",
- multiple_output=True,
- ),
- {
- "pipe": "brainstorm",
- "result": "solutions",
- "multiple_output": True,
- },
- ),
- # Sub pipe with batch_over and batch_as
- (
- SubPipeBlueprint(
- pipe="process_items",
- result="processed_items",
- batch_over="input_list",
- batch_as="current_item",
- ),
- {
- "pipe": "process_items",
- "result": "processed_items",
- "batch_over": "input_list",
- "batch_as": "current_item",
- },
- ),
- # Sub pipe with nb_output and batch fields (no multiple_output)
- (
- SubPipeBlueprint(
- pipe="complex_step",
- result="complex_result",
- nb_output=2,
- batch_over="items",
- batch_as="item",
- ),
- {
- "pipe": "complex_step",
- "result": "complex_result",
- "nb_output": 2,
- "batch_over": "items",
- "batch_as": "item",
- },
- ),
- # Sub pipe with batch_over=False (default, should not be included)
- (
- SubPipeBlueprint(
- pipe="simple_step",
- result="simple_result",
- batch_over=False, # This is the default and should not appear in output
- ),
- {
- "pipe": "simple_step",
- "result": "simple_result",
- # batch_over=False should not be included since it's the default
- },
- ),
- ],
- )
- def test_serialize_sub_pipe(self, sub_pipe_blueprint: SubPipeBlueprint, expected_result: Dict[str, Any]):
- """Test serializing SubPipeBlueprint with various configurations."""
- result = PipelexInterpreter.serialize_sub_pipe(sub_pipe_blueprint)
- assert result == expected_result
-
- @pytest.mark.parametrize(
- "sub_pipe_blueprint,expected_plx_string",
- [
- # Basic sub pipe (step in sequence)
- (
- SubPipeBlueprint(
- pipe="step1",
- result="intermediate1",
- ),
- '{ pipe = "step1", result = "intermediate1" }',
- ),
- # Sub pipe with nb_output
- (
- SubPipeBlueprint(
- pipe="generate_ideas",
- result="ideas",
- nb_output=3,
- ),
- '{ pipe = "generate_ideas", result = "ideas", nb_output = 3 }',
- ),
- # Sub pipe with multiple_output
- (
- SubPipeBlueprint(
- pipe="brainstorm",
- result="solutions",
- multiple_output=True,
- ),
- '{ pipe = "brainstorm", result = "solutions", multiple_output = true }',
- ),
- # Sub pipe with batch_over and batch_as
- (
- SubPipeBlueprint(
- pipe="process_items",
- result="processed_items",
- batch_over="input_list",
- batch_as="current_item",
- ),
- '{ pipe = "process_items", result = "processed_items", batch_over = "input_list", batch_as = "current_item" }',
- ),
- # Sub pipe with batch_over=False (default, should not be included)
- (
- SubPipeBlueprint(
- pipe="simple_step",
- result="simple_result",
- batch_over=False, # This is the default and should not appear in output
- ),
- '{ pipe = "simple_step", result = "simple_result" }',
- ),
- ],
- )
- def test_sub_pipe_to_plx_string(self, sub_pipe_blueprint: SubPipeBlueprint, expected_plx_string: str):
- """Test converting SubPipeBlueprint directly to PLX inline table string."""
- result = PipelexInterpreter.sub_pipe_to_plx_string(sub_pipe_blueprint)
- assert result == expected_plx_string
diff --git a/tests/unit/pipelex/core/test_data/__init__.py b/tests/unit/pipelex/core/test_data/__init__.py
index fcbd717bb..06273a548 100644
--- a/tests/unit/pipelex/core/test_data/__init__.py
+++ b/tests/unit/pipelex/core/test_data/__init__.py
@@ -26,7 +26,7 @@ class InterpreterTestCases:
"""Test cases for PipelexInterpreter with PLX content and expected blueprints."""
# Aggregate all valid test cases from organized modules
- VALID_TEST_CASES: ClassVar[List[Tuple[str, str, PipelexBundleBlueprint]]] = [
+ VALID_TEST_CASES: ClassVar[List[Tuple[str, str, PipelexBundleBlueprint]]] = [ # test_name,plx_content,blueprint
# Domain tests
*DOMAIN_TEST_CASES,
# Concept tests