Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
Refactor/epic library blueprint 5 (#224)
Thomas Hebrard0318270ed6c6
diff --git a/.cursor/rules/pipe-llm.mdc b/.cursor/rules/pipe-llm.mdcindex 0721d4968..3a740fbf6 100644--- a/.cursor/rules/pipe-llm.mdc+++ b/.cursor/rules/pipe-llm.mdc@@ -39,7 +39,7 @@ Extract person information from this text:Place your Pydantic models in `pipelex_libraries/pipelines/your_models.py`:```python-from pipelex.core.stuff_content import StructuredContent+from pipelex.core.stuffs.stuff_content import StructuredContentclass PersonInfo(StructuredContent): # The output models always have to be subclass of StructuredContentname: strdiff --git a/.cursor/rules/pipelex.mdc b/.cursor/rules/pipelex.mdcindex ae078a3f4..c98f83fca 100644--- a/.cursor/rules/pipelex.mdc+++ b/.cursor/rules/pipelex.mdc@@ -99,7 +99,7 @@ from datetime import datetimefrom typing import List, Optionalfrom pydantic import Field-from pipelex.core.stuff_content import StructuredContent+from pipelex.core.stuffs.stuff_content import StructuredContent# IMPORTANT: THE CLASS MUST BE A SUBCLASS OF StructuredContentclass YourModel(StructuredContent): # Always be a subclass of StructuredContentdiff --git a/.cursor/rules/pytest.mdc b/.cursor/rules/pytest.mdcindex fd5a199bd..09994dd3b 100644--- a/.cursor/rules/pytest.mdc+++ b/.cursor/rules/pytest.mdc@@ -69,16 +69,16 @@ Sometimes it can be convenient to access the test's name in its body, for instanimport pytestfrom pytest import FixtureRequestfrom pipelex import log, pretty_print-from pipelex.core.stuff_factory import StuffBlueprint, StuffFactory-from pipelex.core.working_memory_factory import WorkingMemoryFactory+from pipelex.core.stuffs.stuff_factory import StuffBlueprint, StuffFactory+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactoryfrom pipelex.hub import get_report_delegatefrom pipelex_libraries.pipelines.base_library.retrieve import RetrievedExcerptfrom pipelex.config_pipelex import get_configfrom pipelex.core.pipe import PipeAbstract, update_job_metadata_for_pipe-from pipelex.core.pipe_output import PipeOutput, PipeOutputType-from pipelex.core.pipe_run_params import PipeRunParams-from pipelex.core.pipe_run_params import PipeRunParams+from pipelex.core.pipes.pipe_output import PipeOutput, PipeOutputType+from pipelex.core.pipes.pipe_run_params import PipeRunParams+from pipelex.core.pipes.pipe_run_params import PipeRunParamsfrom pipelex.pipe_works.pipe_router_protocol import PipeRouterProtocol```diff --git a/.cursor/rules/structures.mdc b/.cursor/rules/structures.mdcindex c01303cbe..534a97deb 100644--- a/.cursor/rules/structures.mdc+++ b/.cursor/rules/structures.mdc@@ -23,7 +23,7 @@ from datetime import datetimefrom typing import List, Optionalfrom pydantic import BaseModel, Field-from pipelex.core.stuff_content import StructuredContent+from pipelex.core.stuffs.stuff_content import StructuredContentclass YourModel(StructuredContent): # Always be a subclass of StructuredContent# Required fieldsdiff --git a/.pipelex/create/build_pipelines.md b/.pipelex/create/build_pipelines.mdindex 1dc666313..717c735f7 100644--- a/.pipelex/create/build_pipelines.md+++ b/.pipelex/create/build_pipelines.md@@ -1,6 +1,6 @@# Building Pipelex Pipelines from Drafts-This guide explains how to translate a markdown pipeline draft into a formal `PipelineBlueprint` structure that can be validated and executed by Pipelex.+This guide explains how to translate a markdown pipeline draft into a formal `PipelexBundleBlueprint` structure that can be validated and executed by Pipelex.## Overviewdiff --git a/.pipelex/create/design_pipelines.md b/.pipelex/create/design_pipelines.mdindex 2a3fcfb80..9d560dafd 100644--- a/.pipelex/create/design_pipelines.md+++ b/.pipelex/create/design_pipelines.md@@ -6,7 +6,7 @@ This section explains how to design pipelines for Pipelex.- Plan the pipeline in natural language (goal, inputs, steps, outputs).-- Translate the plan into a `PipelineBlueprint` object graph: domain, concepts, pipes.+- Translate the plan into a `PipelexBundleBlueprint` object graph: domain, concepts, pipes.- Define controllers first (`PipeSequence`, `PipeParallel`, `PipeCondition`), then operators (`PipeLLM`, `PipeOcr`, etc.).@@ -16,9 +16,9 @@ This section explains how to design pipelines for Pipelex.- Choose appropriate native or structured output concepts.-### Root blueprint object: PipelineBlueprint+### Root blueprint object: PipelexBundleBlueprint-`PipelineBlueprint` fields:+`PipelexBundleBlueprint` fields:- `domain: str` — required. Single word or snake_case code for the domain.diff --git a/README.md b/README.mdindex 280af8968..ed965193a 100644--- a/README.md+++ b/README.md@@ -22,7 +22,7 @@ Stop reinventing AI workflows from scratch. With Pipelex, your proven methods bealt="PyPI – latest release"><br/><br/>- <a href="https://go.pipelex.com/discord"><img src="https://img.shields.io/badge/Discord-5865F2?logo=discord&logoColor=white" alt="Discord"></a>+ <!-- <a href="https://go.pipelex.com/discord"><img src="https://img.shields.io/badge/Discord-5865F2?logo=discord&logoColor=white" alt="Discord"></a> --><a href="https://www.youtube.com/@PipelexAI"><img src="https://img.shields.io/badge/YouTube-FF0000?logo=youtube&logoColor=white" alt="YouTube"></a><a href="https://pipelex.com"><img src="https://img.shields.io/badge/Homepage-03bb95?logo=google-chrome&logoColor=white&style=flat" alt="Website"></a><a href="https://github.com/Pipelex/pipelex-cookbook"><img src="https://img.shields.io/badge/Cookbook-5a0dad?logo=github&logoColor=white&style=flat" alt="Cookbook"></a>@@ -276,11 +276,11 @@ class optimize_tweet_sequence sub_a;We welcome contributions! Please see our [Contributing Guidelines](CONTRIBUTING.md) for details on how to get started, including development setup and testing information.-## 👥 Join the Community+<!-- ## 👥 Join the Community-Join our vibrant Discord community to connect with other developers, share your experiences, and get help with your Pipelex projects!+Join our vibrant Discord community to connect with other developers, share your experiences, and get help with your Pipelex projects! -->-[](https://go.pipelex.com/discord)+<!-- [](https://go.pipelex.com/discord) -->## 💬 Supportdiff --git a/docs/pages/build-reliable-ai-workflows-with-pipelex/define_your_concepts.md b/docs/pages/build-reliable-ai-workflows-with-pipelex/define_your_concepts.mdindex 0beec74a0..fcc7e34e8 100644--- a/docs/pages/build-reliable-ai-workflows-with-pipelex/define_your_concepts.md+++ b/docs/pages/build-reliable-ai-workflows-with-pipelex/define_your_concepts.md@@ -78,7 +78,7 @@ For each concept that needs structured output, create a corresponding Python clafrom datetime import datetimefrom typing import List, Optionalfrom pydantic import Field-from pipelex.core.stuff_content import StructuredContent+from pipelex.core.stuffs.stuff_content import StructuredContentclass Invoice(StructuredContent):invoice_number: str@@ -99,7 +99,7 @@ Use Pydantic's validation features to ensure data quality:```pythonfrom pydantic import field_validator-from pipelex.core.stuff_content import StructuredContent+from pipelex.core.stuffs.stuff_content import StructuredContentclass Employee(StructuredContent):name: str@@ -129,7 +129,7 @@ Not all data is always available. Use Optional fields with sensible defaults:```pythonfrom typing import Optionalfrom datetime import datetime-from pipelex.core.stuff_content import StructuredContent+from pipelex.core.stuffs.stuff_content import StructuredContentclass Meeting(StructuredContent):title: str@@ -158,7 +158,7 @@ Department = "An organizational unit within a company" # No Python model => tex```python# pipelex_libraries/pipelines/hr.py-from pipelex.core.stuff_content import StructuredContent+from pipelex.core.stuffs.stuff_content import StructuredContentfrom datetime import datetimefrom typing import List, Optional@@ -266,7 +266,7 @@ While TOML refinement is primarily semantic, you can mirror these relationships```python# pipelex_libraries/pipelines/content.py-from pipelex.core.stuff_content import StructuredContent+from pipelex.core.stuffs.stuff_content import StructuredContentfrom datetime import datetimefrom typing import Optional, Listdiff --git a/docs/pages/build-reliable-ai-workflows-with-pipelex/kick-off-a-knowledge-pipeline-project.md b/docs/pages/build-reliable-ai-workflows-with-pipelex/kick-off-a-knowledge-pipeline-project.mdindex afeca0bff..60c06d921 100644--- a/docs/pages/build-reliable-ai-workflows-with-pipelex/kick-off-a-knowledge-pipeline-project.md+++ b/docs/pages/build-reliable-ai-workflows-with-pipelex/kick-off-a-knowledge-pipeline-project.md@@ -69,7 +69,7 @@ A domain consists of:2. **Python Models** (`.py`)```python- from pipelex.core.stuff_content import StructuredContent+ from pipelex.core.stuffs.stuff_content import StructuredContentfrom pydantic import Fieldfrom typing import Listfrom datetime import date@@ -110,7 +110,7 @@ A domain consists of:When using a domain in your code, you refer to concepts with `domain.ConceptName`:```python-from pipelex.core.stuff_factory import StuffFactory+from pipelex.core.stuffs.stuff_factory import StuffFactory# The concept_code combines domain and concept namesinvoice_stuff = StuffFactory.make_stuff(diff --git a/docs/pages/installation/index.md b/docs/pages/installation/index.mdindex 183ed10ef..906ae2301 100644--- a/docs/pages/installation/index.md+++ b/docs/pages/installation/index.md@@ -61,4 +61,5 @@ Learn more about pipelex_libraries in our [Libraries documentation](../build-rel- `pipelex init config`: This cli command will create a `pipelex.toml` file at the root of the project, with basic configuration. This configuration file gathers all configuration for feature flags, logging, cost reporting, and so on... Learn more in our [Configuration documentation](../configuration/index.md)-💡 _Any troubles? Have a look at our [Cookbook](https://github.com/Pipelex/pipelex-cookbook)! and come ask for help on our [Discord](https://go.pipelex.com/discord)_+💡 _Any troubles? Have a look at our [Cookbook](https://github.com/Pipelex/pipelex-cookbook)!+<!-- and come ask for help on our [Discord](https://go.pipelex.com/discord)_ -->diff --git a/docs/pages/quick-start/index.md b/docs/pages/quick-start/index.mdindex 4da126632..fa3d92815 100644--- a/docs/pages/quick-start/index.md+++ b/docs/pages/quick-start/index.md@@ -104,7 +104,7 @@ Using the [Pydantic BaseModel](https://docs.pydantic.dev/latest/) syntax, define`pipelex_libraries/pipelines/characters.py````python-from pipelex.core.stuff_content import StructuredContent+from pipelex.core.stuffs.stuff_content import StructuredContent# Define the structure of your output here# This class must inherit from StructuredContent@@ -160,7 +160,7 @@ We want to extract structured information from the description field. Thus we ha```python# pipelex_libraries/pipelines/character_model.py-from pipelex.core.stuff_content import StructuredContent+from pipelex.core.stuffs.stuff_content import StructuredContent# input classclass Character(StructuredContent):@@ -208,8 +208,8 @@ Learn more about how we use Jinja in the [PipeLLM documentation](../build-reliab```pythonimport asyncio-from pipelex.core.stuff_factory import StuffFactory-from pipelex.core.working_memory_factory import WorkingMemoryFactory+from pipelex.core.stuffs.stuff_factory import StuffFactory+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactoryfrom pipelex.pipelex import Pipelexfrom pipelex.pipeline.execute import execute_pipelinediff --git a/mkdocs.yml b/mkdocs.ymlindex 357eb6e0e..c56c4d6c3 100644--- a/mkdocs.yml+++ b/mkdocs.yml@@ -44,9 +44,9 @@ extra:- icon: fontawesome/brands/twitterlink: https://x.com/PipelexAIname: Pipelex on X- - icon: fontawesome/brands/discord- link: https://go.pipelex.com/discord- name: Pipelex on Discord+ # - icon: fontawesome/brands/discord+ # link: https://go.pipelex.com/discord+ # name: Pipelex on Discordgenerator: falseplugins:diff --git a/pipelex/cli/commands/init_cmd.py b/pipelex/cli/commands/init_cmd.pyindex 22a942796..b7441b739 100644--- a/pipelex/cli/commands/init_cmd.py+++ b/pipelex/cli/commands/init_cmd.py@@ -14,16 +14,16 @@ from pipelex.tools.config.manager import config_managerdef do_init_libraries(directory: str = ".", overwrite: bool = False) -> None:"""Initialize pipelex libraries in a pipelex_libraries folder in the specified directory."""try:- target_path = os.path.join(directory, "pipelex_libraries")+ target_dir = os.path.join(directory, "pipelex_libraries")os.makedirs(directory, exist_ok=True)- library_config = LibraryConfig(config_folder_path=target_path)+ library_config = LibraryConfig(config_dir_path=target_dir)library_config.export_libraries(overwrite=overwrite)if overwrite:- typer.echo(f"✅ Successfully initialized pipelex libraries at '{target_path}' (all files overwritten)")+ typer.echo(f"✅ Successfully initialized pipelex libraries at '{target_dir}' (all files overwritten)")else:- typer.echo(f"✅ Successfully initialized pipelex libraries at '{target_path}' (only created non-existing files)")+ typer.echo(f"✅ Successfully initialized pipelex libraries at '{target_dir}' (only created non-existing files)")except Exception as exc:raise PipelexCLIError(f"Failed to initialize libraries at '{directory}': {exc}") from excdiff --git a/pipelex/cli/commands/validate_cmd.py b/pipelex/cli/commands/validate_cmd.pyindex b56ee1cc9..3458d83b2 100644--- a/pipelex/cli/commands/validate_cmd.py+++ b/pipelex/cli/commands/validate_cmd.py@@ -4,18 +4,12 @@ import asynciofrom typing import Annotatedimport typer-from pydantic import ValidationErrorfrom pipelex import logfrom pipelex.cli.commands.common import is_pipelex_libraries_folder-from pipelex.create.validate_blueprint import validate_blueprint-from pipelex.exceptions import PipelexCLIErrorfrom pipelex.hub import get_pipeline_tracker-from pipelex.libraries.pipeline_blueprint import PipelineBlueprintfrom pipelex.pipe_works.pipe_dry import dry_run_all_pipes, dry_run_single_pipefrom pipelex.pipelex import Pipelex-from pipelex.tools.misc.toml_utils import load_toml_from_path-from pipelex.tools.typing.pydantic_utils import format_pydantic_validation_errordef do_validate_all_libraries_and_dry_run(relative_config_folder_path: str = "./pipelex_libraries") -> None:@@ -48,32 +42,6 @@ def do_dry_run_pipe(pipe_code: str, relative_config_folder_path: str = "./pipeleraise typer.Exit(1)-async def do_validate_blueprint_toml_file(blueprint_path: str, is_error_fixing_enabled: bool) -> None:- """Validate an already generated pipeline blueprint from a TOML file.-- Args:- blueprint_path: Path to the blueprint TOML file to validate-- Raises:- PipelexCLIError: If validation fails- ValidationError: If the TOML structure doesn't match PipelineBlueprint- """- # Load the TOML file- log.info(f"Loading blueprint from '{blueprint_path}'...")- blueprint_dict = load_toml_from_path(path=blueprint_path)-- # Validate and create PipelineBlueprint model- try:- blueprint = PipelineBlueprint.model_validate(blueprint_dict)- except ValidationError as exc:- error_msg = format_pydantic_validation_error(exc=exc)- raise PipelexCLIError(f"Invalid blueprint structure in '{blueprint_path}': {error_msg}") from exc-- log.info(f"Blueprint loaded successfully: domain='{blueprint.domain}'")-- await validate_blueprint(blueprint=blueprint, is_error_fixing_enabled=is_error_fixing_enabled)--# Typer group for validation commandsvalidate_app = typer.Typer(help="Validation and dry-run commands")@@ -95,34 +63,3 @@ def dry_run_pipe_cmd(] = "./pipelex_libraries",) -> None:do_dry_run_pipe(pipe_code=pipe_code, relative_config_folder_path=relative_config_folder_path)---@validate_app.command("blueprint")-def validate_blueprint_cmd(- blueprint_path: Annotated[str, typer.Argument(help="Path to the blueprint TOML file to validate")],- relative_config_folder_path: Annotated[- str, typer.Option("--config-folder-path", "-c", help="Relative path to the config folder path")- ] = "./pipelex_libraries",- is_error_fixing_enabled: Annotated[bool, typer.Option("--fix/--no-fix", "-f/-F", help="Enable or disable error fixing during validation")] = True,-) -> None:- """Validate an already generated pipeline blueprint TOML file.-- This command loads the TOML file, validates it against the `PipelineBlueprint` model,- loads the defined pipes into memory, and performs a dry-run validation of those pipes.- """- if not is_pipelex_libraries_folder(relative_config_folder_path):- typer.echo(f"❌ No pipelex libraries folder found at '{relative_config_folder_path}'")- typer.echo("To create a pipelex libraries folder, run: pipelex init-libraries")- raise typer.Exit(1)-- # Initialize Pipelex (loads libraries and pipes)- Pipelex.make(relative_config_folder_path=relative_config_folder_path, from_file=False)-- # Run validation- asyncio.run(- do_validate_blueprint_toml_file(- blueprint_path=blueprint_path,- is_error_fixing_enabled=is_error_fixing_enabled,- )- )- log.info("Blueprint validation completed successfully.")diff --git a/pipelex/client/api_serializer.py b/pipelex/client/api_serializer.pyindex 8de04fea0..7390dbcb8 100644--- a/pipelex/client/api_serializer.py+++ b/pipelex/client/api_serializer.py@@ -5,10 +5,10 @@ from pathlib import Pathfrom typing import Any, Dict, List, Optional, castfrom pipelex.client.protocol import CompactMemory-from pipelex.core.concept_native import NativeConcept-from pipelex.core.pipe_output import PipeOutput-from pipelex.core.stuff_content import TextContent-from pipelex.core.working_memory import WorkingMemory+from pipelex.core.concepts.concept_native import NativeConcept+from pipelex.core.memory.working_memory import WorkingMemory+from pipelex.core.pipes.pipe_output import PipeOutput+from pipelex.core.stuffs.stuff_content import TextContentclass ApiSerializer:diff --git a/pipelex/client/client.py b/pipelex/client/client.pyindex f5217315d..7da3480ed 100644--- a/pipelex/client/client.py+++ b/pipelex/client/client.py@@ -6,9 +6,9 @@ from typing_extensions import overridefrom pipelex.client.pipeline_request_factory import PipelineRequestFactoryfrom pipelex.client.pipeline_response_factory import PipelineResponseFactoryfrom pipelex.client.protocol import CompactMemory, PipelexProtocol, PipelineResponse-from pipelex.core.pipe_run_params import PipeOutputMultiplicity-from pipelex.core.working_memory import WorkingMemory-from pipelex.core.working_memory_factory import WorkingMemoryFactory+from pipelex.core.memory.working_memory import WorkingMemory+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory+from pipelex.core.pipes.pipe_run_params import PipeOutputMultiplicityfrom pipelex.exceptions import ClientAuthenticationErrorfrom pipelex.tools.environment import get_required_envdiff --git a/pipelex/client/pipeline_request_factory.py b/pipelex/client/pipeline_request_factory.pyindex 6c198361b..51b732806 100644--- a/pipelex/client/pipeline_request_factory.py+++ b/pipelex/client/pipeline_request_factory.py@@ -2,8 +2,8 @@ from typing import Any, Dict, Optionalfrom pipelex.client.api_serializer import ApiSerializerfrom pipelex.client.protocol import COMPACT_MEMORY_KEY, PipelineRequest-from pipelex.core.pipe_run_params import PipeOutputMultiplicity-from pipelex.core.working_memory import WorkingMemory+from pipelex.core.memory.working_memory import WorkingMemory+from pipelex.core.pipes.pipe_run_params import PipeOutputMultiplicityclass PipelineRequestFactory:diff --git a/pipelex/client/pipeline_response_factory.py b/pipelex/client/pipeline_response_factory.pyindex 5014fc153..f0e70db84 100644--- a/pipelex/client/pipeline_response_factory.py+++ b/pipelex/client/pipeline_response_factory.py@@ -2,7 +2,7 @@ from typing import Any, Dict, Optionalfrom pipelex.client.api_serializer import ApiSerializerfrom pipelex.client.protocol import PipelineResponse, PipelineState-from pipelex.core.pipe_output import PipeOutput+from pipelex.core.pipes.pipe_output import PipeOutputclass PipelineResponseFactory:diff --git a/pipelex/client/protocol.py b/pipelex/client/protocol.pyindex 7dcf84246..cddce3300 100644--- a/pipelex/client/protocol.py+++ b/pipelex/client/protocol.py@@ -4,9 +4,9 @@ from typing import Any, Dict, List, Optional, Protocolfrom pydantic import BaseModelfrom typing_extensions import runtime_checkable-from pipelex.core.pipe_run_params import PipeOutputMultiplicity-from pipelex.core.stuff_content import StuffContent-from pipelex.core.working_memory import WorkingMemory+from pipelex.core.memory.working_memory import WorkingMemory+from pipelex.core.pipes.pipe_run_params import PipeOutputMultiplicity+from pipelex.core.stuffs.stuff_content import StuffContentfrom pipelex.types import StrEnumStuffContentOrData = Dict[str, Any] | StuffContent | List[Any] | strdiff --git a/pipelex/cogt/content_generation/content_generator.py b/pipelex/cogt/content_generation/content_generator.pyindex e5e4e088c..da1353f12 100644--- a/pipelex/cogt/content_generation/content_generator.py+++ b/pipelex/cogt/content_generation/content_generator.py@@ -46,7 +46,7 @@ class ContentGenerator(ContentGeneratorProtocol):llm_prompt_for_text: LLMPrompt,) -> str:log.verbose(f"{self.__class__.__name__} make_llm_text: {llm_prompt_for_text}")- log.info(f"llm_setting_main: {llm_setting_main}")+ log.verbose(f"llm_setting_main: {llm_setting_main}")llm_assignment = LLMAssignment(job_metadata=job_metadata,llm_setting=llm_setting_main,diff --git a/pipelex/cogt/llm/llm_models/llm_model_library.py b/pipelex/cogt/llm/llm_models/llm_model_library.pyindex 589668427..0511c190a 100644--- a/pipelex/cogt/llm/llm_models/llm_model_library.py+++ b/pipelex/cogt/llm/llm_models/llm_model_library.py@@ -35,8 +35,8 @@ class LLMModelLibrary(LLMModelProviderAbstract, RootModel[LLMModelLibraryRoot]):library_config: ClassVar[LibraryConfig]@classmethod- def make_empty(cls, config_folder_path: str) -> "LLMModelLibrary":- cls.library_config = LibraryConfig(config_folder_path=config_folder_path)+ def make_empty(cls, config_dir_path: str) -> "LLMModelLibrary":+ cls.library_config = LibraryConfig(config_dir_path=config_dir_path)return cls()@override@@ -69,7 +69,7 @@ class LLMModelLibrary(LLMModelProviderAbstract, RootModel[LLMModelLibraryRoot]):@classmethoddef load_llm_model_library_dict(cls) -> LLMModelLibraryDict:- libraries_path = cls.library_config.llm_integrations_path+ libraries_path = cls.library_config.llm_integrations_dir_pathif not os.path.exists(libraries_path):raise LLMModelLibraryError(f"LLM model library path `{libraries_path}` not found. Please run `pipelex init-libraries` to create it.")llm_library: LLMModelLibraryDict = {}diff --git a/pipelex/core/bundles/pipelex_bundle.py b/pipelex/core/bundles/pipelex_bundle.pynew file mode 100644index 000000000..d5d56a6a9--- /dev/null+++ b/pipelex/core/bundles/pipelex_bundle.py@@ -0,0 +1,15 @@+from typing import Dict++from pydantic import BaseModel++from pipelex.core.concepts.concept import Concept+from pipelex.core.domains.domain import Domain+from pipelex.core.pipes.pipe_abstract import PipeAbstract+++class PipelexBundle(BaseModel):+ """Complete bundle of a pipelex bundle."""++ domain: Domain+ concepts: Dict[str, Concept]+ pipes: Dict[str, PipeAbstract]diff --git a/pipelex/core/bundles/pipelex_bundle_blueprint.py b/pipelex/core/bundles/pipelex_bundle_blueprint.pynew file mode 100644index 000000000..1b7dfb705--- /dev/null+++ b/pipelex/core/bundles/pipelex_bundle_blueprint.py@@ -0,0 +1,45 @@+from typing import Annotated, Dict, Optional, Union++from pydantic import BaseModel, Field++from pipelex.core.concepts.concept_blueprint import ConceptBlueprint+from pipelex.pipe_controllers.pipe_batch_factory import PipeBatchBlueprint+from pipelex.pipe_controllers.pipe_condition_factory import PipeConditionBlueprint+from pipelex.pipe_controllers.pipe_parallel_factory import PipeParallelBlueprint+from pipelex.pipe_controllers.pipe_sequence_factory import PipeSequenceBlueprint+from pipelex.pipe_operators.pipe_func_factory import PipeFuncBlueprint+from pipelex.pipe_operators.pipe_img_gen_factory import PipeImgGenBlueprint+from pipelex.pipe_operators.pipe_jinja2_factory import PipeJinja2Blueprint+from pipelex.pipe_operators.pipe_llm_factory import PipeLLMBlueprint+from pipelex.pipe_operators.pipe_ocr_factory import PipeOcrBlueprint++PipeBlueprintUnion = Annotated[+ Union[+ # Pipe operators+ PipeFuncBlueprint,+ PipeImgGenBlueprint,+ PipeJinja2Blueprint,+ PipeLLMBlueprint,+ PipeOcrBlueprint,+ # Pipe controllers+ PipeBatchBlueprint,+ PipeConditionBlueprint,+ PipeParallelBlueprint,+ PipeSequenceBlueprint,+ ],+ Field(discriminator="type"),+]+++class PipelexBundleBlueprint(BaseModel):+ """Complete blueprint of a pipelex bundle TOML definition."""++ domain: str+ definition: Optional[str] = None+ system_prompt: Optional[str] = None+ system_prompt_to_structure: Optional[str] = None+ prompt_template_to_structure: Optional[str] = None++ concept: Optional[Dict[str, ConceptBlueprint | str]] = Field(default_factory=dict)++ pipe: Optional[Dict[str, PipeBlueprintUnion]] = Field(default_factory=dict)diff --git a/pipelex/core/bundles/pipelex_bundle_factory.py b/pipelex/core/bundles/pipelex_bundle_factory.pynew file mode 100644index 000000000..bf45adbf6--- /dev/null+++ b/pipelex/core/bundles/pipelex_bundle_factory.py@@ -0,0 +1,47 @@+from typing import Dict++from pydantic import BaseModel++from pipelex.core.bundles.pipelex_bundle import PipelexBundle+from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint+from pipelex.core.concepts.concept import Concept+from pipelex.core.concepts.concept_blueprint import ConceptBlueprint+from pipelex.core.concepts.concept_factory import ConceptFactory+from pipelex.core.domains.domain import Domain+from pipelex.core.pipes.pipe_abstract import PipeAbstract+from pipelex.core.pipes.pipe_factory import PipeFactory+++class PipelexBundleFactory(BaseModel):+ @classmethod+ def make_from_blueprint(cls, blueprint: PipelexBundleBlueprint) -> PipelexBundle:+ """Make a PipelexBundle from a PipelexBundleBlueprint."""+ domain = Domain(+ code=blueprint.domain,+ definition=blueprint.definition,+ system_prompt=blueprint.system_prompt,+ system_prompt_to_structure=blueprint.system_prompt_to_structure,+ prompt_template_to_structure=blueprint.prompt_template_to_structure,+ )+ concepts: Dict[str, Concept] = {}+ if blueprint.concept is not None:+ for concept_name, concept_blueprint_or_str in blueprint.concept.items():+ if isinstance(concept_blueprint_or_str, ConceptBlueprint):+ concepts[concept_name] = ConceptFactory.make_concept_from_blueprint(+ domain=blueprint.domain, code=concept_name, concept_blueprint=concept_blueprint_or_str+ )+ else:+ # One line concepts+ concepts[concept_name] = ConceptFactory.make_concept_from_definition_str(+ domain_code=blueprint.domain, concept_str=concept_name, definition=concept_blueprint_or_str+ )++ pipes: Dict[str, PipeAbstract] = {}+ if blueprint.pipe is not None:+ for pipe_name, pipe_blueprint in blueprint.pipe.items():+ pipes[pipe_name] = PipeFactory.make_pipe_from_blueprint(+ domain_code=blueprint.domain,+ pipe_code=pipe_name,+ pipe_blueprint=pipe_blueprint,+ )+ return PipelexBundle(domain=domain, concepts=concepts, pipes=pipes)diff --git a/pipelex/core/concept.py b/pipelex/core/concepts/concept.pysimilarity index 89%rename from pipelex/core/concept.pyrename to pipelex/core/concepts/concept.pyindex e36b5e69b..4b98b9175 100644--- a/pipelex/core/concept.py+++ b/pipelex/core/concepts/concept.py@@ -6,10 +6,10 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validafrom typing_extensions import Selffrom pipelex import log-from pipelex.core.concept_native import NativeConcept-from pipelex.core.domain import SpecialDomain-from pipelex.core.stuff_content import StuffContent-from pipelex.exceptions import ConceptCodeError, ConceptDomainError, ConceptError, StructureClassError+from pipelex.core.concepts.concept_native import NativeConcept+from pipelex.core.domains.domain import SpecialDomain+from pipelex.core.stuffs.stuff_content import StuffContent+from pipelex.exceptions import ConceptCodeError, ConceptDomainError, ConceptErrorfrom pipelex.tools.misc.string_utils import pascal_case_to_sentence@@ -18,11 +18,12 @@ class Concept(BaseModel):code: strdomain: str- structure_class_name: strdefinition: str+ structure_class_name: strrefines: List[str] = Field(default_factory=list)@model_validator(mode="after")+ # TODO: Refacto, its not cleandef validate_code_domain(self) -> Self:if not Concept.concept_str_contains_domain(self.code):raise ConceptCodeError(f"Code must contain a dot (.) for concept with code '{self.code}' and domain '{self.domain}'")@@ -83,24 +84,6 @@ class Concept(BaseModel):return validated_refines- @field_validator("structure_class_name")- @classmethod- def validate_structure_class_name(cls, value: str) -> str:- if not cls.is_valid_structure_class(structure_class_name=value):- raise StructureClassError(f"Could not validate concept because structure_class_name '{value}' is not in class registry. ")- return value-- @classmethod- def is_valid_structure_class(cls, structure_class_name: str) -> bool:- # we get_class_registry directly from KajsonManager instead of pipelex hub to avoid circular import- if KajsonManager.get_class_registry().has_subclass(name=structure_class_name, base_class=StuffContent):- return True- else:- # we get_class_registry directly from KajsonManager instead of pipelex hub to avoid circular import- if KajsonManager.get_class_registry().has_class(name=structure_class_name):- log.warning(f"Concept class '{structure_class_name}' is registered but it's not a subclass of StuffContent")- return False-@classmethoddef extract_domain_and_concept_from_str(cls, concept_str: str) -> Tuple[str, str]:if "." in concept_str:@@ -138,3 +121,14 @@ class Concept(BaseModel):return domain == SpecialDomain.NATIVE.valueelse:return concept_str in NativeConcept.names()++ @classmethod+ def is_valid_structure_class(cls, structure_class_name: str) -> bool:+ # We get_class_registry directly from KajsonManager instead of pipelex hub to avoid circular import+ if KajsonManager.get_class_registry().has_subclass(name=structure_class_name, base_class=StuffContent):+ return True+ else:+ # We get_class_registry directly from KajsonManager instead of pipelex hub to avoid circular import+ if KajsonManager.get_class_registry().has_class(name=structure_class_name):+ log.warning(f"Concept class '{structure_class_name}' is registered but it's not a subclass of StuffContent")+ return Falsediff --git a/pipelex/core/concepts/concept_blueprint.py b/pipelex/core/concepts/concept_blueprint.pynew file mode 100644index 000000000..4012935f5--- /dev/null+++ b/pipelex/core/concepts/concept_blueprint.py@@ -0,0 +1,81 @@+from typing import Any, Dict, List, Optional, Union++from pydantic import BaseModel, ConfigDict, Field, model_validator+from typing_extensions import Self++from pipelex.exceptions import ConceptFactoryError+from pipelex.types import StrEnum+++class ConceptStructureBlueprintError(ConceptFactoryError):+ pass+++class ConceptStructureBlueprintFieldType(StrEnum):+ TEXT = "text"+ LIST = "list"+ DICT = "dict"+ INTEGER = "integer"+ BOOLEAN = "boolean"+ NUMBER = "number"+ DATE = "date"+++class ConceptStructureBlueprint(BaseModel):+ definition: str+ type: ConceptStructureBlueprintFieldType | None = None+ item_type: Optional[str] = None+ key_type: Optional[str] = None+ value_type: Optional[str] = None+ choices: Optional[List[str]] = Field(default_factory=list)+ required: Optional[bool] = Field(default=True)++ @model_validator(mode="after")+ def validate_structure_blueprint(self) -> Self:+ """Validate the structure blueprint according to type rules."""+ # If type is None (array), choices must not be None+ if self.type is None and not self.choices:+ raise ConceptStructureBlueprintError("When type is None (array), choices must not be empty")++ # If type is "dict", key_type and value_type must not be empty+ if self.type == ConceptStructureBlueprintFieldType.DICT:+ if not self.key_type:+ raise ConceptStructureBlueprintError(f"When type is '{ConceptStructureBlueprintFieldType.DICT}', key_type must not be empty")+ if not self.value_type:+ raise ConceptStructureBlueprintError(f"When type is '{ConceptStructureBlueprintFieldType.DICT}', value_type must not be empty")++ return self+++ConceptStructureBlueprintType = Union[str, ConceptStructureBlueprint]+++class ConceptBlueprint(BaseModel):+ model_config = ConfigDict(extra="forbid")++ definition: str+ structure: Optional[Union[str, Dict[str, ConceptStructureBlueprintType]]] = None+ refines: Optional[Union[str, List[str]]] = Field(default_factory=list)++ @model_validator(mode="after")+ def model_validate_blueprint(self) -> Self:+ return self++ def structure_to_field_def(self) -> Dict[str, Any]:+ if isinstance(self.structure, str):+ raise ValueError("structure_to_field_def can only be called when structure is a dict in the blueprint")++ if self.structure is None:+ return {}++ # Process the dict structure+ result: Dict[str, Any] = {}+ for key, value in self.structure.items():+ if isinstance(value, ConceptStructureBlueprint):+ # Use model_dump for ConceptStructureBlueprint instances+ result[key] = value.model_dump()+ else:+ # This shouldn't happen based on the type hints, but handle it gracefully+ result[key] = {"type": ConceptStructureBlueprintFieldType.TEXT, "definition": value}++ return resultdiff --git a/pipelex/core/concept_code_factory.py b/pipelex/core/concepts/concept_code_factory.pysimilarity index 69%rename from pipelex/core/concept_code_factory.pyrename to pipelex/core/concepts/concept_code_factory.pyindex 4a533b39f..dbfd0cd9c 100644--- a/pipelex/core/concept_code_factory.py+++ b/pipelex/core/concepts/concept_code_factory.py@@ -1,7 +1,7 @@from typing import Optional-from pipelex.core.concept import Concept-from pipelex.core.concept_native import NativeConcept+from pipelex.core.concepts.concept import Concept+from pipelex.core.concepts.concept_native import NativeConceptfrom pipelex.exceptions import ConceptFactoryError@@ -13,12 +13,14 @@ class ConceptCodeFactory:return f"{domain}.{code}"@classmethod- def make_concept_code_from_str(cls, concept_str: str, fallback_domain: Optional[str] = None) -> str:+ def make_concept_code_from_str(cls, concept_str: str, domain: Optional[str] = None, fallback_domain: Optional[str] = None) -> str:if Concept.concept_str_contains_domain(concept_str=concept_str):return concept_strelif concept_str in NativeConcept.names():native_concept = NativeConcept(concept_str)return native_concept.code+ elif domain:+ return cls.make_concept_code(domain=domain, code=concept_str)elif fallback_domain:return cls.make_concept_code(domain=fallback_domain, code=concept_str)else:diff --git a/pipelex/core/concept_factory.py b/pipelex/core/concepts/concept_factory.pysimilarity index 55%rename from pipelex/core/concept_factory.pyrename to pipelex/core/concepts/concept_factory.pyindex 19d1e6a53..240e8b5ec 100644--- a/pipelex/core/concept_factory.py+++ b/pipelex/core/concepts/concept_factory.py@@ -1,27 +1,18 @@-from inspect import getsource-from typing import Any, Dict, List, Optional, Type, Union+from typing import Any, Dict, List, Union-from pydantic import BaseModel, ConfigDict, Field, ValidationError+from pydantic import ValidationError-from pipelex.core.concept import Concept-from pipelex.core.concept_code_factory import ConceptCodeFactory-from pipelex.core.concept_native import NativeConcept, NativeConceptClass-from pipelex.core.domain import SpecialDomain-from pipelex.core.stuff_content import TextContent+from pipelex.core.concepts.concept import Concept+from pipelex.core.concepts.concept_blueprint import ConceptBlueprint+from pipelex.core.concepts.concept_code_factory import ConceptCodeFactory+from pipelex.core.concepts.concept_native import NativeConcept, NativeConceptClass+from pipelex.core.domains.domain import SpecialDomain+from pipelex.core.stuffs.stuff_content import TextContentfrom pipelex.create.structured_output_generator import generate_structured_output_from_inline_definitionfrom pipelex.exceptions import ConceptFactoryError, StructureClassErrorfrom pipelex.hub import get_class_registry-class ConceptBlueprint(BaseModel):- model_config = ConfigDict(extra="forbid")-- definition: str- structure: Optional[Union[str, Dict[str, Any]]] = None- refines: Union[str, List[str]] = Field(default_factory=list)- domain: Optional[str] = None--class ConceptFactory:@classmethoddef make_refines(cls, domain: str, refines: Union[str, List[str]]) -> List[str]:@@ -31,57 +22,10 @@ class ConceptFactory:concept_str_list = refinesnew_refines: List[str] = []for concept_str in concept_str_list:- concept_code = ConceptCodeFactory.make_concept_code_from_str(concept_str=concept_str, fallback_domain=domain)+ concept_code = ConceptCodeFactory.make_concept_code_from_str(concept_str=concept_str, domain=domain, fallback_domain=domain)new_refines.append(concept_code)return new_refines- @classmethod- def make_from_details_dict_if_possible(- cls,- domain: str,- code: str,- details_dict: Dict[str, Any],- ) -> Optional[Concept]:- if concept_definition := details_dict.pop("Concept", None):- details_dict["definition"] = concept_definition- details_dict["refines"] = ConceptFactory.make_refines(domain=domain, refines=details_dict.pop("refines", []))- concept_blueprint = ConceptBlueprint.model_validate(details_dict)- the_concept = ConceptFactory.make_concept_from_blueprint(domain=domain, code=code, concept_blueprint=concept_blueprint)- return the_concept- elif "definition" in details_dict:- # legacy format- details_dict["domain"] = domain- details_dict["refines"] = ConceptFactory.make_refines(domain=domain, refines=details_dict.pop("refines", []))- details_dict["code"] = ConceptCodeFactory.make_concept_code(domain, code)- try:- the_concept = Concept.model_validate(details_dict)- except ValidationError as exc:- raise ConceptFactoryError(f"Error validating concept: {exc}") from exc- return the_concept- else:- return None-- @classmethod- def make_from_details_dict(- cls,- domain_code: str,- code: str,- details_dict: Dict[str, Any],- ) -> Concept:- concept_definition = details_dict.pop("Concept", None)- if not concept_definition:- raise ConceptFactoryError(f"Concept '{code}' in domain '{domain_code}' has no definition")- details_dict["definition"] = concept_definition- details_dict["domain"] = domain_code- refines = ConceptFactory.make_refines(domain=domain_code, refines=details_dict.pop("refines", []))- if not refines and not details_dict.get("structure"):- # No structure? this refines Text- refines = [NativeConcept.TEXT.code]- details_dict["refines"] = refines- concept_blueprint = ConceptBlueprint.model_validate(details_dict)- the_concept = ConceptFactory.make_concept_from_blueprint(domain=domain_code, code=code, concept_blueprint=concept_blueprint)- return the_concept-@classmethoddef make_concept_from_definition_str(cls,@@ -122,23 +66,29 @@ class ConceptFactory:code: str,concept_blueprint: ConceptBlueprint,) -> Concept:- structure_class_name: str- if structure := concept_blueprint.structure:- if isinstance(structure, str):- # structure is set explicitly as a class name reference- if not Concept.is_valid_structure_class(structure_class_name=structure):+ current_refines: List[str]+ if concept_blueprint.refines:+ current_refines = cls.make_refines(domain=domain, refines=concept_blueprint.refines)+ else:+ current_refines = []++ structure_class_name: str = code+ if concept_blueprint.structure:+ if isinstance(concept_blueprint.structure, str):+ # Structure is defined inline - generate Python class dynamically+ if not Concept.is_valid_structure_class(structure_class_name=concept_blueprint.structure):raise StructureClassError(- f"Structure class '{structure}' set for concept '{code}' in domain '{domain}' is not a registered subclass of StuffContent"+ f"Structure class '{concept_blueprint.structure}' set for concept '{code}' in domain '{domain}' "+ "is not a registered subclass of StuffContent")- structure_class_name = structure+ structure_class_name = concept_blueprint.structureelse:- # structure is defined inline - generate Python class dynamically- structure_class_name = code+ # Structure is defined as a ConceptStructureBlueprinttry:# Generate Python class from inline definitionpython_code = generate_structured_output_from_inline_definition(class_name=code,- fields_def=structure, # type: ignore[arg-type]+ fields_def=concept_blueprint.structure_to_field_def(),enums=None, # TODO: Handle enums if needed in the future)@@ -153,19 +103,24 @@ class ConceptFactory:except Exception as exc:raise ConceptFactoryError(f"Error generating structure class for concept '{code}' in domain '{domain}': {exc}") from excelif Concept.is_valid_structure_class(structure_class_name=code):- # structure is set implicitly, by the concept's code- structure_class_name = code+ # No structure defined on the blueprint, but the concept code is a valid structure class+ passelse:- structure_class_name = TextContent.__name__-- refines_list = cls.make_refines(domain=domain, refines=concept_blueprint.refines)+ if concept_blueprint.refines:+ # Has a refining element+ pass+ else:+ # Fallback to Text structure+ structure_class_name = TextContent.__name__+ current_refines = [NativeConcept.TEXT.code]+ refines = cls.make_refines(domain=domain, refines=current_refines)return Concept(code=ConceptCodeFactory.make_concept_code(domain, code),domain=domain,definition=concept_blueprint.definition,structure_class_name=structure_class_name,- refines=refines_list,+ refines=refines,)@classmethod@@ -211,11 +166,3 @@ class ConceptFactory:continueconcepts.append(cls.make_native_concept(native_concept=native_concept))return concepts-- @classmethod- def get_concept_class_source_code(cls, concept_name: str, base_class: Type[Any]) -> str:- if not get_class_registry().has_class(concept_name):- raise RuntimeError(f"Class '{concept_name}' not found in registry")-- cls = get_class_registry().get_required_subclass(name=concept_name, base_class=base_class)- return getsource(cls)diff --git a/pipelex/core/concept_library.py b/pipelex/core/concepts/concept_library.pysimilarity index 94%rename from pipelex/core/concept_library.pyrename to pipelex/core/concepts/concept_library.pyindex 99b6f3ded..a79a4d8d1 100644--- a/pipelex/core/concept_library.py+++ b/pipelex/core/concepts/concept_library.py@@ -4,13 +4,13 @@ from pydantic import Field, RootModelfrom typing_extensions import overridefrom pipelex import log-from pipelex.core.concept import Concept-from pipelex.core.concept_code_factory import ConceptCodeFactory-from pipelex.core.concept_factory import ConceptFactory-from pipelex.core.concept_native import NativeConcept-from pipelex.core.concept_provider_abstract import ConceptProviderAbstract-from pipelex.core.domain import SpecialDomain-from pipelex.core.stuff_content import ImageContent+from pipelex.core.concepts.concept import Concept+from pipelex.core.concepts.concept_code_factory import ConceptCodeFactory+from pipelex.core.concepts.concept_factory import ConceptFactory+from pipelex.core.concepts.concept_native import NativeConcept+from pipelex.core.concepts.concept_provider_abstract import ConceptProviderAbstract+from pipelex.core.domains.domain import SpecialDomain+from pipelex.core.stuffs.stuff_content import ImageContentfrom pipelex.exceptions import ConceptLibraryConceptNotFoundError, ConceptLibraryErrorfrom pipelex.hub import get_class_registry@@ -138,14 +138,13 @@ class ConceptLibrary(RootModel[ConceptLibraryRoot], ConceptProviderAbstract):concept_code = f"{domain}.{concept_code}"else:concept_code = f"{SpecialDomain.NATIVE.value}.{concept_code}"-the_concept = self.get_concept(concept_code=concept_code)if not the_concept:if self.is_concept_implicit(concept_code=concept_code):# The implicit concept is obviously coming with a domain (the one it is used in)# TODO: replace this with a concept factory method make_implicit_conceptreturn ConceptFactory.make_concept_from_definition_str(- domain_code="implicit",+ domain_code=SpecialDomain.IMPLICIT,concept_str=Concept.extract_domain_and_concept_from_str(concept_str=concept_code)[1],definition=concept_code,)diff --git a/pipelex/core/concept_native.py b/pipelex/core/concepts/concept_native.pysimilarity index 98%rename from pipelex/core/concept_native.pyrename to pipelex/core/concepts/concept_native.pyindex efc50c3e2..b9c5a349c 100644--- a/pipelex/core/concept_native.py+++ b/pipelex/core/concepts/concept_native.py@@ -1,7 +1,7 @@from enum import Enumfrom typing import List-from pipelex.core.domain import SpecialDomain+from pipelex.core.domains.domain import SpecialDomainfrom pipelex.types import StrEnumdiff --git a/pipelex/core/concept_provider_abstract.py b/pipelex/core/concepts/concept_provider_abstract.pysimilarity index 96%rename from pipelex/core/concept_provider_abstract.pyrename to pipelex/core/concepts/concept_provider_abstract.pyindex 379b96490..5f2f33f4e 100644--- a/pipelex/core/concept_provider_abstract.py+++ b/pipelex/core/concepts/concept_provider_abstract.py@@ -1,7 +1,7 @@from abc import ABC, abstractmethodfrom typing import Any, Dict, List, Optional, Type-from pipelex.core.concept import Concept+from pipelex.core.concepts.concept import ConceptConceptLibraryRoot = Dict[str, Concept]diff --git a/pipelex/core/domain.py b/pipelex/core/domains/domain.pysimilarity index 100%rename from pipelex/core/domain.pyrename to pipelex/core/domains/domain.pydiff --git a/pipelex/core/domain_library.py b/pipelex/core/domains/domain_library.pysimilarity index 93%rename from pipelex/core/domain_library.pyrename to pipelex/core/domains/domain_library.pyindex e2b3b4005..372ab0536 100644--- a/pipelex/core/domain_library.py+++ b/pipelex/core/domains/domain_library.py@@ -4,8 +4,8 @@ from pydantic import RootModelfrom typing_extensions import overridefrom pipelex import log-from pipelex.core.domain import Domain-from pipelex.core.domain_provider_abstract import DomainProviderAbstract+from pipelex.core.domains.domain import Domain+from pipelex.core.domains.domain_provider_abstract import DomainProviderAbstractfrom pipelex.exceptions import DomainLibraryErrorfrom pipelex.tools.misc.file_utils import save_text_to_path@@ -23,7 +23,7 @@ class DomainLibrary(RootModel[DomainLibraryRoot], DomainProviderAbstract):def make_empty(cls):return cls(root={})- def add_domain_details(self, domain: Domain):+ def add_domain(self, domain: Domain):domain_code = domain.codeif existing_domain := self.root.get(domain_code):# merge the new domain with the existing onediff --git a/pipelex/core/domain_provider_abstract.py b/pipelex/core/domains/domain_provider_abstract.pysimilarity index 91%rename from pipelex/core/domain_provider_abstract.pyrename to pipelex/core/domains/domain_provider_abstract.pyindex 0711431f1..08e37bcba 100644--- a/pipelex/core/domain_provider_abstract.py+++ b/pipelex/core/domains/domain_provider_abstract.py@@ -1,7 +1,7 @@from abc import ABC, abstractmethodfrom typing import Dict, List, Optional-from pipelex.core.domain import Domain+from pipelex.core.domains.domain import Domainclass DomainProviderAbstract(ABC):diff --git a/pipelex/core/interpreter.py b/pipelex/core/interpreter.pynew file mode 100644index 000000000..a67c2b748--- /dev/null+++ b/pipelex/core/interpreter.py@@ -0,0 +1,457 @@+from pathlib import Path+from typing import Any, Dict, List, Optional++import toml+from pydantic import BaseModel, model_validator+from typing_extensions import Self++from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint+from pipelex.core.concepts.concept_blueprint import ConceptBlueprint+from pipelex.pipe_controllers.pipe_batch_factory import PipeBatchBlueprint+from pipelex.pipe_controllers.pipe_condition_factory import PipeConditionBlueprint+from pipelex.pipe_controllers.pipe_parallel_factory import PipeParallelBlueprint+from pipelex.pipe_controllers.pipe_sequence_factory import PipeSequenceBlueprint+from pipelex.pipe_operators.pipe_func_factory import PipeFuncBlueprint+from pipelex.pipe_operators.pipe_img_gen_factory import PipeImgGenBlueprint+from pipelex.pipe_operators.pipe_jinja2_factory import PipeJinja2Blueprint+from pipelex.pipe_operators.pipe_llm_factory import PipeLLMBlueprint+from pipelex.pipe_operators.pipe_ocr_factory import PipeOcrBlueprint+from pipelex.tools.misc.toml_utils import clean_trailing_whitespace, dict_to_toml, validate_toml_content, validate_toml_file+++class PipelexInterpreter(BaseModel):+ """TOML -> PipelexBundleBlueprint"""++ file_path: Optional[Path] = None+ file_content: Optional[str] = None++ @model_validator(mode="after")+ def check_file_path_or_file_content(self) -> Self:+ """Need to check if there is at least one of file_path or file_content"""+ if self.file_path is None and self.file_content is None:+ raise ValueError("Either file_path or file_content must be provided")+ return self++ @model_validator(mode="after")+ def validate_file_path(self) -> Self:+ if self.file_path:+ validate_toml_file(path=str(self.file_path))+ if self.file_content:+ validate_toml_content(content=self.file_content, file_path=str(self.file_path))+ return self++ @staticmethod+ def is_pipelex_file(file_path: Path) -> bool:+ """Check if a file is a valid Pipelex TOML file.++ Args:+ file_path: Path to the file to check++ Returns:+ True if the file is a Pipelex file, False otherwise++ Criteria:+ - Has .toml extension+ - Starts with "domain =" (ignoring leading whitespace)+ """+ # Check if it has .toml extension+ if file_path.suffix != ".toml":+ return False++ # Check if file exists+ if not file_path.exists() or not file_path.is_file():+ return False++ try:+ # Read the first few lines to check for "domain ="+ with open(file_path, "r", encoding="utf-8") as f:+ # Read first 100 characters (should be enough to find domain)+ content = f.read(100)+ # Remove leading whitespace and check if it starts with "domain ="+ stripped_content = content.lstrip()+ return stripped_content.startswith("domain =")+ except Exception:+ # If we can't read the file, it's not a valid Pipelex file+ return False++ def _load_toml_content(self) -> str:+ """Load TOML content from file_path or use file_content directly."""+ if self.file_path:+ try:+ with open(self.file_path, "r", encoding="utf-8") as file:+ file_content = file.read()++ # Clean trailing whitespace and write back if needed+ cleaned_content = clean_trailing_whitespace(file_content)+ if file_content != cleaned_content:+ with open(self.file_path, "w", encoding="utf-8") as file:+ file.write(cleaned_content)+ return cleaned_content++ return file_content++ except Exception as exc:+ raise ValueError(f"Failed to read TOML file '{self.file_path}': {exc}") from exc+ else:+ if self.file_content is None:+ raise ValueError("file_content must be provided if file_path is not provided")+ return self.file_content++ def _parse_toml_content(self, content: str) -> Dict[str, Any]:+ """Parse TOML content and return the dictionary."""+ try:+ return toml.loads(content)+ except toml.TomlDecodeError as exc:+ file_path_str = str(self.file_path) if self.file_path else "content"+ raise toml.TomlDecodeError(f"TOML parsing error in '{file_path_str}': {exc}", exc.doc, exc.pos) from exc++ def make_pipelex_bundle_blueprint(self) -> PipelexBundleBlueprint:+ """Make a PipelexBundleBlueprint from the file_path or file_content"""+ file_content = self._load_toml_content()+ toml_data = self._parse_toml_content(file_content)+ return PipelexBundleBlueprint.model_validate(toml_data)++ @staticmethod+ def make_toml_content(blueprint: PipelexBundleBlueprint) -> str:+ """Convert a PipelexBundleBlueprint to properly formatted TOML content."""+ toml_data: Dict[str, Any] = {}++ # Domain-level fields+ toml_data["domain"] = blueprint.domain+ if blueprint.definition:+ toml_data["definition"] = blueprint.definition+ if blueprint.system_prompt:+ toml_data["system_prompt"] = blueprint.system_prompt+ if blueprint.system_prompt_to_structure:+ toml_data["system_prompt_to_structure"] = blueprint.system_prompt_to_structure+ if blueprint.prompt_template_to_structure:+ toml_data["prompt_template_to_structure"] = blueprint.prompt_template_to_structure++ # Concepts section - always include if concepts is not None+ if blueprint.concept is not None:+ toml_data["concept"] = PipelexInterpreter._serialize_concepts(blueprint.concept, blueprint.domain)++ # Pipes section+ if blueprint.pipe:+ toml_data["pipe"] = PipelexInterpreter._serialize_pipes(blueprint.pipe, blueprint.domain)++ return dict_to_toml(toml_data)++ @staticmethod+ def _serialize_concepts(concepts: Optional[Dict[str, ConceptBlueprint | str]], domain: str) -> Dict[str, Any]:+ """Serialize concepts section with domain context."""+ result: Dict[str, Any] = {}+ if concepts is None:+ return {}++ for concept_name, concept_blueprint in concepts.items():+ if isinstance(concept_blueprint, str):+ # Simple string concept+ result[concept_name] = concept_blueprint+ else:+ # Complex ConceptBlueprint+ if hasattr(concept_blueprint, "structure") and concept_blueprint.structure:+ # Structured concept - create nested structure+ concept_data: Dict[str, Any] = {}+ if concept_blueprint.definition:+ concept_data["definition"] = concept_blueprint.definition+ if concept_blueprint.structure:+ concept_data["structure"] = {}+ if isinstance(concept_blueprint.structure, str):+ concept_data["structure"] = concept_blueprint.structure+ else:+ for field_name, field_value in concept_blueprint.structure.items():+ if isinstance(field_value, str):+ concept_data["structure"][field_name] = field_value+ else:+ # ConceptStructureBlueprint+ field_data: Dict[str, Any] = {}+ field_data["type"] = field_value.type+ field_data["definition"] = field_value.definition+ field_data["required"] = field_value.required+ concept_data["structure"][field_name] = field_data+ result[concept_name] = concept_data+ elif hasattr(concept_blueprint, "refines") and concept_blueprint.refines:+ # Concept with refines+ concept_data = {}+ if concept_blueprint.definition:+ concept_data["definition"] = concept_blueprint.definition+ concept_data["refines"] = concept_blueprint.refines+ result[concept_name] = concept_data+ else:+ # Simple concept with just definition+ result[concept_name] = concept_blueprint.definition if concept_blueprint.definition else f"Concept {concept_name}"+ return result++ @staticmethod+ def _serialize_pipes(pipes: Dict[str, Any], domain: str) -> Dict[str, Any]:+ """Serialize pipes section with domain context."""+ result: Dict[str, Any] = {}+ for pipe_name, pipe_blueprint in pipes.items():+ result[pipe_name] = PipelexInterpreter._serialize_pipe(pipe_blueprint, domain)+ return result++ @staticmethod+ def _serialize_pipe(pipe_blueprint: Any, domain: str) -> Dict[str, Any]:+ """Serialize a single pipe blueprint with domain context."""++ if isinstance(pipe_blueprint, PipeLLMBlueprint):+ return PipelexInterpreter._serialize_llm_pipe(pipe_blueprint, domain)+ elif isinstance(pipe_blueprint, PipeOcrBlueprint):+ return PipelexInterpreter._serialize_ocr_pipe(pipe_blueprint, domain)+ elif isinstance(pipe_blueprint, PipeFuncBlueprint):+ return PipelexInterpreter._serialize_func_pipe(pipe_blueprint, domain)+ elif isinstance(pipe_blueprint, PipeImgGenBlueprint):+ return PipelexInterpreter._serialize_img_gen_pipe(pipe_blueprint, domain)+ elif isinstance(pipe_blueprint, PipeJinja2Blueprint):+ return PipelexInterpreter._serialize_jinja2_pipe(pipe_blueprint, domain)+ elif isinstance(pipe_blueprint, PipeSequenceBlueprint):+ return PipelexInterpreter._serialize_sequence_pipe(pipe_blueprint, domain)+ elif isinstance(pipe_blueprint, PipeConditionBlueprint):+ return PipelexInterpreter._serialize_condition_pipe(pipe_blueprint, domain)+ elif isinstance(pipe_blueprint, PipeParallelBlueprint):+ return PipelexInterpreter._serialize_parallel_pipe(pipe_blueprint, domain)+ elif isinstance(pipe_blueprint, PipeBatchBlueprint):+ return PipelexInterpreter._serialize_batch_pipe(pipe_blueprint, domain)+ else:+ raise ValueError(f"Unknown pipe blueprint type: {type(pipe_blueprint)}")++ @staticmethod+ def _serialize_llm_pipe(pipe: PipeLLMBlueprint, domain: str) -> Dict[str, Any]:+ """Serialize PipeLLM blueprint."""+ result: Dict[str, Any] = {+ "type": pipe.type,+ "definition": pipe.definition,+ "output": pipe.output,+ }++ # Add optional fields only if they have values+ if pipe.inputs:+ result["inputs"] = PipelexInterpreter._serialize_inputs(pipe.inputs)+ if pipe.prompt_template:+ result["prompt_template"] = pipe.prompt_template+ if pipe.system_prompt:+ result["system_prompt"] = pipe.system_prompt++ return result++ @staticmethod+ def _serialize_ocr_pipe(pipe: PipeOcrBlueprint, domain: str) -> Dict[str, Any]:+ """Serialize PipeOcr blueprint."""+ result: Dict[str, Any] = {+ "type": pipe.type,+ "definition": pipe.definition,+ "output": pipe.output,+ }++ # Add optional fields only if they have values+ if pipe.inputs:+ result["inputs"] = PipelexInterpreter._serialize_inputs(pipe.inputs)++ return result++ @staticmethod+ def _serialize_func_pipe(pipe: PipeFuncBlueprint, domain: str) -> Dict[str, Any]:+ """Serialize PipeFunc blueprint."""+ result: Dict[str, Any] = {+ "type": pipe.type,+ "definition": pipe.definition,+ "output": pipe.output,+ "function_name": pipe.function_name,+ }++ # Add optional fields only if they have values+ if pipe.inputs:+ result["inputs"] = PipelexInterpreter._serialize_inputs(pipe.inputs)++ return result++ @staticmethod+ def _serialize_img_gen_pipe(pipe: PipeImgGenBlueprint, domain: str) -> Dict[str, Any]:+ """Serialize PipeImgGen blueprint."""+ result: Dict[str, Any] = {+ "type": pipe.type,+ "definition": pipe.definition,+ "output": pipe.output,+ }++ # Add optional fields only if they have values+ if pipe.inputs:+ result["inputs"] = PipelexInterpreter._serialize_inputs(pipe.inputs)+ if pipe.img_gen_prompt:+ result["img_gen_prompt"] = pipe.img_gen_prompt+ if pipe.imgg_handle:+ result["imgg_handle"] = pipe.imgg_handle+ if pipe.aspect_ratio:+ result["aspect_ratio"] = pipe.aspect_ratio+ if pipe.quality:+ result["quality"] = pipe.quality+ if pipe.nb_steps:+ result["nb_steps"] = pipe.nb_steps+ if pipe.guidance_scale:+ result["guidance_scale"] = pipe.guidance_scale+ if pipe.is_moderated is not None:+ result["is_moderated"] = pipe.is_moderated+ if pipe.safety_tolerance:+ result["safety_tolerance"] = pipe.safety_tolerance+ if pipe.is_raw is not None:+ result["is_raw"] = pipe.is_raw+ if pipe.seed:+ result["seed"] = pipe.seed+ if pipe.nb_output:+ result["nb_output"] = pipe.nb_output++ return result++ @staticmethod+ def _serialize_jinja2_pipe(pipe: PipeJinja2Blueprint, domain: str) -> Dict[str, Any]:+ """Serialize PipeJinja2 blueprint."""+ result: Dict[str, Any] = {+ "type": pipe.type,+ "definition": pipe.definition,+ "output": pipe.output,+ }++ # Add optional fields only if they have values+ if pipe.inputs:+ result["inputs"] = PipelexInterpreter._serialize_inputs(pipe.inputs)+ if pipe.jinja2_name:+ result["jinja2_name"] = pipe.jinja2_name+ if pipe.jinja2:+ result["jinja2"] = pipe.jinja2+ if pipe.prompting_style:+ result["prompting_style"] = pipe.prompting_style+ # Only include template_category if it's not the default value+ if pipe.template_category and pipe.template_category.value != "llm_prompt":+ result["template_category"] = pipe.template_category++ return result++ @staticmethod+ def _serialize_sequence_pipe(pipe: PipeSequenceBlueprint, domain: str) -> Dict[str, Any]:+ """Serialize PipeSequence blueprint."""+ result: Dict[str, Any] = {+ "type": pipe.type,+ "definition": pipe.definition,+ "output": pipe.output,+ }++ # Add optional fields only if they have values+ if pipe.inputs:+ result["inputs"] = PipelexInterpreter._serialize_inputs(pipe.inputs)+ if pipe.steps:+ # Serialize steps, but only include non-default values+ serialized_steps: List[Dict[str, Any]] = []+ for step in pipe.steps:+ step_data: Dict[str, Any] = {+ "pipe": step.pipe,+ "result": step.result,+ }+ # Only include optional fields if they have non-default values+ if step.nb_output is not None:+ step_data["nb_output"] = step.nb_output+ if step.multiple_output is not None:+ step_data["multiple_output"] = step.multiple_output+ if step.batch_over is not False: # Only include if not default False+ step_data["batch_over"] = step.batch_over+ if step.batch_as is not None:+ step_data["batch_as"] = step.batch_as+ serialized_steps.append(step_data)+ result["steps"] = serialized_steps++ return result++ @staticmethod+ def _serialize_condition_pipe(pipe: PipeConditionBlueprint, domain: str) -> Dict[str, Any]:+ """Serialize PipeCondition blueprint."""+ result: Dict[str, Any] = {+ "type": pipe.type,+ "definition": pipe.definition,+ "output": pipe.output,+ "pipe_map": pipe.pipe_map,+ }++ # Add optional fields only if they have values+ if pipe.inputs:+ result["inputs"] = PipelexInterpreter._serialize_inputs(pipe.inputs)+ if pipe.expression_template:+ result["expression_template"] = pipe.expression_template+ if pipe.expression:+ result["expression"] = pipe.expression+ if pipe.default_pipe_code:+ result["default_pipe_code"] = pipe.default_pipe_code+ if pipe.add_alias_from_expression_to:+ result["add_alias_from_expression_to"] = pipe.add_alias_from_expression_to++ return result++ @staticmethod+ def _serialize_parallel_pipe(pipe: PipeParallelBlueprint, domain: str) -> Dict[str, Any]:+ """Serialize PipeParallel blueprint."""+ result: Dict[str, Any] = {+ "type": pipe.type,+ "definition": pipe.definition,+ "output": pipe.output,+ }++ # Add optional fields only if they have values+ if pipe.inputs:+ result["inputs"] = PipelexInterpreter._serialize_inputs(pipe.inputs)+ if pipe.parallels:+ # Serialize parallels, but only include non-default values+ serialized_parallels: List[Dict[str, Any]] = []+ for parallel in pipe.parallels:+ parallel_data: Dict[str, Any] = {+ "pipe": parallel.pipe,+ "result": parallel.result,+ }+ # Only include optional fields if they have non-default values+ if parallel.nb_output is not None:+ parallel_data["nb_output"] = parallel.nb_output+ if parallel.multiple_output is not None:+ parallel_data["multiple_output"] = parallel.multiple_output+ if parallel.batch_over is not False: # Only include if not default False+ parallel_data["batch_over"] = parallel.batch_over+ if parallel.batch_as is not None:+ parallel_data["batch_as"] = parallel.batch_as+ serialized_parallels.append(parallel_data)+ result["parallels"] = serialized_parallels+ if pipe.add_each_output is not True: # Only include if not default True+ result["add_each_output"] = pipe.add_each_output+ if pipe.combined_output:+ result["combined_output"] = pipe.combined_output++ return result++ @staticmethod+ def _serialize_batch_pipe(pipe: PipeBatchBlueprint, domain: str) -> Dict[str, Any]:+ """Serialize PipeBatch blueprint."""+ result: Dict[str, Any] = {+ "type": pipe.type,+ "definition": pipe.definition,+ "output": pipe.output,+ "branch_pipe_code": pipe.branch_pipe_code,+ }++ # Add optional fields only if they have values+ if pipe.inputs:+ result["inputs"] = PipelexInterpreter._serialize_inputs(pipe.inputs)+ if pipe.input_list_name:+ result["input_list_name"] = pipe.input_list_name+ if pipe.input_item_name:+ result["input_item_name"] = pipe.input_item_name++ return result++ @staticmethod+ def _serialize_inputs(inputs: Dict[str, Any]) -> Dict[str, str]:+ """Convert InputRequirementBlueprint objects to concept code strings for TOML serialization."""+ result: Dict[str, str] = {}+ for key, value in inputs.items():+ if hasattr(value, "concept_code"):+ # InputRequirementBlueprint object+ result[key] = value.concept_code+ else:+ # Already a string+ result[key] = str(value)+ return resultdiff --git a/pipelex/core/working_memory.py b/pipelex/core/memory/working_memory.pysimilarity index 98%rename from pipelex/core/working_memory.pyrename to pipelex/core/memory/working_memory.pyindex dbee03c2e..d7c54fcca 100644--- a/pipelex/core/working_memory.py+++ b/pipelex/core/memory/working_memory.py@@ -5,10 +5,10 @@ from pydantic import BaseModel, Field, model_validatorfrom typing_extensions import Selffrom pipelex import log, pretty_print-from pipelex.core.concept_native import NativeConcept-from pipelex.core.stuff import Stuff-from pipelex.core.stuff_artefact import StuffArtefact-from pipelex.core.stuff_content import (+from pipelex.core.concepts.concept_native import NativeConcept+from pipelex.core.stuffs.stuff import Stuff+from pipelex.core.stuffs.stuff_artefact import StuffArtefact+from pipelex.core.stuffs.stuff_content import (HtmlContent,ImageContent,ListContent,diff --git a/pipelex/core/working_memory_factory.py b/pipelex/core/memory/working_memory_factory.pysimilarity index 95%rename from pipelex/core/working_memory_factory.pyrename to pipelex/core/memory/working_memory_factory.pyindex e42ab4a1b..0cda7d5cf 100644--- a/pipelex/core/working_memory_factory.py+++ b/pipelex/core/memory/working_memory_factory.py@@ -6,12 +6,12 @@ from pydantic import BaseModelfrom pipelex import logfrom pipelex.client.protocol import CompactMemory, ImplicitMemory-from pipelex.core.concept_native import NativeConcept-from pipelex.core.pipe_input_spec import TypedNamedInputRequirement-from pipelex.core.stuff import Stuff-from pipelex.core.stuff_content import ImageContent, ListContent, PDFContent, StuffContent, TextContent-from pipelex.core.stuff_factory import StuffBlueprint, StuffFactory-from pipelex.core.working_memory import MAIN_STUFF_NAME, StuffDict, WorkingMemory+from pipelex.core.concepts.concept_native import NativeConcept+from pipelex.core.memory.working_memory import MAIN_STUFF_NAME, StuffDict, WorkingMemory+from pipelex.core.pipes.pipe_input_spec import TypedNamedInputRequirement+from pipelex.core.stuffs.stuff import Stuff+from pipelex.core.stuffs.stuff_content import ImageContent, ListContent, PDFContent, StuffContent, TextContent+from pipelex.core.stuffs.stuff_factory import StuffBlueprint, StuffFactoryfrom pipelex.exceptions import WorkingMemoryFactoryErrorfrom pipelex.tools.misc.json_utils import load_json_dict_from_pathdiff --git a/pipelex/core/pipe_blueprint.py b/pipelex/core/pipe_blueprint.pydeleted file mode 100644index 60488a4a0..000000000--- a/pipelex/core/pipe_blueprint.py+++ /dev/null@@ -1,55 +0,0 @@-from typing import Any, Dict, Optional, Protocol, TypeVar--from pydantic import ConfigDict, model_validator-from typing_extensions import Self, runtime_checkable--from pipelex.core.concept_code_factory import ConceptCodeFactory-from pipelex.core.pipe_abstract import PipeAbstract-from pipelex.core.stuff_content import StructuredContent---class PipeBlueprint(StructuredContent):- model_config = ConfigDict(extra="forbid")-- definition: Optional[str] = None- inputs: Optional[Dict[str, str]] = None- output: str- domain: str-- @model_validator(mode="after")- def add_domain_prefix(self) -> Self:- if self.inputs:- for input_name, input_concept_code in self.inputs.items():- self.inputs[input_name] = ConceptCodeFactory.make_concept_code_from_str(- concept_str=input_concept_code,- fallback_domain=self.domain,- )- self.output = ConceptCodeFactory.make_concept_code_from_str(- concept_str=self.output,- fallback_domain=self.domain,- )- return self---PipeBlueprintType = TypeVar("PipeBlueprintType", bound="PipeBlueprint", contravariant=True)--PipeType = TypeVar("PipeType", bound="PipeAbstract", covariant=True)---@runtime_checkable-class PipeSpecificFactoryProtocol(Protocol[PipeBlueprintType, PipeType]):- @classmethod- def make_pipe_from_blueprint(- cls,- domain_code: str,- pipe_code: str,- pipe_blueprint: PipeBlueprintType,- ) -> PipeType: ...-- @classmethod- def make_pipe_from_details_dict(- cls,- domain_code: str,- pipe_code: str,- details_dict: Dict[str, Any],- ) -> PipeType: ...diff --git a/pipelex/core/pipe_input_details.py b/pipelex/core/pipe_input_details.pydeleted file mode 100644index 6e5bc96a0..000000000--- a/pipelex/core/pipe_input_details.py+++ /dev/null@@ -1,76 +0,0 @@-from typing import Callable, Dict, List, Set, Tuple--from pydantic import Field, RootModel, field_validator--from pipelex import log-from pipelex.core.concept_code_factory import ConceptCodeFactory--PipeInputDetailsRoot = Dict[str, str]---class PipeInputDetails(RootModel[PipeInputDetailsRoot]):- root: PipeInputDetailsRoot = Field(default_factory=dict)-- @field_validator("root", mode="wrap")- @classmethod- def validate_concept_codes(cls, input_value: Dict[str, str], handler: Callable[[Dict[str, str]], Dict[str, str]]) -> Dict[str, str]:- # First let Pydantic handle the basic type validation- validated_dict: Dict[str, str] = handler(input_value)-- # Now we can transform and validate the keys and values- transformed_dict: Dict[str, str] = {}- for required_input, concept_str in validated_dict.items():- # in case of sub-attribute, the variable name is the object name, before the 1st dot- transformed_key: str = required_input.split(".", 1)[0]- if transformed_key != required_input:- log.verbose(f"Sub-attribute {required_input} detected, using {transformed_key} as variable name")-- # Validate concept_code- concept_code = ConceptCodeFactory.make_concept_code_from_str(concept_str=concept_str)-- if transformed_key in transformed_dict and transformed_dict[transformed_key] != concept_code:- log.verbose(- f"Variable {transformed_key} already exists with a different concept code: {transformed_dict[transformed_key]} -> {concept_str}"- )- transformed_dict[transformed_key] = concept_code-- return transformed_dict-- def set_default_domain(self, domain: str):- for input_name, input_concept_code in self.root.items():- if "." not in input_concept_code:- self.root[input_name] = f"{domain}.{input_concept_code}"-- def get(self, variable_name: str) -> str:- return self.root[variable_name]-- def add_requirement(self, variable_name: str, concept_code: str):- self.root[variable_name] = concept_code-- @property- def items(self) -> List[Tuple[str, str]]:- return list(self.root.items())-- @property- def concepts(self) -> Set[str]:- return set(self.root.values())-- # @property- # def variables(self) -> List[str]:- # return list(self.root.keys())-- @property- def required_names(self) -> List[str]:- the_required_names: List[str] = []- for requirement_expression in self.root.keys():- required_variable_name = requirement_expression.split(".", 1)[0]- the_required_names.append(required_variable_name)- return the_required_names-- @property- def detailed_requirements(self) -> List[Tuple[str, str, str]]:- the_requirements: List[Tuple[str, str, str]] = []- for requirement_expression, concept_code in self.root.items():- required_variable_name = requirement_expression.split(".", 1)[0]- the_requirements.append((required_variable_name, requirement_expression, concept_code))- return the_requirementsdiff --git a/pipelex/core/pipe_abstract.py b/pipelex/core/pipes/pipe_abstract.pysimilarity index 60%rename from pipelex/core/pipe_abstract.pyrename to pipelex/core/pipes/pipe_abstract.pyindex 2caef096c..76e8192e0 100644--- a/pipelex/core/pipe_abstract.py+++ b/pipelex/core/pipes/pipe_abstract.py@@ -1,12 +1,14 @@from abc import ABC, abstractmethod-from typing import Optional, Set, Type+from typing import Any, Dict, Optional, Set, Type-from pydantic import BaseModel, ConfigDict, Field+from pydantic import BaseModel, ConfigDict, Field, model_validator-from pipelex.core.pipe_input_spec import PipeInputSpec-from pipelex.core.pipe_output import PipeOutput-from pipelex.core.pipe_run_params import PipeRunParams-from pipelex.core.working_memory import WorkingMemory+from pipelex.core.concepts.concept_code_factory import ConceptCodeFactory+from pipelex.core.domains.domain import SpecialDomain+from pipelex.core.memory.working_memory import WorkingMemory+from pipelex.core.pipes.pipe_input_spec import PipeInputSpec+from pipelex.core.pipes.pipe_output import PipeOutput+from pipelex.core.pipes.pipe_run_params import PipeRunParamsfrom pipelex.exceptions import PipeStackOverflowErrorfrom pipelex.pipeline.job_metadata import JobMetadata@@ -21,6 +23,32 @@ class PipeAbstract(ABC, BaseModel):inputs: PipeInputSpec = Field(default_factory=PipeInputSpec)output_concept_code: str+ @model_validator(mode="before")+ @classmethod+ # TODO: This is a hack to add the domain prefix to the concept code.+ # Fix this with a more simple Concept Factory approach.+ def add_domain_prefix(cls, data: Dict[str, Any]) -> Dict[str, Any]:+ # Process output_concept_code - always present+ if "output_concept_code" in data:+ data["output_concept_code"] = ConceptCodeFactory.make_concept_code_from_str(+ domain=data["domain"],+ concept_str=data["output_concept_code"],+ fallback_domain=SpecialDomain.IMPLICIT,+ )++ # Process inputs - always present+ if "inputs" in data:+ inputs = data["inputs"]+ if isinstance(inputs, PipeInputSpec):+ for _, input_requirement in inputs.root.items():+ input_requirement.concept_code = ConceptCodeFactory.make_concept_code_from_str(+ concept_str=input_requirement.concept_code,+ domain=data["domain"],+ fallback_domain=SpecialDomain.IMPLICIT,+ )++ return data+@propertydef class_name(self) -> str:return self.__class__.__name__diff --git a/pipelex/core/pipes/pipe_blueprint.py b/pipelex/core/pipes/pipe_blueprint.pynew file mode 100644index 000000000..962143c38--- /dev/null+++ b/pipelex/core/pipes/pipe_blueprint.py@@ -0,0 +1,48 @@+from typing import Any, Dict, Optional++from pydantic import BaseModel, model_validator+from typing_extensions import Self++from pipelex.core.pipes.pipe_input_spec import InputRequirementBlueprint+from pipelex.types import StrEnum+++class PipeBlueprintError(Exception):+ """Exception raised for errors in the PipeBlueprint class."""++ pass+++class AllowedPipeTypes(StrEnum):+ # Pipe Operators+ PIPE_FUNC = "PipeFunc"+ PIPE_IMG_GEN = "PipeImgGen"+ PIPE_JINJA2 = "PipeJinja2"+ PIPE_LLM = "PipeLLM"+ PIPE_OCR = "PipeOcr"+ # Pipe Controller+ PIPE_BATCH = "PipeBatch"+ PIPE_CONDITION = "PipeCondition"+ PIPE_PARALLEL = "PipeParallel"+ PIPE_SEQUENCE = "PipeSequence"+++class PipeBlueprint(BaseModel):+ """Simple data container for pipe blueprint information.++ The 'type' field uses Any to avoid type override conflicts but is validated+ at runtime to ensure only valid pipe type values are allowed.+ """++ type: Any # TODO: Find a better way to handle this.+ definition: Optional[str] = None+ inputs: Optional[Dict[str, InputRequirementBlueprint]] = None+ output: str++ @model_validator(mode="after")+ def validate_pipe_type(self) -> Self:+ """Validate that the pipe type is one of the allowed values."""+ allowed_types = [_type.value for _type in AllowedPipeTypes]+ if self.type not in allowed_types:+ raise PipeBlueprintError(f"Invalid pipe type '{self.type}'. Must be one of: {sorted(allowed_types)}")+ return selfdiff --git a/pipelex/core/pipes/pipe_factory.py b/pipelex/core/pipes/pipe_factory.pynew file mode 100644index 000000000..eff1b405c--- /dev/null+++ b/pipelex/core/pipes/pipe_factory.py@@ -0,0 +1,56 @@+from typing import Any, Protocol, Type, TypeVar++from kajson.exceptions import ClassRegistryInheritanceError, ClassRegistryNotFoundError+from kajson.kajson_manager import KajsonManager+from typing_extensions import override, runtime_checkable++from pipelex.core.pipes.pipe_abstract import PipeAbstract+from pipelex.core.pipes.pipe_blueprint import PipeBlueprint+from pipelex.exceptions import PipeFactoryError++PipeBlueprintType = TypeVar("PipeBlueprintType", bound="PipeBlueprint", contravariant=True)+PipeType = TypeVar("PipeType", bound="PipeAbstract", covariant=True)+++@runtime_checkable+class PipeFactoryProtocol(Protocol[PipeBlueprintType, PipeType]):+ @classmethod+ def make_pipe_from_blueprint(+ cls,+ domain_code: str,+ pipe_code: str,+ pipe_blueprint: PipeBlueprintType,+ ) -> PipeType: ...+++class PipeFactory(PipeFactoryProtocol[PipeBlueprint, PipeAbstract]):+ @classmethod+ @override+ def make_pipe_from_blueprint(+ cls,+ domain_code: str,+ pipe_code: str,+ pipe_blueprint: PipeBlueprint,+ ) -> PipeAbstract:+ # The factory class name for that specific type of Pipe is the pipe class name with "Factory" suffix+ factory_class_name = f"{pipe_blueprint.type}Factory"+ try:+ pipe_factory: Type[PipeFactoryProtocol[Any, Any]] = KajsonManager.get_class_registry().get_required_subclass(+ name=factory_class_name,+ base_class=PipeFactoryProtocol,+ )+ except ClassRegistryNotFoundError as factory_not_found_error:+ raise PipeFactoryError(+ f"Pipe '{pipe_code}' couldn't be created: factory '{factory_class_name}' not found: {factory_not_found_error}"+ ) from factory_not_found_error+ except ClassRegistryInheritanceError as factory_inheritance_error:+ raise PipeFactoryError(+ f"Pipe '{pipe_code}' couldn't be created: factory '{factory_class_name}' is not a subclass of {type(PipeFactoryProtocol)}."+ ) from factory_inheritance_error++ pipe_from_blueprint: PipeAbstract = pipe_factory.make_pipe_from_blueprint(+ domain_code=domain_code,+ pipe_code=pipe_code,+ pipe_blueprint=pipe_blueprint,+ )+ return pipe_from_blueprintdiff --git a/pipelex/core/pipe_input_spec.py b/pipelex/core/pipes/pipe_input_spec.pysimilarity index 78%rename from pipelex/core/pipe_input_spec.pyrename to pipelex/core/pipes/pipe_input_spec.pyindex 5e489fb47..8b9244c36 100644--- a/pipelex/core/pipe_input_spec.py+++ b/pipelex/core/pipes/pipe_input_spec.py@@ -1,15 +1,28 @@-from typing import Callable, Dict, List, Optional, Set, Tuple, Type+from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type-from pydantic import BaseModel, Field, RootModel, field_validator+from pydantic import BaseModel, Field, RootModel, field_validator, model_validatorfrom typing_extensions import Selffrom pipelex import log-from pipelex.core.concept_code_factory import ConceptCodeFactory-from pipelex.core.pipe_run_params import PipeOutputMultiplicity-from pipelex.core.stuff_content import StuffContent+from pipelex.core.concepts.concept_code_factory import ConceptCodeFactory+from pipelex.core.pipes.pipe_run_params import PipeOutputMultiplicity+from pipelex.core.stuffs.stuff_content import StuffContentfrom pipelex.exceptions import PipeInputNotFoundError+class InputRequirementBlueprint(BaseModel):+ concept_code: str+ multiplicity: Optional[PipeOutputMultiplicity] = None++ @model_validator(mode="before")+ @classmethod+ def validate_input(cls, data: Any) -> Any:+ """If the incoming data is a string, convert it to a dict with concept_code."""+ if isinstance(data, str):+ return {"concept_code": data}+ return data++class InputRequirement(BaseModel):concept_code: strmultiplicity: Optional[PipeOutputMultiplicity] = None@@ -93,11 +106,18 @@ class PipeInputSpec(RootModel[PipeInputSpecRoot]):return cls(root={})@classmethod- def make_from_dict(cls, concepts_dict: Dict[str, str]) -> Self:- the_dict: Dict[str, InputRequirement] = {- var_name: InputRequirement(concept_code=concept_code) for var_name, concept_code in concepts_dict.items()- }- return cls(root=the_dict)+ def make_from_blueprint(cls, domain: str, blueprint: Dict[str, InputRequirementBlueprint]) -> Self:+ for var_name, input_requirement_blueprint in blueprint.items():+ concept_code = ConceptCodeFactory.make_concept_code_from_str(domain=domain, concept_str=input_requirement_blueprint.concept_code)+ blueprint[var_name].concept_code = concept_code+ return cls(+ root={+ var_name: InputRequirement(+ concept_code=input_requirement_blueprint.concept_code, multiplicity=input_requirement_blueprint.multiplicity+ )+ for var_name, input_requirement_blueprint in blueprint.items()+ }+ )@propertydef items(self) -> List[Tuple[str, InputRequirement]]:diff --git a/pipelex/core/pipe_library.py b/pipelex/core/pipes/pipe_library.pysimilarity index 95%rename from pipelex/core/pipe_library.pyrename to pipelex/core/pipes/pipe_library.pyindex ad05d4770..5df5f0419 100644--- a/pipelex/core/pipe_library.py+++ b/pipelex/core/pipes/pipe_library.py@@ -7,8 +7,8 @@ from rich.table import Tablefrom typing_extensions import overridefrom pipelex import pretty_print-from pipelex.core.pipe_abstract import PipeAbstract-from pipelex.core.pipe_provider_abstract import PipeProviderAbstract+from pipelex.core.pipes.pipe_abstract import PipeAbstract+from pipelex.core.pipes.pipe_provider_abstract import PipeProviderAbstractfrom pipelex.exceptions import ConceptError, ConceptLibraryConceptNotFoundError, PipeLibraryError, PipeLibraryPipeNotFoundErrorfrom pipelex.hub import get_concept_provider@@ -46,6 +46,10 @@ class PipeLibrary(RootModel[PipeLibraryRoot], PipeProviderAbstract):raise PipeLibraryError(f"Pipe '{name}' already exists in the library")self.root[pipe.code] = pipe+ def add_pipes(self, pipes: List[PipeAbstract]):+ for pipe in pipes:+ self.add_new_pipe(pipe=pipe)+def add_or_update_pipe(self, pipe: PipeAbstract):name = pipe.codepipe.inputs.set_default_domain(domain=pipe.domain)diff --git a/pipelex/core/pipe_output.py b/pipelex/core/pipes/pipe_output.pysimilarity index 95%rename from pipelex/core/pipe_output.pyrename to pipelex/core/pipes/pipe_output.pyindex 3be2b90b2..1a5143d52 100644--- a/pipelex/core/pipe_output.py+++ b/pipelex/core/pipes/pipe_output.py@@ -2,8 +2,9 @@ from typing import List, Type, TypeVarfrom pydantic import BaseModel, Field-from pipelex.core.stuff import Stuff-from pipelex.core.stuff_content import (+from pipelex.core.memory.working_memory import WorkingMemory+from pipelex.core.stuffs.stuff import Stuff+from pipelex.core.stuffs.stuff_content import (HtmlContent,ImageContent,ListContent,@@ -13,7 +14,6 @@ from pipelex.core.stuff_content import (TextAndImagesContent,TextContent,)-from pipelex.core.working_memory import WorkingMemoryfrom pipelex.pipeline.pipeline_models import SpecialPipelineIddiff --git a/pipelex/core/pipe_provider_abstract.py b/pipelex/core/pipes/pipe_provider_abstract.pysimilarity index 92%rename from pipelex/core/pipe_provider_abstract.pyrename to pipelex/core/pipes/pipe_provider_abstract.pyindex 3f3665c42..ebeb52120 100644--- a/pipelex/core/pipe_provider_abstract.py+++ b/pipelex/core/pipes/pipe_provider_abstract.py@@ -1,7 +1,7 @@from abc import ABC, abstractmethodfrom typing import Dict, List, Optional-from pipelex.core.pipe_abstract import PipeAbstract+from pipelex.core.pipes.pipe_abstract import PipeAbstractPipeLibraryRoot = Dict[str, PipeAbstract]diff --git a/pipelex/core/pipe_run_params.py b/pipelex/core/pipes/pipe_run_params.pysimilarity index 98%rename from pipelex/core/pipe_run_params.pyrename to pipelex/core/pipes/pipe_run_params.pyindex 7c93ee3c2..35eb7872f 100644--- a/pipelex/core/pipe_run_params.py+++ b/pipelex/core/pipes/pipe_run_params.py@@ -4,7 +4,7 @@ from pydantic import BaseModel, Field, field_validatorfrom typing_extensions import Selffrom pipelex import log-from pipelex.core.working_memory import BATCH_ITEM_STUFF_NAME, MAIN_STUFF_NAME+from pipelex.core.memory.working_memory import BATCH_ITEM_STUFF_NAME, MAIN_STUFF_NAMEfrom pipelex.types import StrEnumdiff --git a/pipelex/core/pipe_run_params_factory.py b/pipelex/core/pipes/pipe_run_params_factory.pysimilarity index 89%rename from pipelex/core/pipe_run_params_factory.pyrename to pipelex/core/pipes/pipe_run_params_factory.pyindex 79abb8f3e..65b006529 100644--- a/pipelex/core/pipe_run_params_factory.py+++ b/pipelex/core/pipes/pipe_run_params_factory.py@@ -1,7 +1,7 @@from typing import Any, Dict, Optionalfrom pipelex.config import get_config-from pipelex.core.pipe_run_params import BatchParams, PipeOutputMultiplicity, PipeRunMode, PipeRunParams+from pipelex.core.pipes.pipe_run_params import BatchParams, PipeOutputMultiplicity, PipeRunMode, PipeRunParamsclass PipeRunParamsFactory:diff --git a/pipelex/core/registry_models.py b/pipelex/core/registry_models.pyindex c9b2bfd95..0b3d7843d 100644--- a/pipelex/core/registry_models.py+++ b/pipelex/core/registry_models.py@@ -1,9 +1,9 @@from typing import Any, ClassVar, List-from pipelex.core.pipe_abstract import PipeAbstractType-from pipelex.core.pipe_blueprint import PipeSpecificFactoryProtocol-from pipelex.core.stuff import Stuff-from pipelex.core.stuff_content import (+from pipelex.core.pipes.pipe_abstract import PipeAbstractType+from pipelex.core.pipes.pipe_factory import PipeFactoryProtocol+from pipelex.core.stuffs.stuff import Stuff+from pipelex.core.stuffs.stuff_content import (DynamicContent,HtmlContent,ImageContent,@@ -17,8 +17,7 @@ from pipelex.core.stuff_content import (TextAndImagesContent,TextContent,)-from pipelex.libraries.pipeline_blueprint import PipelineBlueprint-from pipelex.libraries.pipelines.meta.pipeline_draft import PipelineDraft+from pipelex.libraries.pipelines.meta.pipeline_draft import PipelexBundleBlueprint, PipelineDraftfrom pipelex.pipe_controllers.pipe_batch import PipeBatchfrom pipelex.pipe_controllers.pipe_batch_factory import PipeBatchFactoryfrom pipelex.pipe_controllers.pipe_condition import PipeCondition@@ -53,7 +52,7 @@ class PipelexRegistryModels(RegistryModels):PipeOcr,]- PIPE_OPERATORS_FACTORY: ClassVar[List[PipeSpecificFactoryProtocol[Any, Any]]] = [+ PIPE_OPERATORS_FACTORY: ClassVar[List[PipeFactoryProtocol[Any, Any]]] = [PipeFuncFactory,PipeImgGenFactory,PipeJinja2Factory,@@ -68,7 +67,7 @@ class PipelexRegistryModels(RegistryModels):PipeSequence,]- PIPE_CONTROLLERS_FACTORY: ClassVar[List[PipeSpecificFactoryProtocol[Any, Any]]] = [+ PIPE_CONTROLLERS_FACTORY: ClassVar[List[PipeFactoryProtocol[Any, Any]]] = [PipeBatchFactory,PipeConditionFactory,PipeParallelFactory,@@ -88,7 +87,7 @@ class PipelexRegistryModels(RegistryModels):PDFContent,TextAndImagesContent,PageContent,- PipelineBlueprint,+ PipelexBundleBlueprint,PipelineDraft,]diff --git a/pipelex/core/stuff.py b/pipelex/core/stuffs/stuff.pysimilarity index 97%rename from pipelex/core/stuff.pyrename to pipelex/core/stuffs/stuff.pyindex 208c51701..88659b56a 100644--- a/pipelex/core/stuff.py+++ b/pipelex/core/stuffs/stuff.py@@ -4,9 +4,9 @@ from pydantic import ConfigDict, ValidationErrorfrom typing_extensions import overridefrom pipelex import log-from pipelex.core.concept import Concept-from pipelex.core.stuff_artefact import StuffArtefact-from pipelex.core.stuff_content import (+from pipelex.core.concepts.concept import Concept+from pipelex.core.stuffs.stuff_artefact import StuffArtefact+from pipelex.core.stuffs.stuff_content import (HtmlContent,ImageContent,ListContent,diff --git a/pipelex/core/stuff_artefact.py b/pipelex/core/stuffs/stuff_artefact.pysimilarity index 97%rename from pipelex/core/stuff_artefact.pyrename to pipelex/core/stuffs/stuff_artefact.pyindex 45b8d3b9f..e1cc2e771 100644--- a/pipelex/core/stuff_artefact.py+++ b/pipelex/core/stuffs/stuff_artefact.py@@ -4,7 +4,7 @@ from jinja2.runtime import Contextfrom pydantic import RootModelfrom typing_extensions import override-from pipelex.core.stuff_content import StuffContent+from pipelex.core.stuffs.stuff_content import StuffContentfrom pipelex.exceptions import StuffArtefactErrorfrom pipelex.tools.templating.jinja2_models import Jinja2ContextKey, Jinja2TaggableAbstractfrom pipelex.tools.templating.templating_models import TextFormatdiff --git a/pipelex/core/stuff_content.py b/pipelex/core/stuffs/stuff_content.pysimilarity index 100%rename from pipelex/core/stuff_content.pyrename to pipelex/core/stuffs/stuff_content.pydiff --git a/pipelex/core/stuff_factory.py b/pipelex/core/stuffs/stuff_factory.pysimilarity index 97%rename from pipelex/core/stuff_factory.pyrename to pipelex/core/stuffs/stuff_factory.pyindex f5e663611..4fb383025 100644--- a/pipelex/core/stuff_factory.py+++ b/pipelex/core/stuffs/stuff_factory.py@@ -4,12 +4,12 @@ import shortuuidfrom pydantic import BaseModel, ValidationErrorfrom pipelex.client.protocol import StuffContentOrData-from pipelex.core.concept import Concept-from pipelex.core.concept_code_factory import ConceptCodeFactory-from pipelex.core.concept_factory import ConceptFactory-from pipelex.core.concept_native import NativeConcept, NativeConceptClass-from pipelex.core.stuff import Stuff-from pipelex.core.stuff_content import (+from pipelex.core.concepts.concept import Concept+from pipelex.core.concepts.concept_code_factory import ConceptCodeFactory+from pipelex.core.concepts.concept_factory import ConceptFactory+from pipelex.core.concepts.concept_native import NativeConcept, NativeConceptClass+from pipelex.core.stuffs.stuff import Stuff+from pipelex.core.stuffs.stuff_content import (ListContent,StuffContent,StuffContentInitableFromStr,diff --git a/pipelex/create/build_blueprint.py b/pipelex/create/build_blueprint.pyindex feb0b3573..867fe91ce 100644--- a/pipelex/create/build_blueprint.py+++ b/pipelex/create/build_blueprint.py@@ -3,10 +3,9 @@ from __future__ import annotationsfrom typing import Optionalfrom pipelex import log, pretty_print+from pipelex.core.interpreter import PipelexInterpreterfrom pipelex.create.helpers import get_support_file-from pipelex.create.pipeline_toml import save_pipeline_blueprint_toml_to_path-from pipelex.create.validate_blueprint import load_pipes_from_generated_blueprint, validate_blueprint-from pipelex.libraries.pipeline_blueprint import PipelineBlueprint+from pipelex.libraries.pipelines.meta.pipeline_draft import PipelexBundleBlueprintStufffrom pipelex.pipeline.execute import execute_pipelinefrom pipelex.tools.misc.file_utils import save_text_to_pathfrom pipelex.tools.misc.json_utils import save_as_json_to_path@@ -27,30 +26,20 @@ async def do_build_blueprint("requirements": requirements,"draft_pipeline_rules": get_support_file(subpath="create/draft_pipelines.md"),"build_pipeline_rules": get_support_file(subpath="create/build_pipelines.md"),+ "create_structured_output_rules": get_support_file(subpath="create/structures.md"),},)- blueprint = pipe_output.main_stuff_as(content_type=PipelineBlueprint)- pretty_print(blueprint, title="Pipeline Blueprint")+ pretty_print(pipe_output, title="Pipe Output")++ blueprint = pipe_output.main_stuff_as(content_type=PipelexBundleBlueprintStuff)pipeline_draft = pipe_output.working_memory.get_stuff_as_str(name="pipeline_draft")- pretty_print(pipeline_draft, title="Pipeline Draft")# Save or display resultoutput_path_base = output_path or "pipelex/libraries/pipelines/temp/gen_blueprint"draft_path = f"{output_path_base}_draft.md"- save_text_to_path(text=pipeline_draft, path=draft_path)+ save_text_to_path(text=pipeline_draft, path=draft_path, create_directory=True)rough_toml_path = f"{output_path_base}_rough.toml"- save_pipeline_blueprint_toml_to_path(blueprint=blueprint, path=rough_toml_path)+ save_text_to_path(text=PipelexInterpreter.make_toml_content(blueprint=blueprint), path=rough_toml_path)rough_json_path = f"{output_path_base}_rough.json"save_as_json_to_path(object_to_save=blueprint, path=rough_json_path)log.info(f"✅ Rough blueprint saved to '{output_path_base}'")-- load_pipes_from_generated_blueprint(blueprint=blueprint)- log.info("✅ Pipes loaded from blueprint")-- if validate:- await validate_blueprint(blueprint=blueprint, is_error_fixing_enabled=True)- fixed_toml_path = f"{output_path_base}_fixed.toml"- save_pipeline_blueprint_toml_to_path(blueprint=blueprint, path=fixed_toml_path)- fixed_json_path = f"{output_path_base}_fixed.json"- save_as_json_to_path(object_to_save=blueprint, path=fixed_json_path)- log.info(f"✅ Fixed blueprint saved to '{output_path_base}'")diff --git a/pipelex/create/draft_pipeline.py b/pipelex/create/draft_pipeline.pyindex f87dd736e..7b10aec56 100644--- a/pipelex/create/draft_pipeline.py+++ b/pipelex/create/draft_pipeline.py@@ -6,10 +6,10 @@ import typerfrom pipelex import pretty_printfrom pipelex.create.helpers import get_support_file-from pipelex.create.pipeline_toml import dict_to_toml, save_toml_to_pathfrom pipelex.libraries.pipelines.meta.pipeline_draft import PipelineDraftfrom pipelex.pipeline.execute import execute_pipelinefrom pipelex.tools.misc.file_utils import save_text_to_path+from pipelex.tools.misc.toml_utils import dict_to_toml, save_toml_to_pathasync def do_draft_pipeline_text(diff --git a/pipelex/create/pipeline_toml.py b/pipelex/create/pipeline_toml.pydeleted file mode 100644index 9e6bd8a5c..000000000--- a/pipelex/create/pipeline_toml.py+++ /dev/null@@ -1,138 +0,0 @@-from __future__ import annotations--from typing import Any, Dict, List, Mapping, cast--import tomlkit-from tomlkit import array, document, inline_table, table-from tomlkit import string as tk_string--from pipelex.libraries.pipeline_blueprint import PipelineBlueprint-from pipelex.tools.misc.file_utils import save_text_to_path-from pipelex.tools.misc.json_utils import remove_none_values_from_dict-from pipelex.tools.misc.toml_utils import clean_trailing_whitespace---def make_toml_string(- text: str,- prefer_literal: bool = False,- force_multiline: bool = False,- ensure_trailing_newline: bool = True,- ensure_leading_blank_line_in_value: bool = False,-):- """- Build a tomlkit string node.- - If `force_multiline` or the text contains '\\n', we emit a triple-quoted multiline string.- - When multiline, `ensure_trailing_newline` puts the closing quotes on their own line.- - When multiline, `ensure_leading_blank_line_in_value` inserts a real blank line at the start of the value.- """- needs_multiline = force_multiline or ("\n" in text)- normalized = text-- if needs_multiline:- if ensure_leading_blank_line_in_value and not normalized.startswith("\n"):- normalized = "\n" + normalized- if ensure_trailing_newline and not normalized.endswith("\n"):- normalized = normalized + "\n"-- use_literal = prefer_literal and ("'''" not in normalized)- return tk_string(normalized, multiline=needs_multiline, literal=use_literal)---def _convert_to_inline(value: Any):- """Recursively convert Python values; dicts -> inline tables; lists kept as arrays."""- if isinstance(value, Mapping):- value = cast(Mapping[str, Any], value)- inline_table_obj = inline_table()- for key, value_item in value.items():- inline_table_obj[key] = _convert_to_inline(value_item)- return inline_table_obj-- if isinstance(value, list):- value = cast(List[Any], value)- array_obj = array()- array_obj.multiline(True) # set to False for single-line arrays- for element in value:- if isinstance(element, Mapping):- element = cast(Mapping[str, Any], element)- inline_element = inline_table()- for inner_key, inner_value in element.items():- inline_element[inner_key] = _convert_to_inline(inner_value)- array_obj.append(inline_element) # pyright: ignore[reportUnknownMemberType]- else:- array_obj.append(_convert_to_inline(element)) # pyright: ignore[reportUnknownMemberType]- return array_obj-- if isinstance(value, str):- # Triple quotes if needed (or forced); closing quotes on their own line.- return make_toml_string(- value,- prefer_literal=False, # flip to True for '''...'''- force_multiline=False, # flip to True to force """...""" even without \n- ensure_trailing_newline=True, # keep closing """ on its own line- ensure_leading_blank_line_in_value=False, # flip to True to keep a blank first line- )- return value---def _convert_mapping_to_table(mapping: Mapping[str, Any]) -> Any:- """Convert a mapping into a TOML Table where any nested mappings (third level+)- are converted to inline tables.-- This creates a second-level standard table, and only uses inline tables for- third level and deeper structures.- """- tbl = table()- for field_key, field_value in mapping.items():- if isinstance(field_value, Mapping):- # Third-level mapping -> inline table- tbl.add(field_key, _convert_to_inline(field_value))- else:- tbl.add(field_key, _convert_to_inline(field_value))- return tbl---def dict_to_toml(data: Mapping[str, Any]) -> str:- """Top-level keys become tables; second-level mappings become tables; inline tables start at third level."""- data = remove_none_values_from_dict(data=data)- document_root = document()- for section_key, section_value in data.items():- if isinstance(section_value, Mapping):- section_value = cast(Mapping[str, Any], section_value)- table_obj = table()- for field_key, field_value in section_value.items():- if isinstance(field_value, Mapping):- # Second level mapping -> standard table [section.field]- sub_table = _convert_mapping_to_table(cast(Mapping[str, Any], field_value))- table_obj.add(field_key, sub_table)- else:- table_obj.add(field_key, _convert_to_inline(field_value))- document_root.add(section_key, table_obj) # `[section_key]` section- else:- document_root.add(section_key, _convert_to_inline(section_value))- dumped_content = tomlkit.dumps(document_root) # pyright: ignore[reportUnknownMemberType]- return dumped_content---def pipeline_blueprint_to_toml(blueprint: PipelineBlueprint) -> str:- blueprint_dict = blueprint.smart_dump()- return dict_to_toml(data=blueprint_dict)---def save_toml_to_path(data: Dict[str, Any], path: str) -> None:- """Save dictionary as TOML to file path.-- Args:- data: Dictionary to save as TOML- path: Path where to save the TOML file- """- # data_cleaned = remove_none_values_from_dict(data=data)- data_cleaned = data- with open(path, "w", encoding="utf-8") as file:- toml_content: str = dict_to_toml(data=data_cleaned)- cleaned_content = clean_trailing_whitespace(toml_content)- file.write(cleaned_content)---def save_pipeline_blueprint_toml_to_path(blueprint: PipelineBlueprint, path: str) -> None:- pipeline_blueprint_toml = pipeline_blueprint_to_toml(blueprint=blueprint)- save_text_to_path(text=pipeline_blueprint_toml, path=path)diff --git a/pipelex/create/structured_output_generator.py b/pipelex/create/structured_output_generator.pyindex cec834dcd..3c4bd0023 100644--- a/pipelex/create/structured_output_generator.py+++ b/pipelex/create/structured_output_generator.py@@ -2,43 +2,21 @@from __future__ import annotations-from typing import Any, ClassVar, Dict, List, Optional, Union+from typing import Any, Dict, List, Optional, Unionimport tomlkit-from pipelex.types import StrEnum---class FieldType(StrEnum):- """Supported field types for structured outputs."""-- TEXT = "text"- NUMBER = "number"- INTEGER = "integer"- BOOLEAN = "boolean"- LIST = "list"- DICT = "dict"- ENUM = "enum" # For referencing defined enums+from pipelex.core.concepts.concept_blueprint import ConceptStructureBlueprintFieldTypeclass StructuredOutputGenerator:"""Generate Pydantic BaseModel classes from TOML structured output definitions."""- # Map high-level type names to Python types- TYPE_MAPPING: ClassVar[Dict[FieldType, str]] = {- FieldType.TEXT: "str",- FieldType.NUMBER: "float",- FieldType.INTEGER: "int",- FieldType.BOOLEAN: "bool",- FieldType.LIST: "List",- FieldType.DICT: "Dict",- }-def __init__(self):self.imports = {"from typing import Optional, List, Dict, Any, Literal","from enum import Enum",- "from pipelex.core.stuff_content import StructuredContent",+ "from pipelex.core.stuffs.stuff_content import StructuredContent","from pydantic import Field",}self.enum_definitions: Dict[str, Dict[str, Any]] = {} # Store enum definitions@@ -170,9 +148,9 @@ class StructuredOutputGenerator:"""# Handle simple string definitions (just the definition text)if isinstance(field_def, str):- field_def = {"type": FieldType.TEXT, "definition": field_def}+ field_def = {"type": ConceptStructureBlueprintFieldType.TEXT, "definition": field_def}- field_type = field_def.get("type", FieldType.TEXT)+ field_type = field_def.get("type", ConceptStructureBlueprintFieldType.TEXT)definition = field_def.get("definition", f"{field_name} field")required = field_def.get("required", False)default_value = field_def.get("default")@@ -225,7 +203,7 @@ class StructuredOutputGenerator:# Convert string to FieldType if neededif isinstance(field_type, str):try:- field_type_enum = FieldType(field_type)+ field_type_enum = ConceptStructureBlueprintFieldType(field_type)except ValueError:# Unknown type, assume it's a custom type or class referencereturn field_type@@ -233,15 +211,15 @@ class StructuredOutputGenerator:# Use match/case for type handlingmatch field_type:- case FieldType.TEXT:+ case ConceptStructureBlueprintFieldType.TEXT:return "str"- case FieldType.NUMBER:+ case ConceptStructureBlueprintFieldType.NUMBER:return "float"- case FieldType.INTEGER:+ case ConceptStructureBlueprintFieldType.INTEGER:return "int"- case FieldType.BOOLEAN:+ case ConceptStructureBlueprintFieldType.BOOLEAN:return "bool"- case FieldType.LIST:+ case ConceptStructureBlueprintFieldType.LIST:item_type = field_def.get("item_type", "Any")# Check if item_type is an enum referenceif isinstance(item_type, str) and item_type in self.enum_definitions:@@ -249,25 +227,25 @@ class StructuredOutputGenerator:# Recursively handle item typesif isinstance(item_type, str):try:- item_type_enum = FieldType(item_type)+ item_type_enum = ConceptStructureBlueprintFieldType(item_type)item_type = self._get_python_type(item_type_enum, {})except ValueError:# Keep as string if not a known FieldTypepassreturn f"List[{item_type}]"- case FieldType.DICT:+ case ConceptStructureBlueprintFieldType.DICT:key_type = field_def.get("key_type", "str")value_type = field_def.get("value_type", "Any")# Recursively handle key and value typesif isinstance(key_type, str):try:- key_type_enum = FieldType(key_type)+ key_type_enum = ConceptStructureBlueprintFieldType(key_type)key_type = self._get_python_type(key_type_enum, {})except ValueError:passif isinstance(value_type, str):try:- value_type_enum = FieldType(value_type)+ value_type_enum = ConceptStructureBlueprintFieldType(value_type)value_type = self._get_python_type(value_type_enum, {})except ValueError:passdiff --git a/pipelex/create/validate_blueprint.py b/pipelex/create/validate_blueprint.pydeleted file mode 100644index 39fae3dbc..000000000--- a/pipelex/create/validate_blueprint.py+++ /dev/null@@ -1,105 +0,0 @@-from __future__ import annotations--from typing import Any, Dict, cast--from pipelex import log-from pipelex.exceptions import PipeDefinitionError, PipelexCLIError, StaticValidationError, StaticValidationErrorType-from pipelex.hub import get_library_manager, get_required_pipe-from pipelex.libraries.library_manager import LibraryManager-from pipelex.libraries.pipeline_blueprint import PipelineBlueprint-from pipelex.pipe_controllers.pipe_controller import PipeController-from pipelex.pipe_works.pipe_dry import dry_run_pipe_codes-from pipelex.tools.misc.dict_utils import insert_before---async def validate_blueprint(blueprint: PipelineBlueprint, is_error_fixing_enabled: bool) -> None:- # Load pipes from the blueprint- log.dev("Loading pipes from blueprint")- try:- load_pipes_from_generated_blueprint(blueprint=blueprint, updates_allowed=True)- except PipeDefinitionError as exc:- raise PipelexCLIError(f"Failed to load pipes from blueprint for domain '{blueprint.desc}': {exc}") from exc-- try:- await validate_loaded_blueprint(blueprint=blueprint)- except StaticValidationError as static_validation_error:- match static_validation_error.error_type:- case StaticValidationErrorType.MISSING_INPUT_VARIABLE:- if is_error_fixing_enabled:- pipe_code = static_validation_error.pipe_code- if not pipe_code:- raise PipelexCLIError(f"No pipe code found for static validation error: {static_validation_error}")- pipe = get_required_pipe(pipe_code=pipe_code)- if isinstance(pipe, PipeController):- variable_names = static_validation_error.variable_names- required_concept_codes = static_validation_error.required_concept_codes- if not variable_names:- raise PipelexCLIError(f"No variable names found for static validation error: {static_validation_error}")- if not required_concept_codes:- raise PipelexCLIError(f"No required concept codes found for static validation error: {static_validation_error}")- log.error(f"❌ Missing: {variable_names} / concepts: {required_concept_codes}")- new_inputs: Dict[str, Any] = blueprint.pipe[pipe_code].get("inputs") or {}- for variable_name, concept_code in zip(variable_names, required_concept_codes):- new_inputs[variable_name] = concept_code- log.info(f"✅ Added: {variable_name} / {concept_code}")- if "inputs" in blueprint.pipe[pipe_code]:- blueprint.pipe[pipe_code]["inputs"] = new_inputs- else:- blueprint.pipe[pipe_code] = insert_before(- blueprint.pipe[pipe_code],- "output",- "inputs",- new_inputs,- )- log.dev(f"✅ Fixed missing inputs for pipe '{pipe_code}'")- await validate_blueprint(blueprint=blueprint, is_error_fixing_enabled=True)- else:- raise PipelexCLIError(f"Unexpected validation error:\n{static_validation_error}") from static_validation_error- else:- raise static_validation_error- case (- StaticValidationErrorType.EXTRANEOUS_INPUT_VARIABLE- | StaticValidationErrorType.INADEQUATE_INPUT_CONCEPT- | StaticValidationErrorType.TOO_MANY_CANDIDATE_INPUTS- ):- raise static_validation_error- log.info("✅ Validation passed")---async def validate_loaded_blueprint(blueprint: PipelineBlueprint) -> None:- # Run validation- log.dev("Validating blueprint")- get_library_manager().validate_libraries()- generated_pipe_codes = list(blueprint.pipe.keys())- if not generated_pipe_codes:- raise PipelexCLIError("No pipe found in blueprint to validate")- await dry_run_pipe_codes(pipe_codes=generated_pipe_codes)- log.info(f"✅ All pipes validated successfully: {generated_pipe_codes}")---def load_pipes_from_generated_blueprint(blueprint: PipelineBlueprint, updates_allowed: bool = False) -> None:- """Instantiate and register all pipes from a generated PipelineBlueprint.-- This constructs the appropriate typed PipeBlueprint for each pipe entry and- uses LibraryManager.load_pipe_from_blueprint to create the Pipe, then adds it- to the in-memory PipeLibrary so that validation can run against them.- """- library_manager = cast(LibraryManager, get_library_manager())- domain_code = blueprint.domain-- for pipe_code, details in blueprint.pipe.items():- details_dict: Dict[str, Any] = details.copy()-- pipe_blueprint = LibraryManager.make_pipe_blueprint_from_details(- domain_code=domain_code,- details_dict=details_dict,- )- # Create pipe and register in library- pipe = LibraryManager.load_pipe_from_blueprint(- pipe_code=pipe_code,- pipe_blueprint=pipe_blueprint,- )- if updates_allowed:- library_manager.pipe_library.add_or_update_pipe(pipe=pipe)- else:- library_manager.pipe_library.add_new_pipe(pipe=pipe)diff --git a/pipelex/hub.py b/pipelex/hub.pyindex b3d965f68..55e36ac81 100644--- a/pipelex/hub.py+++ b/pipelex/hub.py@@ -1,5 +1,4 @@-from collections import defaultdict-from typing import ClassVar, Dict, List, Optional, Type+from typing import ClassVar, List, Optional, Typefrom kajson.class_registry_abstract import ClassRegistryAbstract@@ -15,12 +14,12 @@ from pipelex.cogt.llm.llm_models.llm_model_provider_abstract import ()from pipelex.cogt.llm.llm_worker_abstract import LLMWorkerAbstractfrom pipelex.cogt.ocr.ocr_worker_abstract import OcrWorkerAbstract-from pipelex.core.concept import Concept-from pipelex.core.concept_provider_abstract import ConceptProviderAbstract-from pipelex.core.domain import Domain-from pipelex.core.domain_provider_abstract import DomainProviderAbstract-from pipelex.core.pipe_abstract import PipeAbstract-from pipelex.core.pipe_provider_abstract import PipeProviderAbstract+from pipelex.core.concepts.concept import Concept+from pipelex.core.concepts.concept_provider_abstract import ConceptProviderAbstract+from pipelex.core.domains.domain import Domain+from pipelex.core.domains.domain_provider_abstract import DomainProviderAbstract+from pipelex.core.pipes.pipe_abstract import PipeAbstract+from pipelex.core.pipes.pipe_provider_abstract import PipeProviderAbstractfrom pipelex.libraries.library_manager_abstract import LibraryManagerAbstractfrom pipelex.pipe_works.pipe_router_protocol import PipeRouterProtocolfrom pipelex.pipeline.activity.activity_manager_protocol import ActivityManagerProtocol@@ -391,10 +390,6 @@ def get_secret(secret_id: str) -> str:return get_secrets_provider().get_secret(secret_id=secret_id)-def get_domain_provider() -> DomainProviderAbstract:- return get_pipelex_hub().get_required_domain_provider()--def get_domains(excluded_domains: Optional[List[str]] = None) -> List[Domain]:domains = get_pipelex_hub().get_required_domain_provider().get_domains()if excluded_domains:@@ -417,18 +412,6 @@ def get_pipe_provider() -> PipeProviderAbstract:return get_pipelex_hub().get_required_pipe_provider()-def get_pipes_by_domain(- excluded_domains: Optional[List[str]] = None,-) -> Dict[str, List[str]]:- pipes = get_pipe_provider().get_pipes()- pipes_by_domain: Dict[str, List[str]] = defaultdict(list)- for pipe in pipes:- if excluded_domains and pipe.domain in excluded_domains:- continue- pipes_by_domain[pipe.domain].append(pipe.code)- return pipes_by_domain--def get_required_pipe(pipe_code: str) -> PipeAbstract:return get_pipelex_hub().get_required_pipe_provider().get_required_pipe(pipe_code=pipe_code)diff --git a/pipelex/libraries/library_config.py b/pipelex/libraries/library_config.pyindex 4af9468e1..eb10ec107 100644--- a/pipelex/libraries/library_config.py+++ b/pipelex/libraries/library_config.py@@ -8,55 +8,55 @@ PIPELEX_LIBRARIES_PATH = "libraries"class LibraryConfig(ConfigModel):package_name: ClassVar[str] = "pipelex"- config_folder_path: str = "pipelex_libraries"+ config_dir_path: str = "pipelex_libraries"@property- def config_file(self) -> str:- return f"{self.config_folder_path}/config.toml"+ def config_file_path(self) -> str:+ return f"{self.config_dir_path}/config.toml"@property- def pipelines_path(self) -> str:- return f"{self.config_folder_path}/pipelines"+ def pipelines_dir_path(self) -> str:+ return f"{self.config_dir_path}/pipelines"@property- def base_pipelines_path(self) -> str:- return f"{self.config_folder_path}/pipelines/base_library"+ def base_pipelines_dir_path(self) -> str:+ return f"{self.config_dir_path}/pipelines/base_library"@property- def llm_integrations_path(self) -> str:- return f"{self.config_folder_path}/llm_integrations"+ def llm_integrations_dir_path(self) -> str:+ return f"{self.config_dir_path}/llm_integrations"@property- def llm_deck_path(self) -> str:- return f"{self.config_folder_path}/llm_deck"+ def llm_deck_dir_path(self) -> str:+ return f"{self.config_dir_path}/llm_deck"@property- def templates_path(self) -> str:- return f"{self.config_folder_path}/templates"+ def templates_dir_path(self) -> str:+ return f"{self.config_dir_path}/templates"@property- def plugins_path(self) -> str:- return f"{self.config_folder_path}/plugins"+ def plugins_dir_path(self) -> str:+ return f"{self.config_dir_path}/plugins"@property- def test_pipelines_path(self) -> str:+ def test_pipelines_dir_path(self) -> str:return "tests/test_pipelines"@property- def failing_pipelines_path(self) -> str:- return "tests/test_pipelines/failing_pipelines.toml"+ def failing_pipelines_file_paths(self) -> List[str]:+ return ["tests/test_pipelines/failing_pipelines.toml"]def get_llm_deck_paths(self) -> List[str]:- llm_deck_paths = [str(path) for path in find_files_in_dir(dir_path=self.llm_deck_path, pattern="*.toml", is_recursive=True)]+ llm_deck_paths = [str(path) for path in find_files_in_dir(dir_path=self.llm_deck_dir_path, pattern="*.toml", is_recursive=True)]llm_deck_paths.sort()return llm_deck_pathsdef get_templates_paths(self) -> List[str]:- return [str(path) for path in find_files_in_dir(dir_path=self.templates_path, pattern="*.toml", is_recursive=True)]+ return [str(path) for path in find_files_in_dir(dir_path=self.templates_dir_path, pattern="*.toml", is_recursive=True)]def get_default_plugin_config_path(self) -> str:"""Get the default plugin config path."""- return f"{self.plugins_path}/plugin_config.toml"+ return f"{self.plugins_dir_path}/plugin_config.toml"def export_libraries(self, overwrite: bool = False) -> None:"""Duplicate pipelex libraries files in the client project, preserving directory structure."""@@ -64,32 +64,32 @@ class LibraryConfig(ConfigModel):copy_folder_from_package(package_name=self.package_name,folder_path_in_package=f"{PIPELEX_LIBRARIES_PATH}/pipelines",- target_dir=self.base_pipelines_path,+ target_dir=self.base_pipelines_dir_path,overwrite=overwrite,)copy_file_from_package(package_name=self.package_name,file_path_in_package=f"{PIPELEX_LIBRARIES_PATH}/__init__.py",- target_path=f"{self.config_folder_path}/__init__.py",+ target_path=f"{self.config_dir_path}/__init__.py",overwrite=overwrite,)copy_file_from_package(package_name=self.package_name,file_path_in_package=f"{PIPELEX_LIBRARIES_PATH}/pipelines/__init__.py",- target_path=f"{self.pipelines_path}/__init__.py",+ target_path=f"{self.pipelines_dir_path}/__init__.py",overwrite=overwrite,)copy_file_from_package(package_name=self.package_name,file_path_in_package=f"{PIPELEX_LIBRARIES_PATH}/pipelines/__init__.py",- target_path=f"{self.base_pipelines_path}/__init__.py",+ target_path=f"{self.base_pipelines_dir_path}/__init__.py",overwrite=overwrite,)# llm_integrationscopy_folder_from_package(package_name=self.package_name,folder_path_in_package=f"{PIPELEX_LIBRARIES_PATH}/llm_integrations",- target_dir=self.llm_integrations_path,+ target_dir=self.llm_integrations_dir_path,overwrite=overwrite,)@@ -97,7 +97,7 @@ class LibraryConfig(ConfigModel):copy_folder_from_package(package_name=self.package_name,folder_path_in_package=f"{PIPELEX_LIBRARIES_PATH}/llm_deck",- target_dir=self.llm_deck_path,+ target_dir=self.llm_deck_dir_path,overwrite=overwrite,non_overwrite_files=["overrides.toml"],)@@ -106,7 +106,7 @@ class LibraryConfig(ConfigModel):copy_folder_from_package(package_name=self.package_name,folder_path_in_package=f"{PIPELEX_LIBRARIES_PATH}/templates",- target_dir=self.templates_path,+ target_dir=self.templates_dir_path,overwrite=overwrite,)@@ -114,6 +114,6 @@ class LibraryConfig(ConfigModel):copy_folder_from_package(package_name=self.package_name,folder_path_in_package=f"{PIPELEX_LIBRARIES_PATH}/plugins",- target_dir=self.plugins_path,+ target_dir=self.plugins_dir_path,overwrite=overwrite,)diff --git a/pipelex/libraries/library_manager.py b/pipelex/libraries/library_manager.pyindex e3fd2435a..a72e32048 100644--- a/pipelex/libraries/library_manager.py+++ b/pipelex/libraries/library_manager.py@@ -1,44 +1,32 @@-import importlibimport osfrom pathlib import Pathfrom typing import Any, ClassVar, Dict, List, Optional, Type-from kajson.exceptions import ClassRegistryInheritanceError, ClassRegistryNotFoundError-from kajson.kajson_manager import KajsonManager-from pydantic import ValidationErrorfrom typing_extensions import overridefrom pipelex import logfrom pipelex.cogt.llm.llm_models.llm_deck import LLMDeckfrom pipelex.config import get_config-from pipelex.core.concept_factory import ConceptFactory-from pipelex.core.concept_library import ConceptLibrary-from pipelex.core.domain import Domain-from pipelex.core.domain_library import DomainLibrary-from pipelex.core.pipe_abstract import PipeAbstract-from pipelex.core.pipe_blueprint import PipeBlueprint, PipeSpecificFactoryProtocol-from pipelex.core.pipe_library import PipeLibrary+from pipelex.core.bundles.pipelex_bundle import PipelexBundle+from pipelex.core.bundles.pipelex_bundle_factory import PipelexBundleFactory+from pipelex.core.concepts.concept_factory import ConceptFactory+from pipelex.core.concepts.concept_library import ConceptLibrary+from pipelex.core.domains.domain_library import DomainLibrary+from pipelex.core.interpreter import PipelexInterpreter+from pipelex.core.pipes.pipe_abstract import PipeAbstract+from pipelex.core.pipes.pipe_library import PipeLibraryfrom pipelex.exceptions import (ConceptLibraryError,LibraryError,- PipeFactoryError,PipeLibraryError,- StaticValidationError,)from pipelex.libraries.library_config import LibraryConfigfrom pipelex.libraries.library_manager_abstract import LibraryManagerAbstract-from pipelex.libraries.pipeline_blueprint import (- ConceptBlueprintError,- PipeBlueprintError,- PipelineBlueprint,- PipelineBlueprintValidationError,-)from pipelex.tools.class_registry_utils import ClassRegistryUtilsfrom pipelex.tools.misc.file_utils import find_files_in_dirfrom pipelex.tools.misc.json_utils import deep_updatefrom pipelex.tools.misc.toml_utils import TOMLValidationError, load_toml_from_path, validate_toml_filefrom pipelex.tools.runtime_manager import runtime_manager-from pipelex.tools.typing.pydantic_utils import format_pydantic_validation_errorfrom pipelex.types import StrEnum@@ -67,210 +55,24 @@ class LibraryManager(LibraryManagerAbstract):"system_prompt_jto_structure","prompt_template_to_structure",]- domain_library: DomainLibrary- concept_library: ConceptLibrary- pipe_library: PipeLibrary- llm_deck: Optional[LLMDeck] = None- library_config: ClassVar[LibraryConfig]-- @classmethod- def make_empty(cls, config_folder_path: str) -> "LibraryManager":- cls.domain_library = DomainLibrary.make_empty()- cls.concept_library = ConceptLibrary.make_empty()- cls.pipe_library = PipeLibrary.make_empty()- cls.library_config = LibraryConfig(config_folder_path=config_folder_path)- return cls()-- @classmethod- def make(- cls, domain_library: DomainLibrary, concept_library: ConceptLibrary, pipe_library: PipeLibrary, config_folder_path: str- ) -> "LibraryManager":- cls.domain_library = domain_library- cls.concept_library = concept_library- cls.pipe_library = pipe_library- cls.library_config = LibraryConfig(config_folder_path=config_folder_path)- return cls()-- @override- def get_plugin_config_path(self) -> str:- return self.library_config.get_default_plugin_config_path()-- @override- def setup(self) -> None:- pass-- @override- def teardown(self) -> None:- self.llm_deck = None- self.pipe_library.teardown()- self.concept_library.teardown()- self.domain_library.teardown()-- def libraries_paths(self) -> List[str]:- library_paths = [self.library_config.pipelines_path]- if runtime_manager.is_unit_testing:- log.debug("Registering test pipeline structures for unit testing")- library_paths += [self.library_config.test_pipelines_path]- return library_paths-- def load_failure_modes(self):- failing_pipelines_path = get_config().pipelex.library_config.failing_pipelines_path- self.load_combo_libraries(library_paths=[Path(failing_pipelines_path)])-- def load_libraries(self):- log.debug("LibraryManager loading separate libraries")- library_paths = self.libraries_paths()- # self._validate_toml_files()- for library_path in library_paths:- ClassRegistryUtils.register_classes_in_folder(- folder_path=library_path,- )-- native_concepts = ConceptFactory.list_native_concepts()- self.concept_library.add_concepts(concepts=native_concepts)-- toml_file_paths = self.list_toml_files_from_path(library_paths=library_paths)- # remove failing_pipelines_path from the list- failing_pipelines_path = get_config().pipelex.library_config.failing_pipelines_path- toml_file_paths = [path for path in toml_file_paths if path != Path(failing_pipelines_path)]- self.load_combo_libraries(library_paths=toml_file_paths)-- def load_deck(self) -> LLMDeck:- llm_deck_paths = self.library_config.get_llm_deck_paths()- full_llm_deck_dict: Dict[str, Any] = {}- if not llm_deck_paths:- raise LLMDeckNotFoundError("No LLM deck paths found. Please run `pipelex init-libraries` to create it.")-- for llm_deck_path in llm_deck_paths:- if not os.path.exists(llm_deck_path):- raise LLMDeckNotFoundError(f"LLM deck path `{llm_deck_path}` not found. Please run `pipelex init-libraries` to create it.")- try:- llm_deck_dict = load_toml_from_path(path=llm_deck_path)- log.debug(f"Loaded LLM deck from {llm_deck_path}")- deep_update(full_llm_deck_dict, llm_deck_dict)- except Exception as exc:- log.error(f"Failed to load LLM deck file '{llm_deck_path}': {exc}")- raise-- self.llm_deck = LLMDeck.model_validate(full_llm_deck_dict)- return self.llm_deck-- def list_toml_files_from_path(self, library_paths: List[str]) -> List[Path]:- toml_file_paths: List[Path] = []- for libraries_path in library_paths:- # Use the existing utility function specifically for TOML files- found_file_paths = find_files_in_dir(- dir_path=libraries_path,- pattern="*.toml",- is_recursive=True,- )- log.debug(f"Searching for TOML files in {libraries_path}, found '{found_file_paths}'")- if not found_file_paths:- log.warning(f"No TOML files found in library path: {libraries_path}")- toml_file_paths.extend(found_file_paths)- return toml_file_paths- @override- def load_combo_libraries(self, library_paths: List[Path]):- log.debug("LibraryManager loading combo libraries")- # Find all .toml files in the directories and their subdirectories-- # First pass: load all domains- for toml_path in library_paths:- blueprint = self._load_blueprint_from_file(toml_path)- domain = Domain(- code=blueprint.domain,- definition=blueprint.definition,- system_prompt=blueprint.system_prompt,- system_prompt_to_structure=blueprint.system_prompt_to_structure,- prompt_template_to_structure=blueprint.prompt_template_to_structure,- )- self.domain_library.add_domain_details(domain=domain)-- # Second pass: load all concepts- for toml_path in library_paths:- nb_concepts_before = len(self.concept_library.root)- blueprint = self._load_blueprint_from_file(toml_path)- self._load_concepts_from_blueprint(blueprint=blueprint, file_path=str(toml_path))- nb_concepts_loaded = len(self.concept_library.root) - nb_concepts_before- log.verbose(f"Loaded {nb_concepts_loaded} concepts from '{toml_path.name}'")-- # Third pass: load all pipes- for toml_path in library_paths:- nb_pipes_before = len(self.pipe_library.root)- blueprint = self._load_blueprint_from_file(toml_path)- self._load_pipes_from_blueprint(blueprint=blueprint, file_path=str(toml_path))- nb_pipes_loaded = len(self.pipe_library.root) - nb_pipes_before- log.verbose(f"Loaded {nb_pipes_loaded} pipes from '{toml_path.name}'")-- def _load_blueprint_from_file(self, toml_path: Path) -> PipelineBlueprint:- """Load and validate a pipeline blueprint from a TOML file."""- try:- toml_data = load_toml_from_path(path=str(toml_path))- blueprint = PipelineBlueprint.model_validate(toml_data)- return blueprint- except ValidationError as exc:- error_msg = format_pydantic_validation_error(exc)- raise PipelineBlueprintValidationError(- file_path=str(toml_path),- validation_error_msg=error_msg,- ) from exc- except Exception as exc:- raise LibraryError(f"Failed to load TOML file '{toml_path}': {exc}") from exc-- def _load_concepts_from_blueprint(self, blueprint: PipelineBlueprint, file_path: str):- """Load concepts from a validated blueprint."""- for concept_name, concept_data in blueprint.concept.items():- try:- if isinstance(concept_data, str):- # Simple string definition- concept_from_def = ConceptFactory.make_concept_from_definition_str(- domain_code=blueprint.domain,- concept_str=concept_name,- definition=concept_data,- )- self.concept_library.add_new_concept(concept=concept_from_def)- else:- # Complex ConceptBlueprint - guaranteed by blueprint schema- concept_from_blueprint = ConceptFactory.make_concept_from_blueprint(- domain=blueprint.domain,- code=concept_name,- concept_blueprint=concept_data,- )- self.concept_library.add_new_concept(concept=concept_from_blueprint)- except ValidationError as exc:- error_msg = format_pydantic_validation_error(exc)- raise ConceptBlueprintError(- file_path=file_path,- concept_name=concept_name,- error_msg=error_msg,- ) from exc-- def _load_pipes_from_blueprint(self, blueprint: PipelineBlueprint, file_path: str):- """Load pipes from a validated blueprint."""- for pipe_name, pipe_data in blueprint.pipe.items():- try:- # pipe_data is guaranteed to be Dict[str, Any] by the blueprint schema- pipe = LibraryManager.make_pipe_from_blueprint(- domain_code=blueprint.domain,- pipe_code=pipe_name,- details_dict=pipe_data.copy(),- )- self.pipe_library.add_new_pipe(pipe=pipe)- except ValidationError as validation_error:- error_msg = format_pydantic_validation_error(validation_error)- raise PipeBlueprintError(- file_path=file_path,- pipe_name=pipe_name,- error_msg=error_msg,- ) from validation_error- except StaticValidationError as static_validation_error:- static_validation_error.file_path = file_path- raise static_validation_error+ def __init__(+ self,+ domain_library: DomainLibrary,+ concept_library: ConceptLibrary,+ pipe_library: PipeLibrary,+ library_config: LibraryConfig,+ ):+ self.domain_library = domain_library+ self.concept_library = concept_library+ self.pipe_library = pipe_library+ self.library_config = library_config+ self.llm_deck: Optional[LLMDeck] = None@overridedef validate_libraries(self):log.debug("LibraryManager validating libraries")+if self.llm_deck is None:raise LibraryError("LLM deck is not loaded")@@ -284,6 +86,7 @@ class LibraryManager(LibraryManagerAbstract):"""Validate all TOML files used by the library manager for formatting issues."""log.debug("LibraryManager validating TOML file formatting")+ # Validation of LLM deck pathsllm_deck_paths = self.library_config.get_llm_deck_paths()for llm_deck_path in llm_deck_paths:if os.path.exists(llm_deck_path):@@ -293,25 +96,7 @@ class LibraryManager(LibraryManagerAbstract):log.error(f"TOML formatting issues in LLM deck file '{llm_deck_path}': {exc}")raise LibraryError(f"TOML validation failed for LLM deck file '{llm_deck_path}': {exc}") from exc- # Validate pipeline library TOML files (same pattern as _load_combo_libraries)- library_paths = self.libraries_paths()- toml_file_paths: List[Path] = []- for libraries_path in library_paths:- if os.path.exists(libraries_path):- found_file_paths = find_files_in_dir(- dir_path=libraries_path,- pattern="*.toml",- is_recursive=True,- )- toml_file_paths.extend(found_file_paths)-- for toml_path in toml_file_paths:- try:- validate_toml_file(str(toml_path))- except TOMLValidationError as exc:- log.error(f"TOML formatting issues in library file '{toml_path}': {exc}")- raise LibraryError(f"TOML validation failed for library file '{toml_path}': {exc}") from exc-+ # Validation of template pathstemplate_paths = self.library_config.get_templates_paths()for template_path in template_paths:if os.path.exists(template_path):@@ -321,181 +106,111 @@ class LibraryManager(LibraryManagerAbstract):log.error(f"TOML formatting issues in template file '{template_path}': {exc}")raise LibraryError(f"TOML validation failed for template file '{template_path}': {exc}") from exc- @classmethod- def make_pipe_from_blueprint(- cls,- domain_code: str,- pipe_code: str,- details_dict: Dict[str, Any],- ) -> PipeAbstract:- pipe_definition: str- pipe_class_name: str- if "type" in details_dict and "definition" in details_dict:- # New format: type = "PipeClassName" and definition = "description"- pipe_class_name = details_dict.pop("type") # Remove type from details_dict- pipe_definition = details_dict["definition"] # Keep definition for the factory- else:- # TODO(DEPRECATED_REMOVAL): Remove old pipe syntax support - scheduled for removal in v0.3.0- # Fallback to old format for backward compatibility:- # PipeClassName = "the pipe's definition in natural language"- try:- pipe_class_name, pipe_definition = next(iter(details_dict.items()))- log.warning(f"""Pipe '{pipe_code}' uses deprecated syntax. Please migrate to new format:-replace this syntax:-```-{pipe_class_name} = "{pipe_definition}"-```-by this:-```-type = "{pipe_class_name}"-definition = "{pipe_definition}"-```-Old syntax will be removed in v0.3.0.- """)- details_dict.pop(pipe_class_name)- except StopIteration as details_dict_empty_error:- raise PipeFactoryError(f"Pipe '{pipe_code}' could not be created because its blueprint is empty.") from details_dict_empty_error-- # the factory class name for that specific type of Pipe is the pipe class name with "Factory" suffix- factory_class_name = f"{pipe_class_name}Factory"- try:- pipe_factory: Type[PipeSpecificFactoryProtocol[Any, Any]] = KajsonManager.get_class_registry().get_required_subclass(- name=factory_class_name,- base_class=PipeSpecificFactoryProtocol,- )- except ClassRegistryNotFoundError as factory_not_found_error:- raise PipeFactoryError(- f"Pipe '{pipe_code}' couldn't be created: factory '{factory_class_name}' not found: {factory_not_found_error}"- ) from factory_not_found_error- except ClassRegistryInheritanceError as factory_inheritance_error:- raise PipeFactoryError(- f"Pipe '{pipe_code}' couldn't be created: factory '{factory_class_name}' is not a subclass of {type(PipeSpecificFactoryProtocol)}."- ) from factory_inheritance_error-- details_dict["definition"] = pipe_definition- details_dict["domain"] = domain_code- pipe_from_blueprint: PipeAbstract = pipe_factory.make_pipe_from_details_dict(- domain_code=domain_code,- pipe_code=pipe_code,- details_dict=details_dict,- )- return pipe_from_blueprint-- @classmethod- def make_pipe_from_details_dict(- cls,- domain_code: str,- pipe_code: str,- details_dict: Dict[str, Any],- ) -> PipeAbstract:- # Delegate to the new make_pipe_from_blueprint method- return cls.make_pipe_from_blueprint(- domain_code=domain_code,- pipe_code=pipe_code,- details_dict=details_dict,- )-- @classmethod- def load_pipe_from_blueprint(- cls,- pipe_code: str,- pipe_blueprint: PipeBlueprint,- ) -> PipeAbstract:- """Create a Pipe from a concrete PipeBlueprint instance.-- This resolves the corresponding factory by converting the blueprint class name- (e.g. "PipeLLMBlueprint") to its factory name ("PipeLLMFactory").- """- # The blueprint must be a concrete subclass like PipeLLMBlueprint, PipeOcrBlueprint, etc.- blueprint_class_name = type(pipe_blueprint).__name__- if blueprint_class_name == "PipeBlueprint":- raise PipeFactoryError("Cannot load pipe from base PipeBlueprint. Please provide a specific blueprint subclass (e.g. PipeLLMBlueprint).")-- # Derive factory name: PipeLLMBlueprint -> PipeLLMFactory- factory_class_name = blueprint_class_name.replace("Blueprint", "Factory")-- try:- pipe_factory: Type[PipeSpecificFactoryProtocol[Any, Any]] = KajsonManager.get_class_registry().get_required_subclass(- name=factory_class_name,- base_class=PipeSpecificFactoryProtocol,+ @override+ def setup(self) -> None:+ native_concepts = ConceptFactory.list_native_concepts()+ self.concept_library.add_concepts(concepts=native_concepts)++ @override+ def teardown(self) -> None:+ self.llm_deck = None+ self.pipe_library.teardown()+ self.concept_library.teardown()+ self.domain_library.teardown()++ @override+ def reset(self) -> None:+ self.teardown()+ self.setup()++ def _get_pipeline_library_dirs(self) -> List[Path]:+ library_dirs = [Path(self.library_config.pipelines_dir_path)]+ if runtime_manager.is_unit_testing:+ log.debug("Registering test pipeline structures for unit testing")+ library_dirs += [Path(self.library_config.test_pipelines_dir_path)]+ return library_dirs++ def _get_pipelex_toml_files_from_dirs(self, dirs: List[Path]) -> List[Path]:+ """Get all valid Pipelex TOML files from the given directories."""++ all_toml_paths: List[Path] = []+ for dir_path in dirs:+ if not dir_path.exists():+ raise LibraryError(f"Directory does not exist: {dir_path}")++ # Find all TOML files in the directory+ toml_files = find_files_in_dir(+ dir_path=str(dir_path),+ pattern="*.toml",+ is_recursive=True,)- except ClassRegistryNotFoundError as factory_not_found_error:- raise PipeFactoryError(- f"Pipe '{pipe_code}' couldn't be created: factory '{factory_class_name}' not found: {factory_not_found_error}"- ) from factory_not_found_error- except ClassRegistryInheritanceError as factory_inheritance_error:- raise PipeFactoryError(- f"Pipe '{pipe_code}' couldn't be created: factory '{factory_class_name}' is not a subclass of {type(PipeSpecificFactoryProtocol)}."- ) from factory_inheritance_error-- # Domain is part of the blueprint- domain_code = pipe_blueprint.domain-- # Let the specific factory build the concrete Pipe instance- pipe_from_blueprint: PipeAbstract = pipe_factory.make_pipe_from_blueprint(- domain_code=domain_code,- pipe_code=pipe_code,- pipe_blueprint=pipe_blueprint, # type: ignore[arg-type]- )- return pipe_from_blueprint-- @classmethod- def make_pipe_blueprint_from_details(- cls,- domain_code: str,- details_dict: Dict[str, Any],- ) -> PipeBlueprint:- """Create a concrete PipeBlueprint subclass instance from details.-- This resolves the factory based on the provided details (supports both- new and legacy formats), locates the corresponding <PipeClassName>Blueprint- in the factory module, and validates the blueprint.- """- log.dev("Making Pipe blueprint")- # Determine pipe class name from details- if "type" in details_dict and "definition" in details_dict:- pipe_class_name = details_dict.pop("type")- normalized_details = details_dict.copy()- log.dev(f"New format, pipe_class_name: {pipe_class_name}")- log.dev(f"Normalized details: {normalized_details}")- else:++ # Filter to only include valid Pipelex files+ for toml_file in toml_files:+ if PipelexInterpreter.is_pipelex_file(toml_file):+ all_toml_paths.append(toml_file)+ else:+ log.debug(f"Skipping non-Pipelex TOML file: {toml_file}")++ return all_toml_paths++ @override+ def load_from_file(self, toml_path: Path) -> None:+ if not PipelexInterpreter.is_pipelex_file(toml_path):+ raise LibraryError(f"File is not a valid Pipelex TOML file: {toml_path}")++ converter = PipelexInterpreter(file_path=toml_path)+ blueprint = converter.make_pipelex_bundle_blueprint()+ pipelex_bundle = PipelexBundleFactory.make_from_blueprint(blueprint=blueprint)+ self.load_from_pipelex_bundle(pipelex_bundle=pipelex_bundle)++ @override+ def load_from_pipelex_bundle(self, pipelex_bundle: PipelexBundle) -> List[PipeAbstract]:+ self.domain_library.add_domain(domain=pipelex_bundle.domain)+ self.concept_library.add_concepts(concepts=list(pipelex_bundle.concepts.values()))+ self.pipe_library.add_pipes(pipes=list(pipelex_bundle.pipes.values()))+ return list(pipelex_bundle.pipes.values())++ @override+ def load_libraries(self, library_dirs: Optional[List[Path]] = None, library_file_paths: Optional[List[Path]] = None) -> None:+ dirs_to_use: List[Path] = self._get_pipeline_library_dirs()+ all_toml_paths: List[Path] = self._get_pipelex_toml_files_from_dirs(dirs_to_use)++ # Remove failing pipelines from the list+ failing_pipelines_file_paths = get_config().pipelex.library_config.failing_pipelines_file_paths+ failing_paths_set = {Path(fp) for fp in failing_pipelines_file_paths}+ all_toml_paths = [path for path in all_toml_paths if path not in failing_paths_set]++ if library_dirs is not None:+ dirs_to_use = library_dirs++ # Register classes in the directories+ for library_dir in dirs_to_use:+ ClassRegistryUtils.register_classes_in_folder(folder_path=str(library_dir))++ if library_file_paths is not None:+ all_toml_paths = library_file_paths++ for toml_file_path in all_toml_paths:+ self.load_from_file(toml_path=toml_file_path)++ # TODO: move to LLMDeckManager+ def load_deck(self) -> LLMDeck:+ llm_deck_paths = self.library_config.get_llm_deck_paths()+ full_llm_deck_dict: Dict[str, Any] = {}+ if not llm_deck_paths:+ raise LLMDeckNotFoundError("No LLM deck paths found. Please run `pipelex init-libraries` to create it.")++ for llm_deck_path in llm_deck_paths:+ if not os.path.exists(llm_deck_path):+ raise LLMDeckNotFoundError(f"LLM deck path `{llm_deck_path}` not found. Please run `pipelex init-libraries` to create it.")try:- pipe_class_name, legacy_definition = next(iter(details_dict.items()))- normalized_details = {"definition": legacy_definition}- log.dev(f"Legacy format, pipe_class_name: {pipe_class_name}")- log.dev(f"Normalized details: {normalized_details}")- except StopIteration as empty_err:- raise PipeFactoryError("Pipe details are empty; cannot determine pipe type") from empty_err-- factory_class_name = f"{pipe_class_name}Factory"- log.dev(f"Factory class name: {factory_class_name}")-- # Resolve factory to locate its module (where the Blueprint class is defined)- try:- pipe_factory: Type[PipeSpecificFactoryProtocol[Any, Any]] = KajsonManager.get_class_registry().get_required_subclass(- name=factory_class_name,- base_class=PipeSpecificFactoryProtocol,- )- except ClassRegistryNotFoundError as factory_not_found_error:- raise PipeFactoryError(- f"Factory '{factory_class_name}' not found for pipe type '{pipe_class_name}': {factory_not_found_error}"- ) from factory_not_found_error- except ClassRegistryInheritanceError as factory_inheritance_error:- raise PipeFactoryError(- f"Factory '{factory_class_name}' is not a subclass of {type(PipeSpecificFactoryProtocol)}."- ) from factory_inheritance_error-- factory_module = importlib.import_module(pipe_factory.__module__)- blueprint_class_name = f"{pipe_class_name}Blueprint"- log.dev(f"Blueprint class name: {blueprint_class_name}")-- try:- blueprint_cls: Type[PipeBlueprint] = getattr(factory_module, blueprint_class_name)- except AttributeError as exc:- raise PipeFactoryError(f"Cannot find blueprint class '{blueprint_class_name}' in module '{pipe_factory.__module__}'") from exc-- details_with_domain = normalized_details.copy()- details_with_domain["domain"] = domain_code-- log.dev(f"Making Pipe blueprint for class {blueprint_class_name} from details: {details_with_domain}")- return blueprint_cls.model_validate(details_with_domain)+ llm_deck_dict = load_toml_from_path(path=llm_deck_path)+ log.debug(f"Loaded LLM deck from {llm_deck_path}")+ deep_update(full_llm_deck_dict, llm_deck_dict)+ except Exception as exc:+ log.error(f"Failed to load LLM deck file '{llm_deck_path}': {exc}")+ raise++ self.llm_deck = LLMDeck.model_validate(full_llm_deck_dict)+ return self.llm_deckdiff --git a/pipelex/libraries/library_manager_abstract.py b/pipelex/libraries/library_manager_abstract.pyindex 7f1d6553f..de1ec0afa 100644--- a/pipelex/libraries/library_manager_abstract.py+++ b/pipelex/libraries/library_manager_abstract.py@@ -1,6 +1,9 @@from abc import ABC, abstractmethodfrom pathlib import Path-from typing import List+from typing import List, Optional++from pipelex.core.bundles.pipelex_bundle import PipelexBundle+from pipelex.core.pipes.pipe_abstract import PipeAbstractclass LibraryManagerAbstract(ABC):@@ -13,13 +16,21 @@ class LibraryManagerAbstract(ABC):pass@abstractmethod- def load_combo_libraries(self, library_paths: List[Path]):+ def reset(self) -> None:pass@abstractmethod- def get_plugin_config_path(self) -> str:+ def validate_libraries(self) -> None:pass@abstractmethod- def validate_libraries(self) -> None:+ def load_libraries(self, library_dirs: Optional[List[Path]] = None, library_file_paths: Optional[List[Path]] = None) -> None:+ pass++ @abstractmethod+ def load_from_file(self, toml_path: Path) -> None:+ pass++ @abstractmethod+ def load_from_pipelex_bundle(self, pipelex_bundle: PipelexBundle) -> List[PipeAbstract]:passdiff --git a/pipelex/libraries/library_manager_factory.py b/pipelex/libraries/library_manager_factory.pynew file mode 100644index 000000000..686b31f29--- /dev/null+++ b/pipelex/libraries/library_manager_factory.py@@ -0,0 +1,38 @@+from pipelex.core.concepts.concept_library import ConceptLibrary+from pipelex.core.domains.domain_library import DomainLibrary+from pipelex.core.pipes.pipe_library import PipeLibrary+from pipelex.libraries.library_config import LibraryConfig+from pipelex.libraries.library_manager import LibraryManager+++class LibraryManagerFactory:+ """Factory for creating LibraryManager instances."""++ @classmethod+ def make_empty(cls, config_dir_path: str) -> "LibraryManager":+ domain_library = DomainLibrary.make_empty()+ concept_library = ConceptLibrary.make_empty()+ pipe_library = PipeLibrary.make_empty()+ library_config = LibraryConfig(config_dir_path=config_dir_path)++ return LibraryManager(+ domain_library=domain_library,+ concept_library=concept_library,+ pipe_library=pipe_library,+ library_config=library_config,+ )++ @classmethod+ def make(+ cls, domain_library: DomainLibrary, concept_library: ConceptLibrary, pipe_library: PipeLibrary, config_dir_path: str+ ) -> "LibraryManager":+ """Create a LibraryManager with provided libraries."""++ library_config = LibraryConfig(config_dir_path=config_dir_path)++ return LibraryManager(+ domain_library=domain_library,+ concept_library=concept_library,+ pipe_library=pipe_library,+ library_config=library_config,+ )diff --git a/pipelex/libraries/llm_deck/base_llm_deck.toml b/pipelex/libraries/llm_deck/base_llm_deck.tomlindex dc7329dba..726851f7e 100644--- a/pipelex/libraries/llm_deck/base_llm_deck.toml+++ b/pipelex/libraries/llm_deck/base_llm_deck.toml@@ -20,11 +20,11 @@ best-gemini = "gemini-2.5-pro"best-mistral = "mistral-large"best-grok = "grok-3"-cheap_llm_for_text = "gpt-5-nano"-cheap_llm_for_object = "gpt-5-nano"+cheap_llm_for_text = "gpt-4o-mini"+cheap_llm_for_object = "gpt-4o-mini"-base_llm_for_text = "gpt-5-nano"-base_llm_for_object = "gpt-5-nano"+base_llm_for_text = "gpt-4o-mini"+base_llm_for_object = "gpt-4o-mini"####################################################################################################diff --git a/pipelex/libraries/pipeline_blueprint.py b/pipelex/libraries/pipeline_blueprint.pydeleted file mode 100644index 9d485045c..000000000--- a/pipelex/libraries/pipeline_blueprint.py+++ /dev/null@@ -1,58 +0,0 @@-from typing import Any, Dict, Optional, Union--from pydantic import Field--from pipelex.core.concept_factory import ConceptBlueprint-from pipelex.core.stuff_content import StructuredContent-from pipelex.exceptions import LibraryError---class PipelineBlueprintValidationError(LibraryError):- """Error raised when a pipeline blueprint fails validation."""-- def __init__(self, file_path: str, validation_error_msg: str):- self.file_path = file_path- self.validation_error_msg = validation_error_msg- super().__init__(f"Pipeline blueprint validation failed for '{file_path}': {validation_error_msg}")---class ConceptBlueprintError(LibraryError):- """Error raised when loading concept blueprints."""-- def __init__(self, file_path: str, concept_name: str, error_msg: str):- self.file_path = file_path- self.concept_name = concept_name- super().__init__(f"Error loading concept '{concept_name}' from '{file_path}': {error_msg}")---class PipeBlueprintError(LibraryError):- """Error raised when loading pipe blueprints."""-- def __init__(self, file_path: str, pipe_name: str, error_msg: str):- self.file_path = file_path- self.pipe_name = pipe_name- super().__init__(f"Error loading pipe '{pipe_name}' from '{file_path}': {error_msg}")---class PipelineBlueprint(StructuredContent):- """Complete blueprint of a pipeline TOML definition."""-- # Domain information (required)- domain: str- definition: Optional[str] = None- system_prompt: Optional[str] = None- system_prompt_to_structure: Optional[str] = None- prompt_template_to_structure: Optional[str] = None-- # Concepts section - concept_name -> definition (string) or blueprint (dict)- concept: Dict[str, Union[str, ConceptBlueprint]] = Field(default_factory=dict)-- # Pipes section - pipe_name -> blueprint dict- pipe: Dict[str, Dict[str, Any]] = Field(default_factory=dict)-- @property- def desc(self) -> str:- if self.definition:- return f"{self.domain} • {self.definition[:100]}"- else:- return self.domaindiff --git a/pipelex/libraries/pipelines/images.toml b/pipelex/libraries/pipelines/images.tomlindex 9d4197279..b46d0869a 100644--- a/pipelex/libraries/pipelines/images.toml+++ b/pipelex/libraries/pipelines/images.toml@@ -10,7 +10,6 @@ refines = "Text"[concept.Photo]definition = "Photo"-structure = "ImageContent"refines = "Image"[pipe]diff --git a/pipelex/libraries/pipelines/meta/meta_pipeline.toml b/pipelex/libraries/pipelines/meta/meta_pipeline.tomlindex 21d6d99d0..cbeb7ad44 100644--- a/pipelex/libraries/pipelines/meta/meta_pipeline.toml+++ b/pipelex/libraries/pipelines/meta/meta_pipeline.toml@@ -5,17 +5,19 @@ definition = "Meta domain is the domain of pipelines about pipelines"PipelineDraft = "Rough solution to create a pipeline"PipelineDraftText = "Rough solution to create a pipeline in text format"PipeDraft = "Rough solution to create a pipe"-PipelineBlueprint = "Details enabling to create a pipeline"ConceptBlueprint = "Details enabling to create a concept"PipeBlueprint = "Details enabling to create a pipe"-[pipe]+[concept.PipelexBundleBlueprint]+definition = "Details enabling to create a pipeline"+structure = "PipelexBundleBlueprintStuff"+[pipe][pipe.build_blueprint]type = "PipeSequence"definition = "Build a pipeline blueprint from natural language requirements"-inputs = { draft_pipeline_rules = "Text", build_pipeline_rules = "Text", requirements = "Text", pipeline_name = "Text", domain = "Text" }-output = "PipelineBlueprint"+inputs = { draft_pipeline_rules = "Text", build_pipeline_rules = "Text", create_structured_output_rules = "Text", requirements = "Text", pipeline_name = "Text", domain = "Text" }+output = "PipelexBundleBlueprint"steps = [{ pipe = "draft_pipeline_text", result = "pipeline_draft" },{ pipe = "structure_blueprint", result = "pipeline_blueprint" },@@ -24,7 +26,7 @@ steps = [[pipe.draft_pipeline_text]type = "PipeLLM"definition = "Generate a rough pipeline draft from natural language requirements"-inputs = { draft_pipeline_rules = "Text", requirements = "Text", pipeline_name = "Text", domain = "Text" }+inputs = { draft_pipeline_rules = "Text", create_structured_output_rules = "Text", requirements = "Text", pipeline_name = "Text", domain = "Text" }output = "PipelineDraftText"llm = "llm_to_pipelex"prompt_template = """@@ -32,6 +34,9 @@ You are a Pipelex pipeline architect. Build a pipeline draft text showing how we@draft_pipeline_rules+Do create structured output for better concept representation.++@create_structured_output_rules---Name of the pipeline: $pipeline_name.@@ -44,7 +49,7 @@ Requirements: $requirementstype = "PipeLLM"definition = "Generate a pipeline blueprint from natural language requirements"inputs = { build_pipeline_rules = "Text", pipeline_draft = "Text", pipeline_name = "Text", domain = "Text", requirements = "Text" }-output = "PipelineBlueprint"+output = "PipelexBundleBlueprint"llm = "llm_to_pipelex"# structuring_method = "preliminary_text"prompt_template = """diff --git a/pipelex/libraries/pipelines/meta/pipeline_draft.py b/pipelex/libraries/pipelines/meta/pipeline_draft.pyindex 26e5ab970..c756a9d6b 100644--- a/pipelex/libraries/pipelines/meta/pipeline_draft.py+++ b/pipelex/libraries/pipelines/meta/pipeline_draft.py@@ -2,7 +2,8 @@ from typing import Dictfrom pydantic import Field-from pipelex.core.stuff_content import StructuredContent+from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint+from pipelex.core.stuffs.stuff_content import StructuredContentclass PipeDraft(StructuredContent):@@ -25,3 +26,9 @@ class PipelineDraft(StructuredContent):# Pipes section - pipe_name -> blueprint dictpipe: Dict[str, PipeDraft] = Field(default_factory=dict)+++class PipelexBundleBlueprintStuff(PipelexBundleBlueprint, StructuredContent):+ """Complete blueprint of a pipelex bundle TOML file."""++ passdiff --git a/pipelex/pipe_controllers/pipe_batch.py b/pipelex/pipe_controllers/pipe_batch.pyindex 95d075f98..f7116e03f 100644--- a/pipelex/pipe_controllers/pipe_batch.py+++ b/pipelex/pipe_controllers/pipe_batch.py@@ -7,13 +7,13 @@ from typing_extensions import Self, overridefrom pipelex import logfrom pipelex.config import get_config-from pipelex.core.pipe_input_spec import PipeInputSpec-from pipelex.core.pipe_output import PipeOutput-from pipelex.core.pipe_run_params import BatchParams, PipeRunMode, PipeRunParams-from pipelex.core.stuff import Stuff-from pipelex.core.stuff_content import ListContent, StuffContent-from pipelex.core.stuff_factory import StuffFactory-from pipelex.core.working_memory import MAIN_STUFF_NAME, WorkingMemory+from pipelex.core.memory.working_memory import MAIN_STUFF_NAME, WorkingMemory+from pipelex.core.pipes.pipe_input_spec import PipeInputSpec+from pipelex.core.pipes.pipe_output import PipeOutput+from pipelex.core.pipes.pipe_run_params import BatchParams, PipeRunMode, PipeRunParams+from pipelex.core.stuffs.stuff import Stuff+from pipelex.core.stuffs.stuff_content import ListContent, StuffContent+from pipelex.core.stuffs.stuff_factory import StuffFactoryfrom pipelex.exceptions import (PipeInputError,PipeInputNotFoundError,diff --git a/pipelex/pipe_controllers/pipe_batch_factory.py b/pipelex/pipe_controllers/pipe_batch_factory.pyindex 82fee5325..7a4797f67 100644--- a/pipelex/pipe_controllers/pipe_batch_factory.py+++ b/pipelex/pipe_controllers/pipe_batch_factory.py@@ -1,21 +1,23 @@-from typing import Any, Dict, Optional+from typing import Literal, Optionalfrom typing_extensions import override-from pipelex.core.pipe_blueprint import PipeBlueprint, PipeSpecificFactoryProtocol-from pipelex.core.pipe_input_spec import PipeInputSpec-from pipelex.core.pipe_run_params import BatchParams+from pipelex.core.pipes.pipe_blueprint import PipeBlueprint+from pipelex.core.pipes.pipe_factory import PipeFactoryProtocol+from pipelex.core.pipes.pipe_input_spec import PipeInputSpec+from pipelex.core.pipes.pipe_run_params import BatchParamsfrom pipelex.pipe_controllers.pipe_batch import PipeBatchclass PipeBatchBlueprint(PipeBlueprint):+ type: Literal["PipeBatch"] = "PipeBatch"branch_pipe_code: strinput_list_name: Optional[str] = Noneinput_item_name: Optional[str] = None-class PipeBatchFactory(PipeSpecificFactoryProtocol[PipeBatchBlueprint, PipeBatch]):+class PipeBatchFactory(PipeFactoryProtocol[PipeBatchBlueprint, PipeBatch]):@classmethod@overridedef make_pipe_from_blueprint(@@ -32,23 +34,8 @@ class PipeBatchFactory(PipeSpecificFactoryProtocol[PipeBatchBlueprint, PipeBatchdomain=domain_code,code=pipe_code,definition=pipe_blueprint.definition,- inputs=PipeInputSpec.make_from_dict(concepts_dict=pipe_blueprint.inputs or {}),+ inputs=PipeInputSpec.make_from_blueprint(domain=domain_code, blueprint=pipe_blueprint.inputs or {}),output_concept_code=pipe_blueprint.output,branch_pipe_code=pipe_blueprint.branch_pipe_code,batch_params=batch_params,)-- @classmethod- @override- def make_pipe_from_details_dict(- cls,- domain_code: str,- pipe_code: str,- details_dict: Dict[str, Any],- ) -> PipeBatch:- pipe_blueprint = PipeBatchBlueprint.model_validate(details_dict)- return cls.make_pipe_from_blueprint(- domain_code=domain_code,- pipe_code=pipe_code,- pipe_blueprint=pipe_blueprint,- )diff --git a/pipelex/pipe_controllers/pipe_condition.py b/pipelex/pipe_controllers/pipe_condition.pyindex 986da64bb..f0252a0ca 100644--- a/pipelex/pipe_controllers/pipe_condition.py+++ b/pipelex/pipe_controllers/pipe_condition.py@@ -6,10 +6,10 @@ from typing_extensions import Self, overridefrom pipelex import logfrom pipelex.config import StaticValidationReaction, get_config-from pipelex.core.pipe_input_spec import PipeInputSpec-from pipelex.core.pipe_output import PipeOutput-from pipelex.core.pipe_run_params import PipeRunParams-from pipelex.core.working_memory import WorkingMemory+from pipelex.core.memory.working_memory import WorkingMemory+from pipelex.core.pipes.pipe_input_spec import PipeInputSpec+from pipelex.core.pipes.pipe_output import PipeOutput+from pipelex.core.pipes.pipe_run_params import PipeRunParamsfrom pipelex.exceptions import (DryRunError,PipeConditionError,diff --git a/pipelex/pipe_controllers/pipe_condition_factory.py b/pipelex/pipe_controllers/pipe_condition_factory.pyindex 3578fde9e..75fae4fbd 100644--- a/pipelex/pipe_controllers/pipe_condition_factory.py+++ b/pipelex/pipe_controllers/pipe_condition_factory.py@@ -1,22 +1,25 @@-from typing import Any, Dict, Optional+from typing import Dict, Literal, Optionalfrom typing_extensions import override-from pipelex.core.pipe_blueprint import PipeBlueprint, PipeSpecificFactoryProtocol-from pipelex.core.pipe_input_spec import PipeInputSpec+from pipelex.core.pipes.pipe_blueprint import PipeBlueprint+from pipelex.core.pipes.pipe_factory import PipeFactoryProtocol+from pipelex.core.pipes.pipe_input_spec import PipeInputSpecfrom pipelex.pipe_controllers.pipe_condition import PipeConditionclass PipeConditionBlueprint(PipeBlueprint):+ type: Literal["PipeCondition"] = "PipeCondition"expression_template: Optional[str] = Noneexpression: Optional[str] = None# TODO: make the values of pipe_map a Union[str, PipeAdapter] or something to set a specific alias+ # TODO: Add a Blueprint for the pipe_mappipe_map: Dict[str, str]default_pipe_code: Optional[str] = Noneadd_alias_from_expression_to: Optional[str] = None-class PipeConditionFactory(PipeSpecificFactoryProtocol[PipeConditionBlueprint, PipeCondition]):+class PipeConditionFactory(PipeFactoryProtocol[PipeConditionBlueprint, PipeCondition]):@classmethod@overridedef make_pipe_from_blueprint(@@ -29,7 +32,7 @@ class PipeConditionFactory(PipeSpecificFactoryProtocol[PipeConditionBlueprint, Pdomain=domain_code,code=pipe_code,definition=pipe_blueprint.definition,- inputs=PipeInputSpec.make_from_dict(concepts_dict=pipe_blueprint.inputs or {}),+ inputs=PipeInputSpec.make_from_blueprint(domain=domain_code, blueprint=pipe_blueprint.inputs or {}),output_concept_code=pipe_blueprint.output,expression_template=pipe_blueprint.expression_template,expression=pipe_blueprint.expression,@@ -37,18 +40,3 @@ class PipeConditionFactory(PipeSpecificFactoryProtocol[PipeConditionBlueprint, Pdefault_pipe_code=pipe_blueprint.default_pipe_code,add_alias_from_expression_to=pipe_blueprint.add_alias_from_expression_to,)-- @classmethod- @override- def make_pipe_from_details_dict(- cls,- domain_code: str,- pipe_code: str,- details_dict: Dict[str, Any],- ) -> PipeCondition:- pipe_blueprint = PipeConditionBlueprint.model_validate(details_dict)- return cls.make_pipe_from_blueprint(- domain_code=domain_code,- pipe_code=pipe_code,- pipe_blueprint=pipe_blueprint,- )diff --git a/pipelex/pipe_controllers/pipe_controller.py b/pipelex/pipe_controllers/pipe_controller.pyindex de4c3aafd..344efc1b7 100644--- a/pipelex/pipe_controllers/pipe_controller.py+++ b/pipelex/pipe_controllers/pipe_controller.py@@ -4,10 +4,10 @@ from typing import Optionalfrom typing_extensions import overridefrom pipelex import log-from pipelex.core.pipe_abstract import PipeAbstract-from pipelex.core.pipe_output import PipeOutput-from pipelex.core.pipe_run_params import PipeRunMode, PipeRunParams-from pipelex.core.working_memory import WorkingMemory+from pipelex.core.memory.working_memory import WorkingMemory+from pipelex.core.pipes.pipe_abstract import PipeAbstract+from pipelex.core.pipes.pipe_output import PipeOutput+from pipelex.core.pipes.pipe_run_params import PipeRunMode, PipeRunParamsfrom pipelex.pipeline.job_metadata import JobMetadatadiff --git a/pipelex/pipe_controllers/pipe_parallel.py b/pipelex/pipe_controllers/pipe_parallel.pyindex fddac805d..9123ce048 100644--- a/pipelex/pipe_controllers/pipe_parallel.py+++ b/pipelex/pipe_controllers/pipe_parallel.py@@ -6,13 +6,13 @@ from typing_extensions import Self, overridefrom pipelex import logfrom pipelex.config import StaticValidationReaction, get_config-from pipelex.core.pipe_input_spec import PipeInputSpec-from pipelex.core.pipe_output import PipeOutput-from pipelex.core.pipe_run_params import PipeRunMode, PipeRunParams-from pipelex.core.stuff import Stuff-from pipelex.core.stuff_content import StuffContent-from pipelex.core.stuff_factory import StuffFactory-from pipelex.core.working_memory import WorkingMemory+from pipelex.core.memory.working_memory import WorkingMemory+from pipelex.core.pipes.pipe_input_spec import PipeInputSpec+from pipelex.core.pipes.pipe_output import PipeOutput+from pipelex.core.pipes.pipe_run_params import PipeRunMode, PipeRunParams+from pipelex.core.stuffs.stuff import Stuff+from pipelex.core.stuffs.stuff_content import StuffContent+from pipelex.core.stuffs.stuff_factory import StuffFactoryfrom pipelex.exceptions import DryRunError, PipeDefinitionError, PipeRunParamsError, StaticValidationError, StaticValidationErrorTypefrom pipelex.hub import get_pipeline_tracker, get_required_pipefrom pipelex.pipe_controllers.pipe_controller import PipeControllerdiff --git a/pipelex/pipe_controllers/pipe_parallel_factory.py b/pipelex/pipe_controllers/pipe_parallel_factory.pyindex 871e2ca0c..d2d25ceb3 100644--- a/pipelex/pipe_controllers/pipe_parallel_factory.py+++ b/pipelex/pipe_controllers/pipe_parallel_factory.py@@ -1,10 +1,11 @@-from typing import Any, Dict, List, Optional+from typing import List, Literal, Optionalfrom typing_extensions import override-from pipelex.core.concept import Concept-from pipelex.core.pipe_blueprint import PipeBlueprint, PipeSpecificFactoryProtocol-from pipelex.core.pipe_input_spec import PipeInputSpec+from pipelex.core.concepts.concept import Concept+from pipelex.core.pipes.pipe_blueprint import PipeBlueprint+from pipelex.core.pipes.pipe_factory import PipeFactoryProtocol+from pipelex.core.pipes.pipe_input_spec import PipeInputSpecfrom pipelex.exceptions import PipeDefinitionErrorfrom pipelex.hub import get_concept_providerfrom pipelex.pipe_controllers.pipe_parallel import PipeParallel@@ -13,12 +14,13 @@ from pipelex.pipe_controllers.sub_pipe_factory import SubPipeBlueprintclass PipeParallelBlueprint(PipeBlueprint):+ type: Literal["PipeParallel"] = "PipeParallel"parallels: List[SubPipeBlueprint]add_each_output: bool = Truecombined_output: Optional[str] = None-class PipeParallelFactory(PipeSpecificFactoryProtocol[PipeParallelBlueprint, PipeParallel]):+class PipeParallelFactory(PipeFactoryProtocol[PipeParallelBlueprint, PipeParallel]):@classmethod@overridedef make_pipe_from_blueprint(@@ -43,24 +45,9 @@ class PipeParallelFactory(PipeSpecificFactoryProtocol[PipeParallelBlueprint, Pipdomain=domain_code,code=pipe_code,definition=pipe_blueprint.definition,- inputs=PipeInputSpec.make_from_dict(concepts_dict=pipe_blueprint.inputs or {}),+ inputs=PipeInputSpec.make_from_blueprint(domain=domain_code, blueprint=pipe_blueprint.inputs or {}),output_concept_code=pipe_blueprint.output,parallel_sub_pipes=parallel_sub_pipes,add_each_output=pipe_blueprint.add_each_output,combined_output=pipe_blueprint.combined_output,)-- @classmethod- @override- def make_pipe_from_details_dict(- cls,- domain_code: str,- pipe_code: str,- details_dict: Dict[str, Any],- ) -> PipeParallel:- pipe_blueprint = PipeParallelBlueprint.model_validate(details_dict)- return cls.make_pipe_from_blueprint(- domain_code=domain_code,- pipe_code=pipe_code,- pipe_blueprint=pipe_blueprint,- )diff --git a/pipelex/pipe_controllers/pipe_sequence.py b/pipelex/pipe_controllers/pipe_sequence.pyindex e6a8f85a4..8d561d186 100644--- a/pipelex/pipe_controllers/pipe_sequence.py+++ b/pipelex/pipe_controllers/pipe_sequence.py@@ -5,10 +5,10 @@ from typing_extensions import Self, overridefrom pipelex import logfrom pipelex.config import StaticValidationReaction, get_config-from pipelex.core.pipe_input_spec import PipeInputSpec-from pipelex.core.pipe_output import PipeOutput-from pipelex.core.pipe_run_params import PipeRunMode, PipeRunParams-from pipelex.core.working_memory import WorkingMemory+from pipelex.core.memory.working_memory import WorkingMemory+from pipelex.core.pipes.pipe_input_spec import PipeInputSpec+from pipelex.core.pipes.pipe_output import PipeOutput+from pipelex.core.pipes.pipe_run_params import PipeRunMode, PipeRunParamsfrom pipelex.exceptions import (PipeRunParamsError,StaticValidationError,diff --git a/pipelex/pipe_controllers/pipe_sequence_factory.py b/pipelex/pipe_controllers/pipe_sequence_factory.pyindex ce5500121..e49b76865 100644--- a/pipelex/pipe_controllers/pipe_sequence_factory.py+++ b/pipelex/pipe_controllers/pipe_sequence_factory.py@@ -1,18 +1,20 @@-from typing import Any, Dict, List+from typing import List, Literalfrom typing_extensions import override-from pipelex.core.pipe_blueprint import PipeBlueprint, PipeSpecificFactoryProtocol-from pipelex.core.pipe_input_spec import PipeInputSpec+from pipelex.core.pipes.pipe_blueprint import PipeBlueprint+from pipelex.core.pipes.pipe_factory import PipeFactoryProtocol+from pipelex.core.pipes.pipe_input_spec import PipeInputSpecfrom pipelex.pipe_controllers.pipe_sequence import PipeSequencefrom pipelex.pipe_controllers.sub_pipe_factory import SubPipeBlueprintclass PipeSequenceBlueprint(PipeBlueprint):+ type: Literal["PipeSequence"] = "PipeSequence"steps: List[SubPipeBlueprint]-class PipeSequenceFactory(PipeSpecificFactoryProtocol[PipeSequenceBlueprint, PipeSequence]):+class PipeSequenceFactory(PipeFactoryProtocol[PipeSequenceBlueprint, PipeSequence]):@classmethod@overridedef make_pipe_from_blueprint(@@ -26,22 +28,7 @@ class PipeSequenceFactory(PipeSpecificFactoryProtocol[PipeSequenceBlueprint, Pipdomain=domain_code,code=pipe_code,definition=pipe_blueprint.definition,- inputs=PipeInputSpec.make_from_dict(concepts_dict=pipe_blueprint.inputs or {}),+ inputs=PipeInputSpec.make_from_blueprint(domain=domain_code, blueprint=pipe_blueprint.inputs or {}),output_concept_code=pipe_blueprint.output,sequential_sub_pipes=pipe_steps,)-- @classmethod- @override- def make_pipe_from_details_dict(- cls,- domain_code: str,- pipe_code: str,- details_dict: Dict[str, Any],- ) -> PipeSequence:- pipe_blueprint = PipeSequenceBlueprint.model_validate(details_dict)- return cls.make_pipe_from_blueprint(- domain_code=domain_code,- pipe_code=pipe_code,- pipe_blueprint=pipe_blueprint,- )diff --git a/pipelex/pipe_controllers/sub_pipe.py b/pipelex/pipe_controllers/sub_pipe.pyindex 0b32fb684..8e7372658 100644--- a/pipelex/pipe_controllers/sub_pipe.py+++ b/pipelex/pipe_controllers/sub_pipe.py@@ -3,9 +3,9 @@ from typing import Optionalfrom pydantic import BaseModelfrom pipelex import log-from pipelex.core.pipe_output import PipeOutput-from pipelex.core.pipe_run_params import BatchParams, PipeOutputMultiplicity, PipeRunMode, PipeRunParams-from pipelex.core.working_memory import WorkingMemory+from pipelex.core.memory.working_memory import WorkingMemory+from pipelex.core.pipes.pipe_output import PipeOutput+from pipelex.core.pipes.pipe_run_params import BatchParams, PipeOutputMultiplicity, PipeRunMode, PipeRunParamsfrom pipelex.exceptions import PipeInputError, WorkingMemoryStuffNotFoundErrorfrom pipelex.hub import get_pipe_router, get_pipeline_tracker, get_required_pipefrom pipelex.pipe_controllers.pipe_batch import PipeBatchdiff --git a/pipelex/pipe_controllers/sub_pipe_factory.py b/pipelex/pipe_controllers/sub_pipe_factory.pyindex 9e256e046..4991b49b6 100644--- a/pipelex/pipe_controllers/sub_pipe_factory.py+++ b/pipelex/pipe_controllers/sub_pipe_factory.py@@ -3,7 +3,7 @@ from typing import Optional, Unionfrom pydantic import BaseModel, ConfigDict, model_validatorfrom typing_extensions import Self-from pipelex.core.pipe_run_params import BatchParams, make_output_multiplicity+from pipelex.core.pipes.pipe_run_params import BatchParams, make_output_multiplicityfrom pipelex.exceptions import PipeDefinitionErrorfrom pipelex.pipe_controllers.sub_pipe import SubPipefrom pipelex.tools.typing.validation_utils import has_more_than_one_among_attributes_from_listdiff --git a/pipelex/pipe_operators/pipe_func.py b/pipelex/pipe_operators/pipe_func.pyindex 1656dfb58..5a803e6e3 100644--- a/pipelex/pipe_operators/pipe_func.py+++ b/pipelex/pipe_operators/pipe_func.py@@ -3,13 +3,13 @@ from typing import List, Optional, Set, cast, get_type_hintsfrom typing_extensions import overridefrom pipelex import log-from pipelex.core.pipe_input_spec import PipeInputSpec, TypedNamedInputRequirement-from pipelex.core.pipe_output import PipeOutput-from pipelex.core.pipe_run_params import PipeRunParams-from pipelex.core.stuff_content import ListContent, StuffContent, TextContent-from pipelex.core.stuff_factory import StuffFactory-from pipelex.core.working_memory import WorkingMemory-from pipelex.core.working_memory_factory import WorkingMemoryFactory+from pipelex.core.memory.working_memory import WorkingMemory+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory+from pipelex.core.pipes.pipe_input_spec import PipeInputSpec, TypedNamedInputRequirement+from pipelex.core.pipes.pipe_output import PipeOutput+from pipelex.core.pipes.pipe_run_params import PipeRunParams+from pipelex.core.stuffs.stuff_content import ListContent, StuffContent, TextContent+from pipelex.core.stuffs.stuff_factory import StuffFactoryfrom pipelex.exceptions import DryRunErrorfrom pipelex.pipe_operators.pipe_operator import PipeOperatorfrom pipelex.pipeline.job_metadata import JobMetadatadiff --git a/pipelex/pipe_operators/pipe_func_factory.py b/pipelex/pipe_operators/pipe_func_factory.pyindex c38ac6c47..c702acd44 100644--- a/pipelex/pipe_operators/pipe_func_factory.py+++ b/pipelex/pipe_operators/pipe_func_factory.py@@ -1,17 +1,19 @@-from typing import Any, Dict+from typing import Literalfrom typing_extensions import override-from pipelex.core.pipe_blueprint import PipeBlueprint, PipeSpecificFactoryProtocol-from pipelex.core.pipe_input_spec import PipeInputSpec+from pipelex.core.pipes.pipe_blueprint import PipeBlueprint+from pipelex.core.pipes.pipe_factory import PipeFactoryProtocol+from pipelex.core.pipes.pipe_input_spec import PipeInputSpecfrom pipelex.pipe_operators.pipe_func import PipeFuncclass PipeFuncBlueprint(PipeBlueprint):+ type: Literal["PipeFunc"] = "PipeFunc"function_name: str-class PipeFuncFactory(PipeSpecificFactoryProtocol[PipeFuncBlueprint, PipeFunc]):+class PipeFuncFactory(PipeFactoryProtocol[PipeFuncBlueprint, PipeFunc]):@classmethod@overridedef make_pipe_from_blueprint(@@ -24,22 +26,7 @@ class PipeFuncFactory(PipeSpecificFactoryProtocol[PipeFuncBlueprint, PipeFunc]):domain=domain_code,code=pipe_code,definition=pipe_blueprint.definition,- inputs=PipeInputSpec.make_from_dict(concepts_dict=pipe_blueprint.inputs or {}),+ inputs=PipeInputSpec.make_from_blueprint(domain=domain_code, blueprint=pipe_blueprint.inputs or {}),output_concept_code=pipe_blueprint.output,function_name=pipe_blueprint.function_name,)-- @classmethod- @override- def make_pipe_from_details_dict(- cls,- domain_code: str,- pipe_code: str,- details_dict: Dict[str, Any],- ) -> PipeFunc:- pipe_blueprint = PipeFuncBlueprint.model_validate(details_dict)- return cls.make_pipe_from_blueprint(- domain_code=domain_code,- pipe_code=pipe_code,- pipe_blueprint=pipe_blueprint,- )diff --git a/pipelex/pipe_operators/pipe_img_gen.py b/pipelex/pipe_operators/pipe_img_gen.pyindex 742b9c18f..de624c0e0 100644--- a/pipelex/pipe_operators/pipe_img_gen.py+++ b/pipelex/pipe_operators/pipe_img_gen.py@@ -10,14 +10,14 @@ from pipelex.cogt.imgg.imgg_handle import ImggHandlefrom pipelex.cogt.imgg.imgg_job_components import AspectRatio, Background, ImggJobParams, Qualityfrom pipelex.cogt.imgg.imgg_prompt import ImggPromptfrom pipelex.config import StaticValidationReaction, get_config-from pipelex.core.concept_native import NativeConcept-from pipelex.core.pipe_input_spec import PipeInputSpec-from pipelex.core.pipe_output import PipeOutput-from pipelex.core.pipe_run_params import PipeOutputMultiplicity, PipeRunMode, PipeRunParams, output_multiplicity_to_apply-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory-from pipelex.core.stuff_content import ImageContent, ListContent, StuffContent-from pipelex.core.stuff_factory import StuffFactory-from pipelex.core.working_memory import WorkingMemory+from pipelex.core.concepts.concept_native import NativeConcept+from pipelex.core.memory.working_memory import WorkingMemory+from pipelex.core.pipes.pipe_input_spec import PipeInputSpec+from pipelex.core.pipes.pipe_output import PipeOutput+from pipelex.core.pipes.pipe_run_params import PipeOutputMultiplicity, PipeRunMode, PipeRunParams, output_multiplicity_to_apply+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory+from pipelex.core.stuffs.stuff_content import ImageContent, ListContent, StuffContent+from pipelex.core.stuffs.stuff_factory import StuffFactoryfrom pipelex.exceptions import (PipeDefinitionError,PipeInputError,@@ -47,6 +47,9 @@ class PipeImgGenOutput(PipeOutput):return the_urls+DEFAULT_PROMPT_VAR_NAME = "prompt"++class PipeImgGen(PipeOperator):output_concept_code: str = NativeConcept.IMAGE.codeimgg_prompt: Optional[str] = None@@ -92,7 +95,7 @@ class PipeImgGen(PipeOperator):if self.img_gen_prompt_var_name:return {self.img_gen_prompt_var_name}else:- return {"imgg_prompt"}+ return {DEFAULT_PROMPT_VAR_NAME}def _validate_inputs(self):concept_provider = get_concept_provider()diff --git a/pipelex/pipe_operators/pipe_img_gen_factory.py b/pipelex/pipe_operators/pipe_img_gen_factory.pyindex 2467681bc..528a0a1ba 100644--- a/pipelex/pipe_operators/pipe_img_gen_factory.py+++ b/pipelex/pipe_operators/pipe_img_gen_factory.py@@ -1,18 +1,20 @@-from typing import Any, Dict, Literal, Optional, Union+from typing import Literal, Optional, Unionfrom pydantic import Field, model_validatorfrom typing_extensions import Self, overridefrom pipelex.cogt.imgg.imgg_handle import ImggHandlefrom pipelex.cogt.imgg.imgg_job_components import AspectRatio, Quality-from pipelex.core.pipe_blueprint import PipeBlueprint, PipeSpecificFactoryProtocol-from pipelex.core.pipe_input_spec import PipeInputSpec+from pipelex.core.pipes.pipe_blueprint import PipeBlueprint+from pipelex.core.pipes.pipe_factory import PipeFactoryProtocol+from pipelex.core.pipes.pipe_input_spec import PipeInputSpecfrom pipelex.exceptions import PipeDefinitionErrorfrom pipelex.pipe_operators.pipe_img_gen import PipeImgGenfrom pipelex.tools.typing.validation_utils import has_more_than_one_among_attributes_from_listsclass PipeImgGenBlueprint(PipeBlueprint):+ type: Literal["PipeImgGen"] = "PipeImgGen"img_gen_prompt: Optional[str] = Noneimgg_handle: Optional[ImggHandle] = Noneaspect_ratio: Optional[AspectRatio] = Field(default=None, strict=False)@@ -24,6 +26,7 @@ class PipeImgGenBlueprint(PipeBlueprint):is_raw: Optional[bool] = Noneseed: Optional[Union[int, Literal["auto"]]] = Nonenb_output: Optional[int] = Field(default=None, ge=1)+ img_gen_prompt_var_name: Optional[str] = None@model_validator(mode="after")def validate_imgg_prompt_and_imgg_prompt_stuff_name(self) -> Self:@@ -37,7 +40,7 @@ class PipeImgGenBlueprint(PipeBlueprint):return self-class PipeImgGenFactory(PipeSpecificFactoryProtocol[PipeImgGenBlueprint, PipeImgGen]):+class PipeImgGenFactory(PipeFactoryProtocol[PipeImgGenBlueprint, PipeImgGen]):@classmethod@overridedef make_pipe_from_blueprint(@@ -51,7 +54,7 @@ class PipeImgGenFactory(PipeSpecificFactoryProtocol[PipeImgGenBlueprint, PipeImgdomain=domain_code,code=pipe_code,definition=pipe_blueprint.definition,- inputs=PipeInputSpec.make_from_dict(concepts_dict=pipe_blueprint.inputs or {}),+ inputs=PipeInputSpec.make_from_blueprint(domain=domain_code, blueprint=pipe_blueprint.inputs or {}),output_concept_code=pipe_blueprint.output,output_multiplicity=output_multiplicity,imgg_prompt=pipe_blueprint.img_gen_prompt,@@ -63,19 +66,5 @@ class PipeImgGenFactory(PipeSpecificFactoryProtocol[PipeImgGenBlueprint, PipeImgsafety_tolerance=pipe_blueprint.safety_tolerance,is_raw=pipe_blueprint.is_raw,seed=pipe_blueprint.seed,- )-- @classmethod- @override- def make_pipe_from_details_dict(- cls,- domain_code: str,- pipe_code: str,- details_dict: Dict[str, Any],- ) -> PipeImgGen:- pipe_blueprint = PipeImgGenBlueprint.model_validate(details_dict)- return cls.make_pipe_from_blueprint(- domain_code=domain_code,- pipe_code=pipe_code,- pipe_blueprint=pipe_blueprint,+ img_gen_prompt_var_name=pipe_blueprint.img_gen_prompt_var_name,)diff --git a/pipelex/pipe_operators/pipe_jinja2.py b/pipelex/pipe_operators/pipe_jinja2.pyindex 8cb3d3d69..b4376ef4c 100644--- a/pipelex/pipe_operators/pipe_jinja2.py+++ b/pipelex/pipe_operators/pipe_jinja2.py@@ -9,14 +9,14 @@ from pipelex import logfrom pipelex.cogt.content_generation.content_generator_dry import ContentGeneratorDryfrom pipelex.cogt.content_generation.content_generator_protocol import ContentGeneratorProtocolfrom pipelex.config import get_config-from pipelex.core.concept_native import NativeConcept-from pipelex.core.pipe_input_spec import PipeInputSpec-from pipelex.core.pipe_output import PipeOutput-from pipelex.core.pipe_run_params import PipeRunMode, PipeRunParams-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory-from pipelex.core.stuff import Stuff-from pipelex.core.stuff_content import TextContent-from pipelex.core.working_memory import WorkingMemory+from pipelex.core.concepts.concept_native import NativeConcept+from pipelex.core.memory.working_memory import WorkingMemory+from pipelex.core.pipes.pipe_input_spec import PipeInputSpec+from pipelex.core.pipes.pipe_output import PipeOutput+from pipelex.core.pipes.pipe_run_params import PipeRunMode, PipeRunParams+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory+from pipelex.core.stuffs.stuff import Stuff+from pipelex.core.stuffs.stuff_content import TextContentfrom pipelex.exceptions import PipeDefinitionError, PipeRunParamsErrorfrom pipelex.hub import get_content_generator, get_template, get_template_providerfrom pipelex.pipe_operators.pipe_operator import PipeOperatordiff --git a/pipelex/pipe_operators/pipe_jinja2_factory.py b/pipelex/pipe_operators/pipe_jinja2_factory.pyindex 13597ff3f..228ec13ad 100644--- a/pipelex/pipe_operators/pipe_jinja2_factory.py+++ b/pipelex/pipe_operators/pipe_jinja2_factory.py@@ -1,10 +1,11 @@-from typing import Any, Dict, Optional+from typing import Literal, Optionalfrom typing_extensions import overridefrom pipelex.config import get_config-from pipelex.core.pipe_blueprint import PipeBlueprint, PipeSpecificFactoryProtocol-from pipelex.core.pipe_input_spec import PipeInputSpec+from pipelex.core.pipes.pipe_blueprint import PipeBlueprint+from pipelex.core.pipes.pipe_factory import PipeFactoryProtocol+from pipelex.core.pipes.pipe_input_spec import PipeInputSpecfrom pipelex.exceptions import PipeDefinitionErrorfrom pipelex.pipe_operators.pipe_jinja2 import PipeJinja2from pipelex.tools.templating.jinja2_parsing import check_jinja2_parsing@@ -14,13 +15,14 @@ from pipelex.tools.templating.templating_models import PromptingStyleclass PipeJinja2Blueprint(PipeBlueprint):+ type: Literal["PipeJinja2"] = "PipeJinja2"jinja2_name: Optional[str] = Nonejinja2: Optional[str] = Noneprompting_style: Optional[PromptingStyle] = Nonetemplate_category: Jinja2TemplateCategory = Jinja2TemplateCategory.LLM_PROMPT-class PipeJinja2Factory(PipeSpecificFactoryProtocol[PipeJinja2Blueprint, PipeJinja2]):+class PipeJinja2Factory(PipeFactoryProtocol[PipeJinja2Blueprint, PipeJinja2]):@classmethod@overridedef make_pipe_from_blueprint(@@ -42,7 +44,7 @@ class PipeJinja2Factory(PipeSpecificFactoryProtocol[PipeJinja2Blueprint, PipeJindomain=domain_code,code=pipe_code,definition=pipe_blueprint.definition,- inputs=PipeInputSpec.make_from_dict(concepts_dict=pipe_blueprint.inputs or {}),+ inputs=PipeInputSpec.make_from_blueprint(domain=domain_code, blueprint=pipe_blueprint.inputs or {}),output_concept_code=pipe_blueprint.output,jinja2_name=pipe_blueprint.jinja2_name,jinja2=preprocessed_template,@@ -50,21 +52,6 @@ class PipeJinja2Factory(PipeSpecificFactoryProtocol[PipeJinja2Blueprint, PipeJintemplate_category=pipe_blueprint.template_category,)- @classmethod- @override- def make_pipe_from_details_dict(- cls,- domain_code: str,- pipe_code: str,- details_dict: Dict[str, Any],- ) -> PipeJinja2:- pipe_blueprint = PipeJinja2Blueprint.model_validate(details_dict)- return cls.make_pipe_from_blueprint(- domain_code=domain_code,- pipe_code=pipe_code,- pipe_blueprint=pipe_blueprint,- )-@classmethoddef make_pipe_jinja2_from_template_str(cls,diff --git a/pipelex/pipe_operators/pipe_llm.py b/pipelex/pipe_operators/pipe_llm.pyindex 62c2294f8..54db47d07 100644--- a/pipelex/pipe_operators/pipe_llm.py+++ b/pipelex/pipe_operators/pipe_llm.py@@ -11,21 +11,21 @@ from pipelex.cogt.llm.llm_models.llm_setting import LLMSetting, LLMSettingChoicefrom pipelex.cogt.llm.llm_prompt import LLMPromptfrom pipelex.cogt.llm.llm_prompt_factory_abstract import LLMPromptFactoryAbstractfrom pipelex.config import StaticValidationReaction, get_config-from pipelex.core.concept import Concept-from pipelex.core.concept_code_factory import ConceptCodeFactory-from pipelex.core.concept_native import NativeConcept, NativeConceptClass-from pipelex.core.domain import Domain, SpecialDomain-from pipelex.core.pipe_input_spec import PipeInputSpec-from pipelex.core.pipe_output import PipeOutput-from pipelex.core.pipe_run_params import (+from pipelex.core.concepts.concept import Concept+from pipelex.core.concepts.concept_code_factory import ConceptCodeFactory+from pipelex.core.concepts.concept_native import NativeConcept, NativeConceptClass+from pipelex.core.domains.domain import Domain, SpecialDomain+from pipelex.core.memory.working_memory import WorkingMemory+from pipelex.core.pipes.pipe_input_spec import PipeInputSpec+from pipelex.core.pipes.pipe_output import PipeOutput+from pipelex.core.pipes.pipe_run_params import (PipeOutputMultiplicity,PipeRunParamKey,PipeRunParams,output_multiplicity_to_apply,)-from pipelex.core.stuff_content import ListContent, StructuredContent, StuffContent, TextContent-from pipelex.core.stuff_factory import StuffFactory-from pipelex.core.working_memory import WorkingMemory+from pipelex.core.stuffs.stuff_content import ListContent, StructuredContent, StuffContent, TextContent+from pipelex.core.stuffs.stuff_factory import StuffFactoryfrom pipelex.exceptions import (PipeDefinitionError,PipeInputError,diff --git a/pipelex/pipe_operators/pipe_llm_factory.py b/pipelex/pipe_operators/pipe_llm_factory.pyindex 69d6d420c..13ebe68c0 100644--- a/pipelex/pipe_operators/pipe_llm_factory.py+++ b/pipelex/pipe_operators/pipe_llm_factory.py@@ -1,12 +1,13 @@-from typing import Any, Dict, List, Optional+from typing import List, Literal, Optionalfrom pydantic import model_validatorfrom typing_extensions import Self, overridefrom pipelex.cogt.llm.llm_models.llm_setting import LLMSettingChoices, LLMSettingOrPresetId-from pipelex.core.pipe_blueprint import PipeBlueprint, PipeSpecificFactoryProtocol-from pipelex.core.pipe_input_spec import PipeInputSpec-from pipelex.core.pipe_run_params import make_output_multiplicity+from pipelex.core.pipes.pipe_blueprint import PipeBlueprint+from pipelex.core.pipes.pipe_factory import PipeFactoryProtocol+from pipelex.core.pipes.pipe_input_spec import PipeInputSpec+from pipelex.core.pipes.pipe_run_params import make_output_multiplicityfrom pipelex.exceptions import PipeDefinitionErrorfrom pipelex.hub import get_concept_provider, get_optional_domainfrom pipelex.pipe_operators.pipe_jinja2 import PipeJinja2@@ -19,6 +20,7 @@ from pipelex.tools.typing.validation_utils import has_more_than_one_among_attribclass PipeLLMBlueprint(PipeBlueprint):+ type: Literal["PipeLLM"] = "PipeLLM"system_prompt_template: Optional[str] = Nonesystem_prompt_template_name: Optional[str] = Nonesystem_prompt_name: Optional[str] = None@@ -53,7 +55,7 @@ class PipeLLMBlueprint(PipeBlueprint):return self-class PipeLLMFactory(PipeSpecificFactoryProtocol[PipeLLMBlueprint, PipeLLM]):+class PipeLLMFactory(PipeFactoryProtocol[PipeLLMBlueprint, PipeLLM]):@classmethod@overridedef make_pipe_from_blueprint(@@ -91,7 +93,7 @@ class PipeLLMFactory(PipeSpecificFactoryProtocol[PipeLLMBlueprint, PipeLLM]):domain_code=domain_code,template_str=pipe_blueprint.prompt_template,template_name=pipe_blueprint.template_name,- inputs=PipeInputSpec.make_from_dict(pipe_blueprint.inputs) if pipe_blueprint.inputs else PipeInputSpec.make_empty(),+ inputs=PipeInputSpec.make_from_blueprint(domain=domain_code, blueprint=pipe_blueprint.inputs or {}),)except Jinja2TemplateError as exc:error_msg = f"Jinja2 syntax error in user prompt for pipe '{pipe_code}' in domain '{domain_code}': {exc}."@@ -114,8 +116,8 @@ class PipeLLMFactory(PipeSpecificFactoryProtocol[PipeLLMBlueprint, PipeLLM]):user_images: List[str] = []if pipe_blueprint.inputs:- for stuff_name, concept_code in (pipe_blueprint.inputs).items():- concept = get_concept_provider().get_required_concept(concept_code=concept_code)+ for stuff_name, requirement in (pipe_blueprint.inputs).items():+ concept = get_concept_provider().get_required_concept(concept_code=requirement.concept_code)if get_concept_provider().is_image_concept(concept_code=concept.code):user_images.append(stuff_name)else:@@ -124,7 +126,7 @@ class PipeLLMFactory(PipeSpecificFactoryProtocol[PipeLLMBlueprint, PipeLLM]):pipe_llm_prompt = PipeLLMPrompt(code="adhoc_for_pipe_llm_prompt",domain=domain_code,- inputs=PipeInputSpec.make_from_dict(concepts_dict=pipe_blueprint.inputs or {}),+ inputs=PipeInputSpec.make_from_blueprint(domain=domain_code, blueprint=pipe_blueprint.inputs or {}),system_prompt_pipe_jinja2=system_prompt_pipe_jinja2,system_prompt_verbatim_name=pipe_blueprint.system_prompt_name,system_prompt=pipe_blueprint.system_prompt or system_prompt,@@ -149,7 +151,7 @@ class PipeLLMFactory(PipeSpecificFactoryProtocol[PipeLLMBlueprint, PipeLLM]):domain=domain_code,code=pipe_code,definition=pipe_blueprint.definition,- inputs=PipeInputSpec.make_from_dict(concepts_dict=pipe_blueprint.inputs or {}),+ inputs=PipeInputSpec.make_from_blueprint(domain=domain_code, blueprint=pipe_blueprint.inputs or {}),output_concept_code=pipe_blueprint.output,pipe_llm_prompt=pipe_llm_prompt,llm_choices=llm_choices,@@ -158,18 +160,3 @@ class PipeLLMFactory(PipeSpecificFactoryProtocol[PipeLLMBlueprint, PipeLLM]):system_prompt_to_structure=pipe_blueprint.system_prompt_to_structure,output_multiplicity=output_multiplicity,)-- @classmethod- @override- def make_pipe_from_details_dict(- cls,- domain_code: str,- pipe_code: str,- details_dict: Dict[str, Any],- ) -> PipeLLM:- pipe_blueprint = PipeLLMBlueprint.model_validate(details_dict)- return cls.make_pipe_from_blueprint(- domain_code=domain_code,- pipe_code=pipe_code,- pipe_blueprint=pipe_blueprint,- )diff --git a/pipelex/pipe_operators/pipe_llm_prompt.py b/pipelex/pipe_operators/pipe_llm_prompt.pyindex 7f65b58e4..612b87536 100644--- a/pipelex/pipe_operators/pipe_llm_prompt.py+++ b/pipelex/pipe_operators/pipe_llm_prompt.py@@ -7,15 +7,15 @@ from pipelex import logfrom pipelex.cogt.image.prompt_image import PromptImagefrom pipelex.cogt.image.prompt_image_factory import PromptImageFactoryfrom pipelex.cogt.llm.llm_prompt import LLMPrompt-from pipelex.core.concept import Concept-from pipelex.core.concept_native import NativeConcept-from pipelex.core.pipe_input_spec import PipeInputSpec-from pipelex.core.pipe_output import PipeOutput-from pipelex.core.pipe_run_params import PipeRunMode, PipeRunParams-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory-from pipelex.core.stuff_content import ImageContent, LLMPromptContent, StuffContent-from pipelex.core.stuff_factory import StuffFactory-from pipelex.core.working_memory import WorkingMemory+from pipelex.core.concepts.concept import Concept+from pipelex.core.concepts.concept_native import NativeConcept+from pipelex.core.memory.working_memory import WorkingMemory+from pipelex.core.pipes.pipe_input_spec import PipeInputSpec+from pipelex.core.pipes.pipe_output import PipeOutput+from pipelex.core.pipes.pipe_run_params import PipeRunMode, PipeRunParams+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory+from pipelex.core.stuffs.stuff_content import ImageContent, LLMPromptContent, StuffContent+from pipelex.core.stuffs.stuff_factory import StuffFactoryfrom pipelex.exceptions import (PipeDefinitionError,PipeInputError,diff --git a/pipelex/pipe_operators/pipe_ocr.py b/pipelex/pipe_operators/pipe_ocr.pyindex b2eb568d3..7746c436e 100644--- a/pipelex/pipe_operators/pipe_ocr.py+++ b/pipelex/pipe_operators/pipe_ocr.py@@ -11,13 +11,13 @@ from pipelex.cogt.ocr.ocr_handle import OcrHandlefrom pipelex.cogt.ocr.ocr_input import OcrInputfrom pipelex.cogt.ocr.ocr_job_components import OcrJobConfig, OcrJobParamsfrom pipelex.config import StaticValidationReaction, get_config-from pipelex.core.concept_native import NativeConcept-from pipelex.core.pipe_input_spec import PipeInputSpec-from pipelex.core.pipe_output import PipeOutput-from pipelex.core.pipe_run_params import PipeRunMode, PipeRunParams-from pipelex.core.stuff_content import ImageContent, ListContent, PageContent, TextAndImagesContent, TextContent-from pipelex.core.stuff_factory import StuffFactory-from pipelex.core.working_memory import WorkingMemory+from pipelex.core.concepts.concept_native import NativeConcept+from pipelex.core.memory.working_memory import WorkingMemory+from pipelex.core.pipes.pipe_input_spec import InputRequirementBlueprint, PipeInputSpec+from pipelex.core.pipes.pipe_output import PipeOutput+from pipelex.core.pipes.pipe_run_params import PipeRunMode, PipeRunParams+from pipelex.core.stuffs.stuff_content import ImageContent, ListContent, PageContent, TextAndImagesContent, TextContent+from pipelex.core.stuffs.stuff_factory import StuffFactoryfrom pipelex.exceptions import (PipeDefinitionError,StaticValidationError,@@ -98,6 +98,7 @@ class PipeOcr(PipeOperator):log.error(inadequate_input_concept_error.desc())case StaticValidationReaction.RAISE:raise inadequate_input_concept_error+if len(candidate_prompt_var_names) > 1:too_many_candidate_inputs_error = StaticValidationError(error_type=StaticValidationErrorType.TOO_MANY_CANDIDATE_INPUTS,@@ -130,7 +131,10 @@ class PipeOcr(PipeOperator):@overridedef needed_inputs(self) -> PipeInputSpec:- return PipeInputSpec.make_from_dict({PIPE_OCR_INPUT_NAME: self.inputs.root[PIPE_OCR_INPUT_NAME].concept_code})+ return PipeInputSpec.make_from_blueprint(+ domain=self.domain,+ blueprint={PIPE_OCR_INPUT_NAME: InputRequirementBlueprint(concept_code=self.inputs.root[PIPE_OCR_INPUT_NAME].concept_code)},+ )@overrideasync def _run_operator_pipe(diff --git a/pipelex/pipe_operators/pipe_ocr_factory.py b/pipelex/pipe_operators/pipe_ocr_factory.pyindex ad9d015a9..621a7b7d8 100644--- a/pipelex/pipe_operators/pipe_ocr_factory.py+++ b/pipelex/pipe_operators/pipe_ocr_factory.py@@ -1,4 +1,4 @@-from typing import Any, Dict, Optional+from typing import Literal, Optionalfrom typing_extensions import override@@ -6,21 +6,22 @@ from pipelex.cogt.ocr.ocr_engine_factory import OcrEngineFactoryfrom pipelex.cogt.ocr.ocr_handle import OcrHandlefrom pipelex.cogt.ocr.ocr_platform import OcrPlatformfrom pipelex.config import get_config-from pipelex.core.pipe_blueprint import PipeBlueprint, PipeSpecificFactoryProtocol-from pipelex.core.pipe_input_spec import PipeInputSpec+from pipelex.core.pipes.pipe_blueprint import PipeBlueprint+from pipelex.core.pipes.pipe_factory import PipeFactoryProtocol+from pipelex.core.pipes.pipe_input_spec import PipeInputSpecfrom pipelex.pipe_operators.pipe_ocr import PipeOcrclass PipeOcrBlueprint(PipeBlueprint):- definition: Optional[str] = None+ type: Literal["PipeOcr"] = "PipeOcr"ocr_platform: Optional[OcrPlatform] = None- page_images: bool = False- page_image_captions: bool = False- page_views: bool = False+ page_images: Optional[bool] = None+ page_image_captions: Optional[bool] = None+ page_views: Optional[bool] = Nonepage_views_dpi: Optional[int] = None-class PipeOcrFactory(PipeSpecificFactoryProtocol[PipeOcrBlueprint, PipeOcr]):+class PipeOcrFactory(PipeFactoryProtocol[PipeOcrBlueprint, PipeOcr]):@classmethod@overridedef make_pipe_from_blueprint(@@ -40,24 +41,9 @@ class PipeOcrFactory(PipeSpecificFactoryProtocol[PipeOcrBlueprint, PipeOcr]):definition=pipe_blueprint.definition,ocr_engine=ocr_engine,output_concept_code=pipe_blueprint.output,- inputs=PipeInputSpec.make_from_dict(concepts_dict=pipe_blueprint.inputs or {}),- should_include_images=pipe_blueprint.page_images,- should_caption_images=pipe_blueprint.page_image_captions,- should_include_page_views=pipe_blueprint.page_views,+ inputs=PipeInputSpec.make_from_blueprint(domain=domain_code, blueprint=pipe_blueprint.inputs or {}),+ should_include_images=pipe_blueprint.page_images or False,+ should_caption_images=pipe_blueprint.page_image_captions or False,+ should_include_page_views=pipe_blueprint.page_views or False,page_views_dpi=pipe_blueprint.page_views_dpi or get_config().cogt.ocr_config.default_page_views_dpi,)-- @classmethod- @override- def make_pipe_from_details_dict(- cls,- domain_code: str,- pipe_code: str,- details_dict: Dict[str, Any],- ) -> PipeOcr:- pipe_blueprint = PipeOcrBlueprint.model_validate(details_dict)- return cls.make_pipe_from_blueprint(- domain_code=domain_code,- pipe_code=pipe_code,- pipe_blueprint=pipe_blueprint,- )diff --git a/pipelex/pipe_operators/pipe_operator.py b/pipelex/pipe_operators/pipe_operator.pyindex 1b4c64218..6ff71c733 100644--- a/pipelex/pipe_operators/pipe_operator.py+++ b/pipelex/pipe_operators/pipe_operator.py@@ -4,10 +4,10 @@ from typing import Optionalfrom typing_extensions import overridefrom pipelex import log-from pipelex.core.pipe_abstract import PipeAbstract-from pipelex.core.pipe_output import PipeOutput-from pipelex.core.pipe_run_params import PipeRunMode, PipeRunParams-from pipelex.core.working_memory import WorkingMemory+from pipelex.core.memory.working_memory import WorkingMemory+from pipelex.core.pipes.pipe_abstract import PipeAbstract+from pipelex.core.pipes.pipe_output import PipeOutput+from pipelex.core.pipes.pipe_run_params import PipeRunMode, PipeRunParamsfrom pipelex.hub import get_activity_managerfrom pipelex.pipeline.activity.activity_models import ActivityReportfrom pipelex.pipeline.job_metadata import JobMetadatadiff --git a/pipelex/pipe_operators/piped_llm_prompt_factory.py b/pipelex/pipe_operators/piped_llm_prompt_factory.pyindex 542b9d435..911c49b93 100644--- a/pipelex/pipe_operators/piped_llm_prompt_factory.py+++ b/pipelex/pipe_operators/piped_llm_prompt_factory.py@@ -5,9 +5,9 @@ from typing_extensions import overridefrom pipelex.cogt.llm.llm_prompt import LLMPromptfrom pipelex.cogt.llm.llm_prompt_factory_abstract import LLMPromptFactoryAbstract, make_empty_promptfrom pipelex.cogt.llm.llm_prompt_template_inputs import LLMPromptTemplateInputs-from pipelex.core.pipe_output import PipeOutput-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory-from pipelex.core.working_memory_factory import WorkingMemoryFactory+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory+from pipelex.core.pipes.pipe_output import PipeOutput+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactoryfrom pipelex.pipe_operators.pipe_llm_prompt import PipeLLMPrompt, PipeLLMPromptOutputfrom pipelex.pipeline.job_metadata import JobMetadatadiff --git a/pipelex/pipe_works/pipe_dry.py b/pipelex/pipe_works/pipe_dry.pyindex 01be2580a..278313f41 100644--- a/pipelex/pipe_works/pipe_dry.py+++ b/pipelex/pipe_works/pipe_dry.py@@ -6,12 +6,12 @@ from typing import Dict, List, Tuplefrom pipelex import logfrom pipelex.config import get_config-from pipelex.core.pipe_abstract import PipeAbstract-from pipelex.core.pipe_input_spec import PipeInputSpec, TypedNamedInputRequirement-from pipelex.core.pipe_run_params import PipeRunMode-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory-from pipelex.core.stuff_content import StuffContent, TextContent-from pipelex.core.working_memory_factory import WorkingMemoryFactory+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory+from pipelex.core.pipes.pipe_abstract import PipeAbstract+from pipelex.core.pipes.pipe_input_spec import PipeInputSpec, TypedNamedInputRequirement+from pipelex.core.pipes.pipe_run_params import PipeRunMode+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory+from pipelex.core.stuffs.stuff_content import StuffContent, TextContentfrom pipelex.hub import get_class_registry, get_concept_provider, get_pipe_providerfrom pipelex.pipeline.job_metadata import JobMetadata@@ -53,6 +53,13 @@ async def dry_run_pipe_codes(pipe_codes: List[str]) -> Dict[str, str]:return await dry_run_pipes(pipes=pipes)+async def dry_run_pipe(pipe: PipeAbstract):+ """+ Dry run a pipe.+ """+ return await dry_run_pipes(pipes=[pipe])++async def dry_run_pipes(pipes: List[PipeAbstract]) -> Dict[str, str]:"""Dry run all pipes in the library using ThreadPoolExecutor for true parallelism.diff --git a/pipelex/pipe_works/pipe_job.py b/pipelex/pipe_works/pipe_job.pyindex 6454c9577..d25b0f61a 100644--- a/pipelex/pipe_works/pipe_job.py+++ b/pipelex/pipe_works/pipe_job.py@@ -2,9 +2,9 @@ from typing import Optionalfrom pydantic import BaseModel, Field-from pipelex.core.pipe_abstract import PipeAbstract-from pipelex.core.pipe_run_params import PipeRunParams-from pipelex.core.working_memory import WorkingMemory+from pipelex.core.memory.working_memory import WorkingMemory+from pipelex.core.pipes.pipe_abstract import PipeAbstract+from pipelex.core.pipes.pipe_run_params import PipeRunParamsfrom pipelex.pipeline.job_metadata import JobMetadatadiff --git a/pipelex/pipe_works/pipe_job_factory.py b/pipelex/pipe_works/pipe_job_factory.pyindex e1f351d85..73340757e 100644--- a/pipelex/pipe_works/pipe_job_factory.py+++ b/pipelex/pipe_works/pipe_job_factory.py@@ -1,10 +1,10 @@from typing import Optional-from pipelex.core.pipe_abstract import PipeAbstract-from pipelex.core.pipe_run_params import PipeRunParams-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory-from pipelex.core.working_memory import WorkingMemory-from pipelex.core.working_memory_factory import WorkingMemoryFactory+from pipelex.core.memory.working_memory import WorkingMemory+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory+from pipelex.core.pipes.pipe_abstract import PipeAbstract+from pipelex.core.pipes.pipe_run_params import PipeRunParams+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactoryfrom pipelex.pipe_works.pipe_job import PipeJobfrom pipelex.pipeline.job_metadata import JobMetadatadiff --git a/pipelex/pipe_works/pipe_router.py b/pipelex/pipe_works/pipe_router.pyindex d4339ed77..a581c66de 100644--- a/pipelex/pipe_works/pipe_router.py+++ b/pipelex/pipe_works/pipe_router.py@@ -3,9 +3,9 @@ from typing import Optional, castfrom typing_extensions import overridefrom pipelex import log-from pipelex.core.pipe_output import PipeOutputType-from pipelex.core.pipe_run_params import PipeRunParams-from pipelex.core.working_memory import WorkingMemory+from pipelex.core.memory.working_memory import WorkingMemory+from pipelex.core.pipes.pipe_output import PipeOutputType+from pipelex.core.pipes.pipe_run_params import PipeRunParamsfrom pipelex.hub import get_required_pipefrom pipelex.pipe_works.pipe_job import PipeJobfrom pipelex.pipe_works.pipe_job_factory import PipeJobFactorydiff --git a/pipelex/pipe_works/pipe_router_protocol.py b/pipelex/pipe_works/pipe_router_protocol.pyindex ee014f4ee..b58d8c5d8 100644--- a/pipelex/pipe_works/pipe_router_protocol.py+++ b/pipelex/pipe_works/pipe_router_protocol.py@@ -1,8 +1,8 @@from typing import Optional, Protocol-from pipelex.core.pipe_output import PipeOutputType-from pipelex.core.pipe_run_params import PipeRunParams-from pipelex.core.working_memory import WorkingMemory+from pipelex.core.memory.working_memory import WorkingMemory+from pipelex.core.pipes.pipe_output import PipeOutputType+from pipelex.core.pipes.pipe_run_params import PipeRunParamsfrom pipelex.pipe_works.pipe_job import PipeJobfrom pipelex.pipeline.job_metadata import JobMetadatadiff --git a/pipelex/pipelex.py b/pipelex/pipelex.pyindex 068171963..ee74a1c9d 100644--- a/pipelex/pipelex.py+++ b/pipelex/pipelex.py@@ -21,13 +21,13 @@ from pipelex.cogt.inference.inference_manager import InferenceManagerfrom pipelex.cogt.llm.llm_models.llm_model import LATEST_VERSION_NAMEfrom pipelex.cogt.llm.llm_models.llm_model_library import LLMModelLibraryfrom pipelex.config import PipelexConfig, get_config-from pipelex.core.concept_library import ConceptLibrary-from pipelex.core.domain_library import DomainLibrary-from pipelex.core.pipe_library import PipeLibrary+from pipelex.core.concepts.concept_library import ConceptLibrary+from pipelex.core.domains.domain_library import DomainLibrary+from pipelex.core.pipes.pipe_library import PipeLibraryfrom pipelex.core.registry_models import PipelexRegistryModelsfrom pipelex.exceptions import PipelexConfigError, PipelexSetupErrorfrom pipelex.hub import PipelexHub, set_pipelex_hub-from pipelex.libraries.library_manager import LibraryManager+from pipelex.libraries.library_manager_factory import LibraryManagerFactoryfrom pipelex.pipe_works.pipe_router import PipeRouterfrom pipelex.pipe_works.pipe_router_protocol import PipeRouterProtocolfrom pipelex.pipeline.activity.activity_manager import ActivityManager@@ -61,7 +61,7 @@ PACKAGE_VERSION = metadata(PACKAGE_NAME)["Version"]class Pipelex(metaclass=MetaSingleton):def __init__(self,- config_folder_path: str,+ config_dir_path: str,# Dependency injectionpipelex_hub: Optional[PipelexHub] = None,config_cls: Optional[Type[ConfigRoot]] = None,@@ -74,7 +74,7 @@ class Pipelex(metaclass=MetaSingleton):activity_manager: Optional[ActivityManagerProtocol] = None,reporting_delegate: Optional[ReportingProtocol] = None,) -> None:- self.config_folder_path = config_folder_path+ self.config_dir_path = config_dir_pathself.pipelex_hub = pipelex_hub or PipelexHub()set_pipelex_hub(self.pipelex_hub)@@ -95,7 +95,7 @@ class Pipelex(metaclass=MetaSingleton):log.debug("Logs are configured")# tools- self.template_provider = template_provider or TemplateLibrary.make_empty(config_folder_path=config_folder_path)+ self.template_provider = template_provider or TemplateLibrary.make_empty(config_dir_path=config_dir_path)self.pipelex_hub.set_template_provider(self.template_provider)self.class_registry = class_registry or ClassRegistry()@@ -105,7 +105,7 @@ class Pipelex(metaclass=MetaSingleton):# cogtself.plugin_manager = PluginManager()self.pipelex_hub.set_plugin_manager(self.plugin_manager)- self.llm_model_provider = llm_model_provider or LLMModelLibrary.make_empty(config_folder_path=config_folder_path)+ self.llm_model_provider = llm_model_provider or LLMModelLibrary.make_empty(config_dir_path=config_dir_path)self.pipelex_hub.set_llm_models_provider(self.llm_model_provider)self.inference_manager = inference_manager or InferenceManager()self.pipelex_hub.set_inference_manager(self.inference_manager)@@ -124,13 +124,13 @@ class Pipelex(metaclass=MetaSingleton):self.pipelex_hub.set_domain_provider(domain_provider=domain_library)self.pipelex_hub.set_concept_provider(concept_provider=concept_library)self.pipelex_hub.set_pipe_provider(pipe_provider=pipe_library)- self.library_manager = LibraryManager.make(++ self.library_manager = LibraryManagerFactory.make(domain_library=domain_library,concept_library=concept_library,pipe_library=pipe_library,- config_folder_path=config_folder_path,+ config_dir_path=config_dir_path,)- self.library_manager.setup()self.pipelex_hub.set_library_manager(library_manager=self.library_manager)# pipelex pipeline@@ -188,6 +188,7 @@ class Pipelex(metaclass=MetaSingleton):try:self.template_provider.setup()self.llm_model_provider.setup()+ self.library_manager.setup()llm_deck = self.library_manager.load_deck()for llm_model in self.llm_model_provider.get_all_llm_models():if llm_model.version == LATEST_VERSION_NAME:@@ -277,15 +278,15 @@ class Pipelex(metaclass=MetaSingleton):raise PipelexSetupError("Could not find relative config folder path because of: Failed to get caller frame")caller_file = current_frame.f_back.f_code.co_filenamecaller_dir = os.path.dirname(os.path.abspath(caller_file))- config_folder_path = os.path.abspath(os.path.join(caller_dir, relative_config_folder_path))+ config_dir_path = os.path.abspath(os.path.join(caller_dir, relative_config_folder_path))else:- config_folder_path = os.path.abspath(os.path.join(os.getcwd(), relative_config_folder_path))+ config_dir_path = os.path.abspath(os.path.join(os.getcwd(), relative_config_folder_path))elif absolute_config_folder_path is not None:- config_folder_path = absolute_config_folder_path+ config_dir_path = absolute_config_folder_pathelse:- config_folder_path = "./pipelex_libraries"+ config_dir_path = "./pipelex_libraries"- pipelex_instance = cls(config_folder_path=config_folder_path)+ pipelex_instance = cls(config_dir_path=config_dir_path)pipelex_instance.setup()pipelex_instance.setup_libraries()log.info(f"Pipelex {PACKAGE_VERSION} initialized.")diff --git a/pipelex/pipelex.toml b/pipelex/pipelex.tomlindex 8aeaf2513..b9ca86d27 100644--- a/pipelex/pipelex.toml+++ b/pipelex/pipelex.toml@@ -205,7 +205,7 @@ apply_to_jinja2_rendering = falsetext_gen_truncate_length = 256nb_list_items = 3nb_ocr_pages = 4-allowed_to_fail_pipes = ["infinite_loop_1"]+allowed_to_fail_pipes = ["infinite_loop_1", "build_blueprint", "structure_blueprint"]image_urls = ["https://storage.googleapis.com/public_test_files_7fa6_4277_9ab/fashion/fashion_photo_1.jpg","https://storage.googleapis.com/public_test_files_7fa6_4277_9ab/fashion/fashion_photo_2.png",diff --git a/pipelex/pipeline/activity/activity_handler.py b/pipelex/pipeline/activity/activity_handler.pyindex 67e3929bb..2d9a38396 100644--- a/pipelex/pipeline/activity/activity_handler.py+++ b/pipelex/pipeline/activity/activity_handler.py@@ -3,8 +3,8 @@ from typing import Set, castfrom pipelex import logfrom pipelex.config import get_config-from pipelex.core.stuff import Stuff-from pipelex.core.stuff_content import (+from pipelex.core.stuffs.stuff import Stuff+from pipelex.core.stuffs.stuff_content import (HtmlContent,ImageContent,ListContent,diff --git a/pipelex/pipeline/execute.py b/pipelex/pipeline/execute.pyindex 79981e5f4..d34b19253 100644--- a/pipelex/pipeline/execute.py+++ b/pipelex/pipeline/execute.py@@ -1,15 +1,15 @@from typing import List, Optionalfrom pipelex.client.protocol import ImplicitMemory-from pipelex.core.pipe_output import PipeOutput-from pipelex.core.pipe_run_params import (+from pipelex.core.memory.working_memory import WorkingMemory+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory+from pipelex.core.pipes.pipe_output import PipeOutput+from pipelex.core.pipes.pipe_run_params import (FORCE_DRY_RUN_MODE_ENV_KEY,PipeOutputMultiplicity,PipeRunMode,)-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory-from pipelex.core.working_memory import WorkingMemory-from pipelex.core.working_memory_factory import WorkingMemoryFactory+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactoryfrom pipelex.exceptions import PipelineInputErrorfrom pipelex.hub import (get_pipe_router,diff --git a/pipelex/pipeline/start.py b/pipelex/pipeline/start.pyindex 85027c1e2..995bd72fd 100644--- a/pipelex/pipeline/start.py+++ b/pipelex/pipeline/start.py@@ -2,11 +2,11 @@ import asynciofrom typing import Optionalfrom pipelex.client.protocol import CompactMemory-from pipelex.core.pipe_output import PipeOutput-from pipelex.core.pipe_run_params import PipeOutputMultiplicity, PipeRunMode-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory-from pipelex.core.working_memory import WorkingMemory-from pipelex.core.working_memory_factory import WorkingMemoryFactory+from pipelex.core.memory.working_memory import WorkingMemory+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory+from pipelex.core.pipes.pipe_output import PipeOutput+from pipelex.core.pipes.pipe_run_params import PipeOutputMultiplicity, PipeRunMode+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactoryfrom pipelex.exceptions import StartPipelineExceptionfrom pipelex.hub import get_pipe_router, get_pipeline_manager, get_report_delegate, get_required_pipefrom pipelex.pipe_works.pipe_job_factory import PipeJobFactorydiff --git a/pipelex/pipeline/track/pipeline_tracker.py b/pipelex/pipeline/track/pipeline_tracker.pyindex 0486e4719..3cb99fb9f 100644--- a/pipelex/pipeline/track/pipeline_tracker.py+++ b/pipelex/pipeline/track/pipeline_tracker.py@@ -8,8 +8,8 @@ import networkx as nxfrom typing_extensions import overridefrom pipelex import log-from pipelex.core.concept import Concept-from pipelex.core.stuff import Stuff+from pipelex.core.concepts.concept import Concept+from pipelex.core.stuffs.stuff import Stufffrom pipelex.exceptions import JobHistoryErrorfrom pipelex.pipe_controllers.pipe_condition_details import PipeConditionDetailsfrom pipelex.pipeline.track.flow_chart import PipelineFlowChart@@ -45,6 +45,11 @@ class PipelineTracker(PipelineTrackerProtocol):self.nx_graph = nx.DiGraph()self.start_node = None+ @override+ def reset(self):+ self.teardown()+ self.setup()+def _get_node_name(self, node: str) -> Optional[str]:node_attributes = self.nx_graph.nodes[node]node_name = node_attributes[NodeAttributeKey.NAME]@@ -332,10 +337,10 @@ class PipelineTracker(PipelineTrackerProtocol):title_to_print += f" for {title}"print_mermaid_url(url=url, title=title_to_print)- def _print_mermaid_flowchart_url(self, title: Optional[str] = None, subtitle: Optional[str] = None):+ def _print_mermaid_flowchart_url(self, title: Optional[str] = None, subtitle: Optional[str] = None) -> Optional[str]:if not self.nx_graph.nodes:log.info("No nodes in the pipeline tracker")- return+ return Noneif self.start_node is None:raise JobHistoryError("Start node is not set")flowchart = PipelineFlowChart(nx_graph=self.nx_graph, start_node=self.start_node, tracker_config=self._tracker_config)@@ -344,6 +349,7 @@ class PipelineTracker(PipelineTrackerProtocol):if title:title_to_print += f" for {title}"print_mermaid_url(url=url, title=title_to_print)+ return url@overridedef output_flowchart(@@ -351,8 +357,9 @@ class PipelineTracker(PipelineTrackerProtocol):title: Optional[str] = None,subtitle: Optional[str] = None,is_detailed: bool = False,- ):+ ) -> Optional[str]:if is_detailed:self._print_mermaid_flowchart_code_and_url(title=title, subtitle=subtitle)else:- self._print_mermaid_flowchart_url(title=title, subtitle=subtitle)+ return self._print_mermaid_flowchart_url(title=title, subtitle=subtitle)+ return Nonediff --git a/pipelex/pipeline/track/pipeline_tracker_protocol.py b/pipelex/pipeline/track/pipeline_tracker_protocol.pyindex 95cb08528..1228130d9 100644--- a/pipelex/pipeline/track/pipeline_tracker_protocol.py+++ b/pipelex/pipeline/track/pipeline_tracker_protocol.py@@ -6,7 +6,7 @@ from typing import List, Optional, Protocolfrom typing_extensions import override-from pipelex.core.stuff import Stuff+from pipelex.core.stuffs.stuff import Stufffrom pipelex.pipe_controllers.pipe_condition_details import PipeConditionDetails@@ -15,7 +15,8 @@ class PipelineTrackerProtocol(Protocol):def teardown(self): ...- # def reset(self): ...+ def reset(self): ...+def add_pipe_step(self,from_stuff: Optional[Stuff],@@ -66,7 +67,7 @@ class PipelineTrackerProtocol(Protocol):title: Optional[str] = None,subtitle: Optional[str] = None,is_detailed: bool = False,- ): ...+ ) -> Optional[str]: ...class PipelineTrackerNoOp(PipelineTrackerProtocol):@@ -82,6 +83,10 @@ class PipelineTrackerNoOp(PipelineTrackerProtocol):def teardown(self) -> None:pass+ @override+ def reset(self) -> None:+ pass+@overridedef add_pipe_step(self,diff --git a/pipelex/test_extras/registry_test_models.py b/pipelex/test_extras/registry_test_models.pyindex 24c198c70..d453f729c 100644--- a/pipelex/test_extras/registry_test_models.py+++ b/pipelex/test_extras/registry_test_models.py@@ -1,6 +1,6 @@from typing import ClassVar, List-from pipelex.core.stuff_content import StructuredContent+from pipelex.core.stuffs.stuff_content import StructuredContentfrom pipelex.tools.registry_models import ModelType, RegistryModelsdiff --git a/pipelex/test_extras/shared_pytest_plugins.py b/pipelex/test_extras/shared_pytest_plugins.pyindex b05811580..c42082988 100644--- a/pipelex/test_extras/shared_pytest_plugins.py+++ b/pipelex/test_extras/shared_pytest_plugins.py@@ -1,7 +1,7 @@import pytestfrom pytest import FixtureRequest, Parser-from pipelex.core.pipe_run_params import PipeRunMode+from pipelex.core.pipes.pipe_run_params import PipeRunModefrom pipelex.tools.runtime_manager import RunMode, runtime_managerdiff --git a/pipelex/tools/misc/mermaid_utils.py b/pipelex/tools/misc/mermaid_utils.pyindex 0afe479be..b88f9e43a 100644--- a/pipelex/tools/misc/mermaid_utils.py+++ b/pipelex/tools/misc/mermaid_utils.py@@ -24,7 +24,7 @@ def make_mermaid_url(mermaid_code: str) -> str:},}encoded = encode_pako_from_string(json.dumps(as_dict))- url = f"https://mermaid.live/edit#{encoded}"+ url = f"https://mermaid.ink/img/{encoded}"return urldiff --git a/pipelex/tools/misc/toml_utils.py b/pipelex/tools/misc/toml_utils.pyindex 4dd7d9a79..ad2cf078d 100644--- a/pipelex/tools/misc/toml_utils.py+++ b/pipelex/tools/misc/toml_utils.py@@ -1,10 +1,15 @@from __future__ import annotations-from typing import Any, Dict, List, Optional+import re+from typing import Any, Dict, List, Mapping, Optional, castimport toml+import tomlkit+from tomlkit import array, document, inline_table, table+from tomlkit import string as tk_stringfrom pipelex.tools.misc.file_utils import path_exists+from pipelex.tools.misc.json_utils import remove_none_values_from_dictclass TOMLValidationError(Exception):@@ -13,7 +18,7 @@ class TOMLValidationError(Exception):pass-def _validate_toml_content(content: str, file_path: str) -> None:+def validate_toml_content(content: str, file_path: Optional[str] = None) -> None:"""Validate TOML content for common formatting issues."""lines = content.splitlines()issues: List[str] = []@@ -52,7 +57,7 @@ def validate_toml_file(path: str) -> None:"""with open(path, "r", encoding="utf-8") as file:content = file.read()- _validate_toml_content(content, path)+ validate_toml_content(content, path)def clean_trailing_whitespace(content: str) -> str:@@ -122,3 +127,188 @@ def failable_load_toml_from_path(path: str) -> Optional[Dict[str, Any]]:except toml.TomlDecodeError as exc:print(f"Failed to parse TOML file '{path}': {exc}")return None+++def make_toml_string(+ text: str,+ prefer_literal: bool = False,+ force_multiline: bool = False,+ ensure_trailing_newline: bool = True,+ ensure_leading_blank_line_in_value: bool = False,+):+ """+ Build a tomlkit string node.+ - If `force_multiline` or the text contains '\\n', we emit a triple-quoted multiline string.+ - When multiline, `ensure_trailing_newline` puts the closing quotes on their own line.+ - When multiline, `ensure_leading_blank_line_in_value` inserts a real blank line at the start of the value.+ """+ needs_multiline = force_multiline or ("\n" in text)+ normalized = text++ if needs_multiline:+ if ensure_leading_blank_line_in_value and not normalized.startswith("\n"):+ normalized = "\n" + normalized+ if ensure_trailing_newline and not normalized.endswith("\n"):+ normalized = normalized + "\n"++ use_literal = prefer_literal and ("'''" not in normalized)+ return tk_string(normalized, multiline=needs_multiline, literal=use_literal)+++def _convert_to_inline(value: Any):+ """Recursively convert Python values; dicts -> inline tables; lists kept as arrays."""+ if isinstance(value, Mapping):+ value = cast(Mapping[str, Any], value)+ inline_table_obj = inline_table()+ for key, value_item in value.items():+ inline_table_obj[key] = _convert_to_inline(value_item)++ return inline_table_obj++ if isinstance(value, list):+ value = cast(List[Any], value)+ array_obj = array()+ array_obj.multiline(True) # set to False for single-line arrays+ for element in value:+ if isinstance(element, Mapping):+ element = cast(Mapping[str, Any], element)+ inline_element = inline_table()+ for inner_key, inner_value in element.items():+ inline_element[inner_key] = _convert_to_inline(inner_value)+ array_obj.append(inline_element) # pyright: ignore[reportUnknownMemberType]+ else:+ array_obj.append(_convert_to_inline(element)) # pyright: ignore[reportUnknownMemberType]+ return array_obj++ if isinstance(value, str):+ # Triple quotes if needed (or forced); closing quotes on their own line.+ return make_toml_string(+ value,+ prefer_literal=False, # flip to True for '''...'''+ force_multiline=False, # flip to True to force """...""" even without \n+ ensure_trailing_newline=True, # keep closing """ on its own line+ ensure_leading_blank_line_in_value=False, # flip to True to keep a blank first line+ )+ return value+++def _filter_empty_values(value: Any) -> Any:+ """Filter out empty lists and None values from data structures."""+ if isinstance(value, dict):+ filtered: Dict[str, Any] = {}+ for k, v in cast(Dict[str, Any], value).items():+ filtered_v = _filter_empty_values(v)+ # Keep empty dicts but skip empty lists and None values+ if filtered_v is not None and (not isinstance(filtered_v, list) or filtered_v):+ filtered[k] = filtered_v+ return filtered+ elif isinstance(value, list):+ return [_filter_empty_values(item) for item in cast(List[Any], value) if _filter_empty_values(item) is not None]+ else:+ return value+++def _create_ordered_inline_table(data: Mapping[str, Any]) -> Any:+ """Create an inline table with fields in the expected order."""+ inline_table_obj = inline_table()++ # Define the preferred order for concept structure fields+ field_order = ["type", "definition", "required", "choices", "item_type", "key_type", "value_type"]++ # Add fields in preferred order first+ for field in field_order:+ if field in data:+ value = data[field]+ # Skip empty lists+ if isinstance(value, list) and not value:+ continue+ inline_table_obj[field] = _convert_to_inline(value)++ # Add any remaining fields not in the preferred order+ for key, value in data.items():+ if key not in field_order:+ # Skip empty lists+ if isinstance(value, list) and not value:+ continue+ inline_table_obj[key] = _convert_to_inline(value)++ return inline_table_obj+++def dict_to_toml(data: Mapping[str, Any]) -> str:+ """Convert dictionary to TOML format matching Pipelex expectations."""+ data = remove_none_values_from_dict(data=data)+ data = _filter_empty_values(data)+ document_root = document()++ # Handle top-level fields first (domain, definition, system_prompt, etc.)+ for key, value in data.items():+ if not isinstance(value, Mapping):+ document_root.add(key, _convert_to_inline(value))++ # Handle sections (concepts, pipes)+ for section_key, section_value in data.items():+ if isinstance(section_value, Mapping):+ section_value = cast(Mapping[str, Any], section_value)++ # Skip empty sections+ if not section_value:+ continue++ # Create the section table+ section_table = table()++ # Process each item in the section+ for item_key, item_value in section_value.items():+ if isinstance(item_value, str):+ # Simple string value (e.g., "SimpleData = 'Simple data concept'")+ section_table.add(item_key, _convert_to_inline(item_value))+ elif isinstance(item_value, Mapping):+ # Complex object that needs its own table+ item_value = cast(Mapping[str, Any], item_value)+ item_table = table()++ # Handle the structure field specially+ for field_key, field_value in item_value.items():+ if field_key == "structure" and isinstance(field_value, Mapping):+ # Structure should be its own table [section.item.structure]+ structure_table = table()+ for struct_key, struct_value in cast(Mapping[str, Any], field_value).items():+ if isinstance(struct_value, Mapping):+ structure_table.add(struct_key, _create_ordered_inline_table(cast(Mapping[str, Any], struct_value)))+ else:+ structure_table.add(struct_key, _convert_to_inline(struct_value))+ item_table.add(field_key, structure_table)+ else:+ # Skip empty lists like refines = []+ if isinstance(field_value, list) and not field_value:+ continue+ item_table.add(field_key, _convert_to_inline(field_value))++ section_table.add(item_key, item_table)++ document_root.add(section_key, section_table)++ dumped_content = tomlkit.dumps(document_root) # pyright: ignore[reportUnknownMemberType]++ # Post-process to fix inline table spacing: {key = "value"} -> { key = "value" }++ # Add space after opening brace and before closing brace for TOML inline tables only+ # This regex matches TOML inline tables (containing = signs) but not Jinja2 templates+ dumped_content = re.sub(r"\{([^}]*=[^}]*)\}", r"{ \1 }", dumped_content)++ return dumped_content+++def save_toml_to_path(data: Dict[str, Any], path: str) -> None:+ """Save dictionary as TOML to file path.++ Args:+ data: Dictionary to save as TOML+ path: Path where to save the TOML file+ """+ data_cleaned = data+ with open(path, "w", encoding="utf-8") as file:+ toml_content: str = dict_to_toml(data=data_cleaned)+ cleaned_content = clean_trailing_whitespace(toml_content)+ file.write(cleaned_content)diff --git a/pipelex/tools/templating/template_library.py b/pipelex/tools/templating/template_library.pyindex e94006862..70e7a8624 100644--- a/pipelex/tools/templating/template_library.py+++ b/pipelex/tools/templating/template_library.py@@ -26,8 +26,8 @@ class TemplateLibrary(TemplateProviderAbstract, RootModel[TemplateLibraryRoot]):library_config: ClassVar[LibraryConfig]@classmethod- def make_empty(cls, config_folder_path: str) -> "TemplateLibrary":- cls.library_config = LibraryConfig(config_folder_path=config_folder_path)+ def make_empty(cls, config_dir_path: str) -> "TemplateLibrary":+ cls.library_config = LibraryConfig(config_dir_path=config_dir_path)return cls()@overridediff --git a/pyproject.toml b/pyproject.tomlindex 5a681376a..194524aff 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -1,6 +1,6 @@[project]name = "pipelex"-version = "0.6.10"+version = "0.7.0"description = "Pipelex is an open-source dev tool based on a simple declarative language that lets you define replicable, structured, composable LLM pipelines."authors = [{ name = "Evotis S.A.S.", email = "[email protected]" }]maintainers = [{ name = "Pipelex staff", email = "[email protected]" }]diff --git a/tests/cases/source_code.py b/tests/cases/source_code.pyindex 463b191db..4253aa1b9 100644--- a/tests/cases/source_code.py+++ b/tests/cases/source_code.py@@ -2,8 +2,8 @@from typing import List-from pipelex.core.stuff_content import TextContent-from pipelex.core.working_memory import WorkingMemory+from pipelex.core.memory.working_memory import WorkingMemory+from pipelex.core.stuffs.stuff_content import TextContentdef wrap_lines(working_memory: WorkingMemory) -> TextContent:diff --git a/tests/conftest.py b/tests/conftest.pyindex 685dc6078..1e93d46e4 100644--- a/tests/conftest.py+++ b/tests/conftest.py@@ -7,7 +7,7 @@ import pipelex.configimport pipelex.pipelexfrom pipelex import logfrom pipelex.config import get_config-from pipelex.core.concept_provider_abstract import ConceptProviderAbstract+from pipelex.core.concepts.concept_provider_abstract import ConceptProviderAbstractfrom pipelex.hub import get_concept_providerfrom tests.cases.registry import Fruitdiff --git a/tests/data/tools_data/mock_folder_with_classes/subfolder/test_model_c.py b/tests/data/tools_data/mock_folder_with_classes/subfolder/test_model_c.pyindex b8e966651..3340ac437 100644--- a/tests/data/tools_data/mock_folder_with_classes/subfolder/test_model_c.py+++ b/tests/data/tools_data/mock_folder_with_classes/subfolder/test_model_c.py@@ -1,4 +1,4 @@-from pipelex.core.stuff_content import StuffContent+from pipelex.core.stuffs.stuff_content import StuffContentclass Class3(StuffContent):diff --git a/tests/data/tools_data/mock_folder_with_classes/test_model_a.py b/tests/data/tools_data/mock_folder_with_classes/test_model_a.pyindex d1252e29d..f8c231d70 100644--- a/tests/data/tools_data/mock_folder_with_classes/test_model_a.py+++ b/tests/data/tools_data/mock_folder_with_classes/test_model_a.py@@ -1,6 +1,6 @@from pydantic import BaseModel-from pipelex.core.stuff_content import StuffContent+from pipelex.core.stuffs.stuff_content import StuffContentclass ClassA(BaseModel):diff --git a/tests/data/tools_data/mock_folder_with_classes/test_model_b.py b/tests/data/tools_data/mock_folder_with_classes/test_model_b.pyindex e263e3f23..c7a3c1fdd 100644--- a/tests/data/tools_data/mock_folder_with_classes/test_model_b.py+++ b/tests/data/tools_data/mock_folder_with_classes/test_model_b.py@@ -1,4 +1,4 @@-from pipelex.core.stuff_content import StructuredContent, StuffContent+from pipelex.core.stuffs.stuff_content import StructuredContent, StuffContentclass Class2(StuffContent):diff --git a/tests/data/tools_data/problematic_test_cases.toml b/tests/data/tools_data/problematic_test_cases.tomlindex d88e90a07..9b2977c95 100644--- a/tests/data/tools_data/problematic_test_cases.toml+++ b/tests/data/tools_data/problematic_test_cases.toml@@ -6,7 +6,8 @@ TestInput = "A test input concept"[pipe][pipe.test_pipe_with_issues]-PipeLLM = "Test pipe with trailing whitespace issues"+type = "PipeLLM"+definition = "Test pipe with trailing whitespace issues"inputs = { input_data = "TestInput" }output = "native.Text"prompt_template = """@@ -14,7 +15,8 @@ This template has trailing whitespace after closing quotes"""[pipe.another_problematic_pipe]-PipeLLM = "Another pipe with issues"+type = "PipeLLM"+definition = "Another pipe with issues"output = "native.Text"prompt_template = """Multiple issues herediff --git a/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_1.py b/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_1.pyindex 16cd21ea5..9862c2c57 100644--- a/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_1.py+++ b/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_1.py@@ -2,11 +2,11 @@ import pytestfrom pytest import FixtureRequestfrom pipelex import pretty_print-from pipelex.core.pipe_output import PipeOutput-from pipelex.core.pipe_run_params import PipeRunMode-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory-from pipelex.core.stuff_factory import StuffFactory-from pipelex.core.working_memory_factory import WorkingMemoryFactory+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory+from pipelex.core.pipes.pipe_output import PipeOutput+from pipelex.core.pipes.pipe_run_params import PipeRunMode+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory+from pipelex.core.stuffs.stuff_factory import StuffFactoryfrom pipelex.hub import get_pipe_router, get_report_delegatefrom pipelex.pipeline.job_metadata import JobMetadatafrom tests.test_pipelines.pipe_controllers.pipe_condition.pipe_condition import CategoryInputdiff --git a/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_2.py b/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_2.pyindex 18dd9e1a9..a466918d0 100644--- a/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_2.py+++ b/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_2.py@@ -2,11 +2,11 @@ import pytestfrom pytest import FixtureRequestfrom pipelex import pretty_print-from pipelex.core.pipe_output import PipeOutput-from pipelex.core.pipe_run_params import PipeRunMode-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory-from pipelex.core.stuff_factory import StuffFactory-from pipelex.core.working_memory_factory import WorkingMemoryFactory+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory+from pipelex.core.pipes.pipe_output import PipeOutput+from pipelex.core.pipes.pipe_run_params import PipeRunMode+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory+from pipelex.core.stuffs.stuff_factory import StuffFactoryfrom pipelex.hub import get_pipe_router, get_report_delegatefrom pipelex.pipeline.job_metadata import JobMetadatafrom tests.test_pipelines.pipe_controllers.pipe_condition.pipe_condition import CategoryInputdiff --git a/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.py b/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.pyindex 3abd11548..133375a42 100644--- a/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.py+++ b/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.py@@ -3,10 +3,10 @@import pytestfrom pipelex import pretty_print-from pipelex.core.pipe_input_spec import PipeInputSpec, TypedNamedInputRequirement-from pipelex.core.pipe_run_params import PipeRunMode-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory-from pipelex.core.working_memory_factory import WorkingMemoryFactory+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory+from pipelex.core.pipes.pipe_input_spec import InputRequirementBlueprint, PipeInputSpec, TypedNamedInputRequirement+from pipelex.core.pipes.pipe_run_params import PipeRunMode+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactoryfrom pipelex.exceptions import DryRunErrorfrom pipelex.pipe_controllers.pipe_condition import PipeConditionfrom pipelex.pipeline.job_metadata import JobMetadata@@ -23,7 +23,9 @@ class TestPipeConditionSimple:pipe_condition = PipeCondition(code="test_condition_fail",domain="test_domain",- inputs=PipeInputSpec.make_from_dict(concepts_dict={"user_category": "test_pipe_condition.CategoryInput"}),+ inputs=PipeInputSpec.make_from_blueprint(+ domain="test_domain", blueprint={"user_category": InputRequirementBlueprint(concept_code="test_pipe_condition.CategoryInput")}+ ),output_concept_code="native.Text",expression_template="{{ user_category.category }}",pipe_map={"small": "process_small", "medium": "process_medium", "large": "process_large"},@@ -52,7 +54,9 @@ class TestPipeConditionSimple:pipe_condition = PipeCondition(code="test_condition_succeed",domain="test_domain",- inputs=PipeInputSpec.make_from_dict(concepts_dict={"user_status": "test_pipe_condition.CategoryInput"}),+ inputs=PipeInputSpec.make_from_blueprint(+ domain="test_domain", blueprint={"user_status": InputRequirementBlueprint(concept_code="test_pipe_condition.CategoryInput")}+ ),output_concept_code="native.Text",expression_template="{{ user_status.category }}",pipe_map={diff --git a/tests/e2e/pipelex/pipes/pipe_controller/pipe_parallel/test_pipe_parallel_1.py b/tests/e2e/pipelex/pipes/pipe_controller/pipe_parallel/test_pipe_parallel_1.pyindex 1652398c7..f22730e93 100644--- a/tests/e2e/pipelex/pipes/pipe_controller/pipe_parallel/test_pipe_parallel_1.py+++ b/tests/e2e/pipelex/pipes/pipe_controller/pipe_parallel/test_pipe_parallel_1.py@@ -4,11 +4,11 @@ import pytestfrom pytest import FixtureRequestfrom pipelex import pretty_print-from pipelex.core.pipe_output import PipeOutput-from pipelex.core.pipe_run_params import PipeRunMode-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory-from pipelex.core.stuff_factory import StuffFactory-from pipelex.core.working_memory_factory import WorkingMemoryFactory+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory+from pipelex.core.pipes.pipe_output import PipeOutput+from pipelex.core.pipes.pipe_run_params import PipeRunMode+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory+from pipelex.core.stuffs.stuff_factory import StuffFactoryfrom pipelex.hub import get_pipe_router, get_report_delegatefrom pipelex.pipeline.job_metadata import JobMetadatafrom tests.test_pipelines.pipe_controllers.pipe_parallel.pipe_parallel import ContentAnalysis, DocumentInput, LengthAnalysisdiff --git a/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_batching.py b/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_batching.pyindex 00d947ccf..72f3517f1 100644--- a/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_batching.py+++ b/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_batching.py@@ -5,11 +5,11 @@ from typing import castimport pytestfrom pipelex import log-from pipelex.core.pipe_input_spec import TypedNamedInputRequirement-from pipelex.core.pipe_run_params import PipeRunMode-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory-from pipelex.core.stuff_factory import StuffFactory-from pipelex.core.working_memory_factory import WorkingMemoryFactory+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory+from pipelex.core.pipes.pipe_input_spec import TypedNamedInputRequirement+from pipelex.core.pipes.pipe_run_params import PipeRunMode+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory+from pipelex.core.stuffs.stuff_factory import StuffFactoryfrom pipelex.hub import get_required_pipefrom pipelex.pipeline.execute import execute_pipelinefrom pipelex.pipeline.job_metadata import JobMetadatadiff --git a/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_multiplicity.py b/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_multiplicity.pyindex ed03fecf5..489e5e2b8 100644--- a/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_multiplicity.py+++ b/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_multiplicity.py@@ -3,10 +3,10 @@import pytestfrom pytest import FixtureRequest-from pipelex.core.pipe_run_params import PipeRunMode-from pipelex.core.stuff_content import TextContent-from pipelex.core.stuff_factory import StuffFactory-from pipelex.core.working_memory_factory import WorkingMemoryFactory+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory+from pipelex.core.pipes.pipe_run_params import PipeRunMode+from pipelex.core.stuffs.stuff_content import TextContent+from pipelex.core.stuffs.stuff_factory import StuffFactoryfrom pipelex.pipeline.execute import execute_pipelinediff --git a/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_simple.py b/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_simple.pyindex 901ee4667..0e561f39f 100644--- a/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_simple.py+++ b/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_simple.py@@ -3,11 +3,11 @@import pytestfrom pipelex import pretty_print-from pipelex.core.pipe_run_params import PipeRunMode-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory-from pipelex.core.stuff_content import TextContent-from pipelex.core.stuff_factory import StuffFactory-from pipelex.core.working_memory_factory import WorkingMemoryFactory+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory+from pipelex.core.pipes.pipe_run_params import PipeRunMode+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory+from pipelex.core.stuffs.stuff_content import TextContent+from pipelex.core.stuffs.stuff_factory import StuffFactoryfrom pipelex.hub import get_required_pipefrom pipelex.pipeline.job_metadata import JobMetadatadiff --git a/tests/integration/pipelex/client/test_client.py b/tests/integration/pipelex/client/test_client.pyindex 24bf1f8aa..7fa887796 100644--- a/tests/integration/pipelex/client/test_client.py+++ b/tests/integration/pipelex/client/test_client.py@@ -6,10 +6,10 @@ from pydantic import BaseModelfrom pipelex import pretty_printfrom pipelex.client.client import PipelexClientfrom pipelex.client.protocol import COMPACT_MEMORY_KEY, PipelineState-from pipelex.core.stuff import Stuff-from pipelex.core.stuff_content import TextContent-from pipelex.core.stuff_factory import StuffFactory-from pipelex.core.working_memory import WorkingMemory+from pipelex.core.memory.working_memory import WorkingMemory+from pipelex.core.stuffs.stuff import Stuff+from pipelex.core.stuffs.stuff_content import TextContent+from pipelex.core.stuffs.stuff_factory import StuffFactoryclass Example(BaseModel):diff --git a/tests/integration/pipelex/cogt/test_content_generator.py b/tests/integration/pipelex/cogt/test_content_generator.pyindex 3a2e75623..b0614b2b3 100644--- a/tests/integration/pipelex/cogt/test_content_generator.py+++ b/tests/integration/pipelex/cogt/test_content_generator.py@@ -125,7 +125,6 @@ class TestContentGenerator:context=context,jinja2="The answer is: {{ the_answer }}",)- pretty_print(jinja2_text, title="jinja2_text")assert isinstance(jinja2_text, str)assert jinja2_text == "The answer is: elementary, my dear Watson"diff --git a/tests/integration/pipelex/conftest.py b/tests/integration/pipelex/conftest.pyindex ab299e51c..c576962c0 100644--- a/tests/integration/pipelex/conftest.py+++ b/tests/integration/pipelex/conftest.py@@ -23,12 +23,12 @@ def llm_preset_id(request: pytest.FixtureRequest) -> str:params=[# "o1",# "gpt-4o",- # "gpt-4o-mini",+ "gpt-4o-mini",# "gpt-4-5-preview",# "o1-mini",# "o3-mini",- # "gpt-5-mini",- "gpt-5-nano",+ # "gpt-5-mini", # TODO: fix this+ # "gpt-5-nano", # TODO: fix this# "best-gpt",# "claude-3-haiku",# "claude-3-5-sonnet",@@ -62,9 +62,9 @@ def llm_handle(request: pytest.FixtureRequest) -> str:# "o1",# "o3-mini",# "gpt-4o",- # "gpt-4o-mini",+ "gpt-4o-mini",# "gpt-5-mini",- "gpt-5-nano",+ # "gpt-5-nano",# "gpt-5-chat",# "gpt-4-5-preview",# "claude-3-haiku",diff --git a/tests/integration/pipelex/pipes/conftest.py b/tests/integration/pipelex/pipes/conftest.pyindex dc04dd01f..33ff3509f 100644--- a/tests/integration/pipelex/pipes/conftest.py+++ b/tests/integration/pipelex/pipes/conftest.py@@ -4,7 +4,7 @@ from typing import AsyncIterator, Awaitable, Callableimport pytest_asynciofrom pytest import FixtureRequest-from pipelex.core.pipe_output import PipeOutput+from pipelex.core.pipes.pipe_output import PipeOutputfrom pipelex.hub import get_activity_managerfrom pipelex.pipeline.activity.activity_handler import ActivityHandlerForResultFilesfrom pipelex.tools.misc.file_utils import get_incremental_directory_path, remove_folderdiff --git a/tests/integration/pipelex/pipes/pipe_controller/pipe_batch/test_pipe_batch_simple.py b/tests/integration/pipelex/pipes/pipe_controller/pipe_batch/test_pipe_batch_simple.pyindex 7a7a0e7f9..69c0003d6 100644--- a/tests/integration/pipelex/pipes/pipe_controller/pipe_batch/test_pipe_batch_simple.py+++ b/tests/integration/pipelex/pipes/pipe_controller/pipe_batch/test_pipe_batch_simple.py@@ -6,12 +6,12 @@ import pytestfrom pytest import FixtureRequestfrom pipelex import pretty_print-from pipelex.core.pipe_input_spec import PipeInputSpec-from pipelex.core.pipe_run_params import BatchParams, PipeRunMode-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory-from pipelex.core.stuff_content import ListContent, StuffContent, TextContent-from pipelex.core.stuff_factory import StuffFactory-from pipelex.core.working_memory_factory import WorkingMemoryFactory+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory+from pipelex.core.pipes.pipe_input_spec import InputRequirementBlueprint, PipeInputSpec+from pipelex.core.pipes.pipe_run_params import BatchParams, PipeRunMode+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory+from pipelex.core.stuffs.stuff_content import ListContent, StuffContent, TextContent+from pipelex.core.stuffs.stuff_factory import StuffFactoryfrom pipelex.pipe_controllers.pipe_batch import PipeBatchfrom pipelex.pipeline.job_metadata import JobMetadata@@ -29,7 +29,13 @@ class TestPipeBatchSimple:domain="test_integration",code="simple_batch",branch_pipe_code="uppercase_transformer", # This exists in the TOML file- inputs=PipeInputSpec.make_from_dict(concepts_dict={"text_list": "Text", "text_item": "Text"}),+ inputs=PipeInputSpec.make_from_blueprint(+ domain="test_integration",+ blueprint={+ "text_list": InputRequirementBlueprint(concept_code="Text"),+ "text_item": InputRequirementBlueprint(concept_code="Text"),+ },+ ),output_concept_code="test_integration.UppercaseText",batch_params=BatchParams(input_list_stuff_name="text_list", input_item_stuff_name="text_item"),)diff --git a/tests/integration/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_complex.py b/tests/integration/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_complex.pyindex 3c9ea97ff..3a29a8fb5 100644--- a/tests/integration/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_complex.py+++ b/tests/integration/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_complex.py@@ -6,12 +6,12 @@ import pytestfrom pytest import FixtureRequestfrom pipelex import pretty_print-from pipelex.core.pipe_output import PipeOutput-from pipelex.core.pipe_run_params import PipeRunMode-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory-from pipelex.core.stuff_content import TextContent-from pipelex.core.stuff_factory import StuffFactory-from pipelex.core.working_memory_factory import WorkingMemoryFactory+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory+from pipelex.core.pipes.pipe_output import PipeOutput+from pipelex.core.pipes.pipe_run_params import PipeRunMode+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory+from pipelex.core.stuffs.stuff_content import TextContent+from pipelex.core.stuffs.stuff_factory import StuffFactoryfrom pipelex.exceptions import DryRunErrorfrom pipelex.hub import get_pipe_routerfrom pipelex.pipeline.job_metadata import JobMetadatadiff --git a/tests/integration/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.py b/tests/integration/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.pyindex 9179ded28..269b15903 100644--- a/tests/integration/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.py+++ b/tests/integration/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.py@@ -6,13 +6,13 @@ import pytestfrom pytest import FixtureRequestfrom pipelex import pretty_print-from pipelex.core.pipe_input_spec import PipeInputSpec-from pipelex.core.pipe_output import PipeOutput-from pipelex.core.pipe_run_params import PipeRunMode-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory-from pipelex.core.stuff_content import TextContent-from pipelex.core.stuff_factory import StuffFactory-from pipelex.core.working_memory_factory import WorkingMemoryFactory+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory+from pipelex.core.pipes.pipe_input_spec import InputRequirementBlueprint, PipeInputSpec+from pipelex.core.pipes.pipe_output import PipeOutput+from pipelex.core.pipes.pipe_run_params import PipeRunMode+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory+from pipelex.core.stuffs.stuff_content import TextContent+from pipelex.core.stuffs.stuff_factory import StuffFactoryfrom pipelex.exceptions import DryRunErrorfrom pipelex.hub import get_pipe_routerfrom pipelex.pipe_controllers.pipe_condition import PipeCondition@@ -31,7 +31,9 @@ class TestPipeConditionSimple:pipe_condition = PipeCondition(domain="test_integration",code="text_length_condition",- inputs=PipeInputSpec.make_from_dict(concepts_dict={"input_text": "Text"}),+ inputs=PipeInputSpec.make_from_blueprint(+ domain="test_integration", blueprint={"input_text": InputRequirementBlueprint(concept_code="Text")}+ ),output_concept_code="Text",expression_template="{% if input_text.text|length > 5 %}long{% else %}short{% endif %}",pipe_map={"long": "capitalize_long_text", "short": "add_prefix_short_text"},@@ -100,7 +102,9 @@ class TestPipeConditionSimple:pipe_condition = PipeCondition(domain="test_integration",code="text_length_condition",- inputs=PipeInputSpec.make_from_dict(concepts_dict={"input_text": "Text"}),+ inputs=PipeInputSpec.make_from_blueprint(+ domain="test_integration", blueprint={"input_text": InputRequirementBlueprint(concept_code="Text")}+ ),output_concept_code="Text",expression_template="{% if input_text.text|length > 5 %}long{% else %}short{% endif %}",pipe_map={"long": "capitalize_long_text", "short": "add_prefix_short_text"},diff --git a/tests/integration/pipelex/pipes/pipe_controller/pipe_parallel/test_pipe_parallel_simple.py b/tests/integration/pipelex/pipes/pipe_controller/pipe_parallel/test_pipe_parallel_simple.pyindex 4ee80eea5..f5191c50d 100644--- a/tests/integration/pipelex/pipes/pipe_controller/pipe_parallel/test_pipe_parallel_simple.py+++ b/tests/integration/pipelex/pipes/pipe_controller/pipe_parallel/test_pipe_parallel_simple.py@@ -5,12 +5,12 @@ from typing import castimport pytestfrom pytest import FixtureRequest-from pipelex.core.pipe_input_spec import PipeInputSpec-from pipelex.core.pipe_run_params import PipeRunMode-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory-from pipelex.core.stuff_content import TextContent-from pipelex.core.stuff_factory import StuffFactory-from pipelex.core.working_memory_factory import WorkingMemoryFactory+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory+from pipelex.core.pipes.pipe_input_spec import InputRequirementBlueprint, PipeInputSpec+from pipelex.core.pipes.pipe_run_params import PipeRunMode+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory+from pipelex.core.stuffs.stuff_content import TextContent+from pipelex.core.stuffs.stuff_factory import StuffFactoryfrom pipelex.pipe_controllers.pipe_parallel import PipeParallelfrom pipelex.pipe_controllers.sub_pipe import SubPipefrom pipelex.pipeline.job_metadata import JobMetadata@@ -28,7 +28,9 @@ class TestPipeParallelSimple:pipe_parallel = PipeParallel(domain="test_integration",code="parallel_text_analyzer",- inputs=PipeInputSpec.make_from_dict(concepts_dict={"input_text": "Text"}),+ inputs=PipeInputSpec.make_from_blueprint(+ domain="test_integration", blueprint={"input_text": InputRequirementBlueprint(concept_code="Text")}+ ),output_concept_code="Text",parallel_sub_pipes=[SubPipe(pipe_code="analyze_sentiment", output_name="sentiment_result"),@@ -140,7 +142,9 @@ class TestPipeParallelSimple:pipe_parallel = PipeParallel(domain="test_integration",code="parallel_text_analyzer",- inputs=PipeInputSpec.make_from_dict(concepts_dict={"input_text": "Text"}),+ inputs=PipeInputSpec.make_from_blueprint(+ domain="test_integration", blueprint={"input_text": InputRequirementBlueprint(concept_code="Text")}+ ),output_concept_code="Text",parallel_sub_pipes=[SubPipe(pipe_code="analyze_sentiment", output_name="sentiment_result"),diff --git a/tests/integration/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_dry_run.py b/tests/integration/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_dry_run.pyindex cfc1daea9..b49f15b07 100644--- a/tests/integration/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_dry_run.py+++ b/tests/integration/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_dry_run.py@@ -4,14 +4,14 @@ import pytestfrom pytest import FixtureRequestfrom pipelex import pretty_print-from pipelex.core.pipe_output import PipeOutput-from pipelex.core.pipe_run_params import PipeRunMode-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory-from pipelex.core.stuff import Stuff-from pipelex.core.stuff_content import ListContent-from pipelex.core.stuff_factory import StuffFactory-from pipelex.core.working_memory import WorkingMemory-from pipelex.core.working_memory_factory import WorkingMemoryFactory+from pipelex.core.memory.working_memory import WorkingMemory+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory+from pipelex.core.pipes.pipe_output import PipeOutput+from pipelex.core.pipes.pipe_run_params import PipeRunMode+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory+from pipelex.core.stuffs.stuff import Stuff+from pipelex.core.stuffs.stuff_content import ListContent+from pipelex.core.stuffs.stuff_factory import StuffFactoryfrom pipelex.hub import get_pipe_router, get_report_delegatefrom pipelex.pipeline.job_metadata import JobMetadatafrom pipelex.tools.misc.json_utils import load_json_list_from_pathdiff --git a/tests/integration/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_simple.py b/tests/integration/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_simple.pyindex 965565348..704ea1ab7 100644--- a/tests/integration/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_simple.py+++ b/tests/integration/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_simple.py@@ -6,12 +6,12 @@ import pytestfrom pytest import FixtureRequestfrom pipelex import pretty_print-from pipelex.core.pipe_input_spec import PipeInputSpec-from pipelex.core.pipe_run_params import PipeRunMode-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory-from pipelex.core.stuff_content import TextContent-from pipelex.core.stuff_factory import StuffFactory-from pipelex.core.working_memory_factory import WorkingMemoryFactory+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory+from pipelex.core.pipes.pipe_input_spec import InputRequirementBlueprint, PipeInputSpec+from pipelex.core.pipes.pipe_run_params import PipeRunMode+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory+from pipelex.core.stuffs.stuff_content import TextContent+from pipelex.core.stuffs.stuff_factory import StuffFactoryfrom pipelex.pipe_controllers.pipe_sequence import PipeSequencefrom pipelex.pipe_controllers.sub_pipe import SubPipefrom pipelex.pipeline.job_metadata import JobMetadata@@ -29,7 +29,9 @@ class TestPipeSequenceSimple:pipe_sequence = PipeSequence(domain="test_integration",code="simple_sequence",- inputs=PipeInputSpec.make_from_dict(concepts_dict={"input_text": "Text"}),+ inputs=PipeInputSpec.make_from_blueprint(+ domain="test_integration", blueprint={"input_text": InputRequirementBlueprint(concept_code="Text")}+ ),output_concept_code="Text",sequential_sub_pipes=[SubPipe(pipe_code="capitalize_text", output_name="capitalized_text"),diff --git a/tests/integration/pipelex/pipes/pipe_operator/pipe_func/test_pipe_func.py b/tests/integration/pipelex/pipes/pipe_operator/pipe_func/test_pipe_func.pyindex d36d6219f..5a8ec7a0e 100644--- a/tests/integration/pipelex/pipes/pipe_operator/pipe_func/test_pipe_func.py+++ b/tests/integration/pipelex/pipes/pipe_operator/pipe_func/test_pipe_func.py@@ -1,12 +1,12 @@import pytestfrom pipelex import log, pretty_print-from pipelex.core.concept_native import NativeConcept-from pipelex.core.pipe_run_params import PipeRunMode-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory-from pipelex.core.stuff_content import TextContent-from pipelex.core.stuff_factory import StuffFactory-from pipelex.core.working_memory_factory import WorkingMemoryFactory+from pipelex.core.concepts.concept_native import NativeConcept+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory+from pipelex.core.pipes.pipe_run_params import PipeRunMode+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory+from pipelex.core.stuffs.stuff_content import TextContent+from pipelex.core.stuffs.stuff_factory import StuffFactoryfrom pipelex.hub import get_pipe_router, get_report_delegatefrom pipelex.pipe_operators.pipe_func import PipeFunc, PipeFuncOutputfrom pipelex.pipe_works.pipe_job_factory import PipeJobFactorydiff --git a/tests/integration/pipelex/pipes/pipe_operator/pipe_imgg/test_pipe_imgg.py b/tests/integration/pipelex/pipes/pipe_operator/pipe_imgg/test_pipe_imgg.pyindex 82ed87fee..35c71093c 100644--- a/tests/integration/pipelex/pipes/pipe_operator/pipe_imgg/test_pipe_imgg.py+++ b/tests/integration/pipelex/pipes/pipe_operator/pipe_imgg/test_pipe_imgg.py@@ -2,9 +2,9 @@ import pytestfrom pipelex import pretty_printfrom pipelex.cogt.imgg.imgg_handle import ImggHandle-from pipelex.core.concept_native import NativeConcept-from pipelex.core.pipe_run_params import PipeRunMode-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory+from pipelex.core.concepts.concept_native import NativeConcept+from pipelex.core.pipes.pipe_run_params import PipeRunMode+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactoryfrom pipelex.hub import get_pipe_routerfrom pipelex.pipe_operators.pipe_img_gen import PipeImgGen, PipeImgGenOutputfrom pipelex.pipe_works.pipe_job_factory import PipeJobFactorydiff --git a/tests/integration/pipelex/pipes/pipe_operator/pipe_llm/test_generate_pipeline_blueprint.py b/tests/integration/pipelex/pipes/pipe_operator/pipe_llm/test_generate_pipeline_blueprint.pydeleted file mode 100644index 1e1805bba..000000000--- a/tests/integration/pipelex/pipes/pipe_operator/pipe_llm/test_generate_pipeline_blueprint.py+++ /dev/null@@ -1,46 +0,0 @@-import pytest--from pipelex import pretty_print-from pipelex.core.pipe_run_params import PipeRunMode-from pipelex.create.helpers import get_support_file-from pipelex.libraries.pipeline_blueprint import PipelineBlueprint-from pipelex.pipeline.execute import execute_pipeline--[email protected](loop_scope="class")[email protected]_runnable- "requirements",- [- (- "Create a domain 'sample' to describe simple demo pipelines. "- "Define concept ThingDescription = 'A short natural language description of a thing'. "- "Add a pipe write_thing that takes input { text = Text } and outputs ThingDescription using PipeLLM."- ),- (- "Domain: gadgets. Concept: GadgetSpec = 'Technical description of a gadget'. "- "Add pipe summarize_gadget: inputs { text = Text } -> output GadgetSpec via PipeLLM."- ),- ],-)-async def test_generate_pipeline_blueprint(pipe_run_mode: PipeRunMode, requirements: str):- pipe_output = await execute_pipeline(- pipe_code="build_blueprint",- input_memory={- "domain": "test_domain",- "pipeline_name": "test_pipeline",- "requirements": requirements,- "draft_pipeline_rules": get_support_file(subpath="create/draft_pipelines.md"),- "build_pipeline_rules": get_support_file(subpath="create/build_pipelines.md"),- },- pipe_run_mode=pipe_run_mode,- )-- blueprint = pipe_output.main_stuff_as(content_type=PipelineBlueprint)- assert isinstance(blueprint, PipelineBlueprint)- # Basic sanity checks- assert isinstance(blueprint.domain, str) and blueprint.domain != ""-- pretty_print(blueprint, title="PipelineBlueprint")diff --git a/tests/integration/pipelex/pipes/pipe_operator/pipe_llm/test_pipe_llm.py b/tests/integration/pipelex/pipes/pipe_operator/pipe_llm/test_pipe_llm.pyindex 862a0f0d1..0f93d28a1 100644--- a/tests/integration/pipelex/pipes/pipe_operator/pipe_llm/test_pipe_llm.py+++ b/tests/integration/pipelex/pipes/pipe_operator/pipe_llm/test_pipe_llm.py@@ -3,12 +3,12 @@ from typing import Listimport pytestfrom pipelex import log, pretty_print-from pipelex.core.concept_native import NativeConcept-from pipelex.core.pipe_input_spec import PipeInputSpec-from pipelex.core.pipe_run_params import PipeRunMode-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory-from pipelex.core.stuff import Stuff-from pipelex.core.working_memory_factory import WorkingMemoryFactory+from pipelex.core.concepts.concept_native import NativeConcept+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory+from pipelex.core.pipes.pipe_input_spec import InputRequirementBlueprint, PipeInputSpec+from pipelex.core.pipes.pipe_run_params import PipeRunMode+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory+from pipelex.core.stuffs.stuff import Stufffrom pipelex.hub import get_pipe_router, get_report_delegatefrom pipelex.pipe_operators.pipe_llm import PipeLLM, PipeLLMOutputfrom pipelex.pipe_operators.pipe_llm_prompt import PipeLLMPrompt@@ -67,7 +67,9 @@ class TestPipeLLM:pipe=PipeLLM(code="adhoc_for_test_pipe_llm_image",domain="generic",- inputs=PipeInputSpec.make_from_dict(concepts_dict={stuff_name: stuff.concept_code}),+ inputs=PipeInputSpec.make_from_blueprint(+ domain="generic", blueprint={stuff_name: InputRequirementBlueprint(concept_code=stuff.concept_code)}+ ),output_concept_code=NativeConcept.TEXT.code,pipe_llm_prompt=PipeLLMPrompt(code="adhoc_for_test_pipe_llm_image",diff --git a/tests/integration/pipelex/pipes/pipe_operator/pipe_ocr/test_pipe_ocr.py b/tests/integration/pipelex/pipes/pipe_operator/pipe_ocr/test_pipe_ocr.pyindex 7b11c43b0..87a6b6757 100644--- a/tests/integration/pipelex/pipes/pipe_operator/pipe_ocr/test_pipe_ocr.py+++ b/tests/integration/pipelex/pipes/pipe_operator/pipe_ocr/test_pipe_ocr.py@@ -1,12 +1,12 @@import pytestfrom pipelex import pretty_print-from pipelex.core.concept_native import NativeConcept-from pipelex.core.pipe_input_spec import PipeInputSpec-from pipelex.core.pipe_run_params import PipeRunMode-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory-from pipelex.core.stuff_content import PageContent-from pipelex.core.working_memory_factory import WorkingMemoryFactory+from pipelex.core.concepts.concept_native import NativeConcept+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory+from pipelex.core.pipes.pipe_input_spec import InputRequirementBlueprint, PipeInputSpec+from pipelex.core.pipes.pipe_run_params import PipeRunMode+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory+from pipelex.core.stuffs.stuff_content import PageContentfrom pipelex.hub import get_pipe_routerfrom pipelex.pipe_operators.pipe_ocr import PIPE_OCR_INPUT_NAME, PipeOcr, PipeOcrOutputfrom pipelex.pipe_works.pipe_job_factory import PipeJobFactory@@ -28,7 +28,7 @@ class TestPipeOCR:pipe=PipeOcr(code="adhoc_for_test_pipe_ocr_image",domain="generic",- inputs=PipeInputSpec.make_from_dict(concepts_dict={"page_scan": "Image"}),+ inputs=PipeInputSpec.make_from_blueprint(domain="generic", blueprint={"page_scan": InputRequirementBlueprint(concept_code="Image")}),should_include_images=True,should_caption_images=False,should_include_page_views=True,@@ -58,7 +58,9 @@ class TestPipeOCR:pipe=PipeOcr(code="adhoc_for_test_pipe_ocr_pdf",domain="generic",- inputs=PipeInputSpec.make_from_dict(concepts_dict={PIPE_OCR_INPUT_NAME: "PDF"}),+ inputs=PipeInputSpec.make_from_blueprint(+ domain="generic", blueprint={PIPE_OCR_INPUT_NAME: InputRequirementBlueprint(concept_code=NativeConcept.PDF.code)}+ ),should_include_images=True,should_caption_images=False,should_include_page_views=True,diff --git a/tests/integration/pipelex/pipes/test_image_inputs.py b/tests/integration/pipelex/pipes/test_image_inputs.pyindex bc5d8f6fb..d5eef6d24 100644--- a/tests/integration/pipelex/pipes/test_image_inputs.py+++ b/tests/integration/pipelex/pipes/test_image_inputs.py@@ -2,12 +2,12 @@ import pytestfrom pytest import FixtureRequestfrom pipelex import pretty_print-from pipelex.core.pipe_output import PipeOutput-from pipelex.core.pipe_run_params import PipeRunMode-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory-from pipelex.core.stuff_content import ImageContent, PageContent, TextAndImagesContent, TextContent-from pipelex.core.stuff_factory import StuffFactory-from pipelex.core.working_memory_factory import WorkingMemoryFactory+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory+from pipelex.core.pipes.pipe_output import PipeOutput+from pipelex.core.pipes.pipe_run_params import PipeRunMode+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory+from pipelex.core.stuffs.stuff_content import ImageContent, PageContent, TextAndImagesContent, TextContent+from pipelex.core.stuffs.stuff_factory import StuffFactoryfrom pipelex.hub import get_pipe_router, get_report_delegatefrom pipelex.pipeline.job_metadata import JobMetadatafrom tests.cases import ImageTestCasesdiff --git a/tests/integration/pipelex/pipes/test_pipe_jinja2.py b/tests/integration/pipelex/pipes/test_pipe_jinja2.pyindex 438b01219..988869d06 100644--- a/tests/integration/pipelex/pipes/test_pipe_jinja2.py+++ b/tests/integration/pipelex/pipes/test_pipe_jinja2.py@@ -1,9 +1,9 @@import pytestfrom pipelex import pretty_print-from pipelex.core.pipe_run_params import PipeRunMode-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory-from pipelex.core.working_memory_factory import WorkingMemoryFactory+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory+from pipelex.core.pipes.pipe_run_params import PipeRunMode+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactoryfrom pipelex.hub import get_pipe_routerfrom pipelex.pipe_operators.pipe_jinja2 import PipeJinja2, PipeJinja2Outputfrom pipelex.pipe_works.pipe_job_factory import PipeJobFactorydiff --git a/tests/integration/pipelex/pipes/test_pipe_llm_prompt.py b/tests/integration/pipelex/pipes/test_pipe_llm_prompt.pyindex 7aa68df24..00354d843 100644--- a/tests/integration/pipelex/pipes/test_pipe_llm_prompt.py+++ b/tests/integration/pipelex/pipes/test_pipe_llm_prompt.py@@ -2,11 +2,11 @@ from typing import List, Tuple, Typeimport pytest-from pipelex.core.concept_native import NativeConcept-from pipelex.core.pipe_run_params import PipeRunMode-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory-from pipelex.core.stuff_content import StructuredContent-from pipelex.core.working_memory_factory import WorkingMemoryFactory+from pipelex.core.concepts.concept_native import NativeConcept+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory+from pipelex.core.pipes.pipe_run_params import PipeRunMode+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory+from pipelex.core.stuffs.stuff_content import StructuredContentfrom pipelex.hub import get_pipe_routerfrom pipelex.pipe_operators.pipe_llm_prompt import PipeLLMPrompt, PipeLLMPromptOutputfrom pipelex.pipe_works.pipe_job_factory import PipeJobFactorydiff --git a/tests/integration/pipelex/pipes/test_pipe_running_variants.py b/tests/integration/pipelex/pipes/test_pipe_running_variants.pyindex 4be3baa88..09d0a9e06 100644--- a/tests/integration/pipelex/pipes/test_pipe_running_variants.py+++ b/tests/integration/pipelex/pipes/test_pipe_running_variants.py@@ -6,13 +6,13 @@ from pytest import FixtureRequestfrom pipelex import log, pretty_printfrom pipelex.config import get_config-from pipelex.core.pipe_output import PipeOutput-from pipelex.core.pipe_run_params import PipeOutputMultiplicity, PipeRunMode-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory-from pipelex.core.stuff import Stuff-from pipelex.core.stuff_factory import StuffBlueprint-from pipelex.core.working_memory import WorkingMemory-from pipelex.core.working_memory_factory import WorkingMemoryFactory+from pipelex.core.memory.working_memory import WorkingMemory+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory+from pipelex.core.pipes.pipe_output import PipeOutput+from pipelex.core.pipes.pipe_run_params import PipeOutputMultiplicity, PipeRunMode+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory+from pipelex.core.stuffs.stuff import Stuff+from pipelex.core.stuffs.stuff_factory import StuffBlueprintfrom pipelex.hub import get_library_manager, get_pipe_router, get_report_delegatefrom pipelex.pipeline.activity.activity_handler import ActivityHandlerForResultFilesfrom pipelex.pipeline.job_metadata import JobMetadata@@ -147,9 +147,11 @@ class TestPipeRunningVariants:exception: Type[Exception],expected_error_message: str,):- failing_pipelines_path = get_config().pipelex.library_config.failing_pipelines_path+ failing_pipelines_file_paths = get_config().pipelex.library_config.failing_pipelines_file_pathslibrary_manager = get_library_manager()- library_manager.load_combo_libraries(library_paths=[Path(failing_pipelines_path)])+ library_manager.load_libraries(+ library_file_paths=[Path(failing_pipeline_file_path) for failing_pipeline_file_path in failing_pipelines_file_paths]+ )log.verbose(f"This pipe '{pipe_code}' is supposed to cause an error of type: {exception.__name__}")with pytest.raises(exception) as exc:diff --git a/tests/integration/pipelex/plugins/test_external_plugin.py b/tests/integration/pipelex/plugins/test_external_plugin.pyindex def57eaec..0e10cbcec 100644--- a/tests/integration/pipelex/plugins/test_external_plugin.py+++ b/tests/integration/pipelex/plugins/test_external_plugin.py@@ -11,7 +11,7 @@ from pipelex.cogt.llm.llm_job_factory import LLMJobFactoryfrom pipelex.cogt.llm.llm_models.llm_setting import LLMSetting, LLMSettingChoicesfrom pipelex.cogt.llm.llm_worker_abstract import LLMWorkerAbstractfrom pipelex.cogt.llm.token_category import NbTokensByCategoryDict, TokenCategory-from pipelex.core.concept_native import NativeConcept+from pipelex.core.concepts.concept_native import NativeConceptfrom pipelex.hub import get_inference_manager, get_pipe_router, get_report_delegatefrom pipelex.pipe_operators.pipe_llm import PipeLLM, PipeLLMOutputfrom pipelex.pipe_operators.pipe_llm_prompt import PipeLLMPromptdiff --git a/tests/integration/pipelex/test_data.py b/tests/integration/pipelex/test_data.pyindex eba365325..c1f817596 100644--- a/tests/integration/pipelex/test_data.py+++ b/tests/integration/pipelex/test_data.py@@ -1,15 +1,15 @@from typing import ClassVar, List, Optional, Tuple, Type-from pipelex.core.pipe_run_params import PipeOutputMultiplicity-from pipelex.core.stuff import Stuff-from pipelex.core.stuff_content import (+from pipelex.core.pipes.pipe_run_params import PipeOutputMultiplicity+from pipelex.core.stuffs.stuff import Stuff+from pipelex.core.stuffs.stuff_content import (ImageContent,ListContent,PDFContent,StructuredContent,TextContent,)-from pipelex.core.stuff_factory import StuffBlueprint, StuffFactory+from pipelex.core.stuffs.stuff_factory import StuffBlueprint, StuffFactoryfrom pipelex.exceptions import PipeStackOverflowErrorfrom pipelex.pipe_operators.pipe_ocr import PIPE_OCR_INPUT_NAMEfrom tests.cases import ImageTestCases, PDFTestCasesdiff --git a/tests/integration/pipelex/test_fundamentals.py b/tests/integration/pipelex/test_fundamentals.pyindex 6e751b674..551cf042a 100644--- a/tests/integration/pipelex/test_fundamentals.py+++ b/tests/integration/pipelex/test_fundamentals.py@@ -13,8 +13,7 @@ class TestFundamentals:passdef test_validate_libraries(self):- pipelex_instance = Pipelex.get_instance()- pipelex_instance.validate_libraries()+ Pipelex.get_instance().validate_libraries()@pytest.mark.asyncio(loop_scope="class")async def test_dry_run_all_pipes(self):diff --git a/tests/integration/pipelex/test_inline_structure_concepts.py b/tests/integration/pipelex/test_inline_structure_concepts.pyindex ee6c326fd..dd1ba11ab 100644--- a/tests/integration/pipelex/test_inline_structure_concepts.py+++ b/tests/integration/pipelex/test_inline_structure_concepts.py@@ -1,10 +1,18 @@"""Integration tests for inline structure definitions in concepts."""+from typing import Dict+import pytest-from pipelex.core.concept_factory import ConceptBlueprint, ConceptFactory-from pipelex.core.stuff_content import StructuredContent-from pipelex.exceptions import ConceptFactoryError, StructureClassError+from pipelex.core.concepts.concept_blueprint import (+ ConceptBlueprint,+ ConceptStructureBlueprint,+ ConceptStructureBlueprintFieldType,+ ConceptStructureBlueprintType,+)+from pipelex.core.concepts.concept_factory import ConceptFactory+from pipelex.core.stuffs.stuff_content import StructuredContent+from pipelex.exceptions import StructureClassErrorfrom pipelex.hub import get_class_registry@@ -14,11 +22,19 @@ class TestInlineStructureConcepts:def test_inline_structure_definition_creation(self):"""Test that concepts with inline structure definitions are created correctly."""# Define inline structure with mixed syntax- inline_structure = {- "dominant_feature": {"type": "text", "definition": "The most important feature", "required": True},- "visual_elements": {"type": "text", "definition": "Key visual elements", "required": True},- "composition": "Analysis of the image composition",- "color_palette": "Description of the main colors",+ inline_structure: Dict[str, ConceptStructureBlueprintType] = {+ "dominant_feature": ConceptStructureBlueprint(+ type=ConceptStructureBlueprintFieldType.TEXT, definition="The most important feature", required=False+ ),+ "visual_elements": ConceptStructureBlueprint(+ type=ConceptStructureBlueprintFieldType.TEXT, definition="Key visual elements", required=False+ ),+ "composition": ConceptStructureBlueprint(+ type=ConceptStructureBlueprintFieldType.TEXT, definition="Analysis of the image composition", required=False+ ),+ "color_palette": ConceptStructureBlueprint(+ type=ConceptStructureBlueprintFieldType.TEXT, definition="Description of the main colors", required=False+ ),"mood_atmosphere": "The overall mood or atmosphere",}@@ -82,13 +98,26 @@ class TestInlineStructureConcepts:def test_inline_structure_with_complex_types(self):"""Test inline structure with complex field types."""- inline_structure = {- "title": {"type": "text", "definition": "Document title", "required": True},- "tags": {"type": "list", "item_type": "text", "definition": "List of tags"},- "metadata": {"type": "dict", "key_type": "text", "value_type": "text", "definition": "Metadata dictionary"},- "priority": {"choices": ["low", "medium", "high"], "definition": "Priority level"},- "page_count": {"type": "integer", "definition": "Number of pages"},- "is_active": {"type": "boolean", "definition": "Whether document is active"},+ inline_structure: Dict[str, ConceptStructureBlueprintType] = {+ "title": ConceptStructureBlueprint(type=ConceptStructureBlueprintFieldType.TEXT, definition="Document title"),+ "tags": ConceptStructureBlueprint(+ type=ConceptStructureBlueprintFieldType.LIST,+ item_type=ConceptStructureBlueprintFieldType.TEXT,+ definition="List of tags",+ required=False,+ ),+ "metadata": ConceptStructureBlueprint(+ type=ConceptStructureBlueprintFieldType.DICT,+ key_type=ConceptStructureBlueprintFieldType.TEXT,+ value_type=ConceptStructureBlueprintFieldType.TEXT,+ definition="Metadata dictionary",+ required=False,+ ),+ "priority": ConceptStructureBlueprint(choices=["low", "medium", "high"], definition="Priority level", required=False),+ "page_count": ConceptStructureBlueprint(type=ConceptStructureBlueprintFieldType.INTEGER, definition="Number of pages", required=False),+ "is_active": ConceptStructureBlueprint(+ type=ConceptStructureBlueprintFieldType.BOOLEAN, definition="Whether document is active", required=False+ ),}blueprint = ConceptBlueprint(definition="Complex document structure", structure=inline_structure)@@ -121,38 +150,32 @@ class TestInlineStructureConcepts:def test_invalid_string_reference_raises_error(self):"""Test that invalid string references raise appropriate errors."""- blueprint = ConceptBlueprint(definition="Test invalid reference", structure="NonExistentClass")-- with pytest.raises(StructureClassError, match="Structure class 'NonExistentClass'"):- ConceptFactory.make_concept_from_blueprint(domain="test_domain", code="TestInvalid", concept_blueprint=blueprint)-- def test_inline_structure_generation_error_handling(self):- """Test error handling when inline structure generation fails."""- # Create invalid structure definition that should cause generation to fail- invalid_structure = {- "invalid_field": {"type": "nonexistent_type", "definition": "This should fail"},- }-- blueprint = ConceptBlueprint(definition="Test error handling", structure=invalid_structure)-- with pytest.raises(ConceptFactoryError, match="Error generating structure class"):- ConceptFactory.make_concept_from_blueprint(domain="test_domain", code="TestError", concept_blueprint=blueprint)+ with pytest.raises(+ StructureClassError,+ match="Structure class 'NonExistentClass' set for concept 'TestInvalidRef' in domain 'test_domain' is \+not a registered subclass of StuffContent",+ ):+ _ = ConceptFactory.make_concept_from_blueprint(+ domain="test_domain",+ code="TestInvalidRef",+ concept_blueprint=ConceptBlueprint(definition="Test invalid reference", structure="NonExistentClass"),+ )def test_multiple_inline_structures_do_not_conflict(self):"""Test that multiple inline structures with same field names don't conflict."""# First structure- structure1 = {- "name": {"type": "text", "definition": "Person name", "required": True},- "age": {"type": "integer", "definition": "Person age"},+ structure1: Dict[str, ConceptStructureBlueprintType] = {+ "name": ConceptStructureBlueprint(type=ConceptStructureBlueprintFieldType.TEXT, definition="Person name"),+ "age": ConceptStructureBlueprint(type=ConceptStructureBlueprintFieldType.INTEGER, definition="Person age", required=False),}blueprint1 = ConceptBlueprint(definition="Person information", structure=structure1)concept1 = ConceptFactory.make_concept_from_blueprint(domain="test_domain", code="Person", concept_blueprint=blueprint1)# Second structure with same field names but different context- structure2 = {- "name": {"type": "text", "definition": "Product name", "required": True},- "age": {"type": "integer", "definition": "Product age in days"},+ structure2: Dict[str, ConceptStructureBlueprintType] = {+ "name": "Product name",+ "age": ConceptStructureBlueprint(type=ConceptStructureBlueprintFieldType.INTEGER, definition="Product age in days", required=False),}blueprint2 = ConceptBlueprint(definition="Product information", structure=structure2)diff --git a/tests/integration/pipelex/test_libraries.py b/tests/integration/pipelex/test_libraries.pyindex 428383e0c..89f42cb75 100644--- a/tests/integration/pipelex/test_libraries.py+++ b/tests/integration/pipelex/test_libraries.py@@ -2,6 +2,7 @@make t TEST=TestLibraries"""+from pathlib import Pathfrom typing import Optionalimport pytest@@ -11,9 +12,9 @@ from rich.table import Tablefrom pipelex import pretty_printfrom pipelex.config import get_config-from pipelex.core.concept_library import ConceptLibrary-from pipelex.core.pipe_library import PipeLibrary-from pipelex.libraries.library_manager import LibraryManager+from pipelex.core.concepts.concept_library import ConceptLibrary+from pipelex.core.pipes.pipe_library import PipeLibrary+from pipelex.libraries.library_manager_factory import LibraryManagerFactoryfrom tests.integration.pipelex.test_data import LibraryTestCases@@ -90,10 +91,9 @@ class TestLibraries:known_concept: str,known_pipe: str,):- library_manager = LibraryManager.make_empty(config_folder_path="pipelex_libraries")- test_pipelines_path = get_config().pipelex.library_config.test_pipelines_path- test_library_paths = library_manager.list_toml_files_from_path(library_paths=[test_pipelines_path])- library_manager.load_combo_libraries(library_paths=test_library_paths)+ library_manager = LibraryManagerFactory.make_empty(config_dir_path="pipelex/libraries")+ test_pipelines_dir = [Path(get_config().pipelex.library_config.test_pipelines_dir_path)]+ library_manager.load_libraries(library_dirs=test_pipelines_dir)# Verify that libraries were loadedassert len(library_manager.concept_library.root) > 0, "No concepts were loaded"assert len(library_manager.pipe_library.root) > 0, "No pipes were loaded"diff --git a/tests/integration/pipelex/tools/class_registry/test_class_registry_utils.py b/tests/integration/pipelex/tools/class_registry/test_class_registry_utils.pyindex 6e42f5fa9..2e4e40af4 100644--- a/tests/integration/pipelex/tools/class_registry/test_class_registry_utils.py+++ b/tests/integration/pipelex/tools/class_registry/test_class_registry_utils.py@@ -2,7 +2,7 @@ from pathlib import Pathimport pytest-from pipelex.core.stuff_content import StuffContent+from pipelex.core.stuffs.stuff_content import StuffContentfrom pipelex.hub import get_class_registryfrom pipelex.tools.class_registry_utils import ClassRegistryUtilsfrom tests.cases import ClassRegistryTestCasesdiff --git a/tests/test_pipelines/concept_library/is_compatible_concept_code.py b/tests/test_pipelines/concept_library/is_compatible_concept_code.pyindex 14ebdaaa9..e3ddcd65d 100644--- a/tests/test_pipelines/concept_library/is_compatible_concept_code.py+++ b/tests/test_pipelines/concept_library/is_compatible_concept_code.py@@ -2,7 +2,7 @@ from typing import Dict, List, Optionalfrom pydantic import Field-from pipelex.core.stuff_content import StructuredContent+from pipelex.core.stuffs.stuff_content import StructuredContentclass FundamentalsDoc(StructuredContent):diff --git a/tests/test_pipelines/datetime.py b/tests/test_pipelines/datetime.pyindex e4315e917..1cbb94df8 100644--- a/tests/test_pipelines/datetime.py+++ b/tests/test_pipelines/datetime.py@@ -1,6 +1,6 @@from datetime import datetime-from pipelex.core.stuff_content import StructuredContent+from pipelex.core.stuffs.stuff_content import StructuredContentclass DateTimeEvent(StructuredContent):diff --git a/tests/test_pipelines/discord_newsletter.py b/tests/test_pipelines/discord_newsletter.pyindex 62a12a059..2680b2110 100644--- a/tests/test_pipelines/discord_newsletter.py+++ b/tests/test_pipelines/discord_newsletter.py@@ -2,7 +2,7 @@ from typing import Listfrom pydantic import Field-from pipelex.core.stuff_content import StructuredContent+from pipelex.core.stuffs.stuff_content import StructuredContentclass Attachment(StructuredContent):diff --git a/tests/test_pipelines/misc_tests/test_structures.py b/tests/test_pipelines/misc_tests/test_structures.pyindex 5444d6a55..ea913265b 100644--- a/tests/test_pipelines/misc_tests/test_structures.py+++ b/tests/test_pipelines/misc_tests/test_structures.py@@ -1,4 +1,4 @@-from pipelex.core.stuff_content import StructuredContent+from pipelex.core.stuffs.stuff_content import StructuredContentclass Article(StructuredContent):diff --git a/tests/test_pipelines/pipe_controllers/pipe_batch/text_list.py b/tests/test_pipelines/pipe_controllers/pipe_batch/text_list.pyindex 1b8052713..242ee47a1 100644--- a/tests/test_pipelines/pipe_controllers/pipe_batch/text_list.py+++ b/tests/test_pipelines/pipe_controllers/pipe_batch/text_list.py@@ -1,4 +1,4 @@-from pipelex.core.stuff_content import ListContent, TextContent+from pipelex.core.stuffs.stuff_content import ListContent, TextContentclass TextList(ListContent[TextContent]):diff --git a/tests/test_pipelines/pipe_controllers/pipe_condition/pipe_condition.py b/tests/test_pipelines/pipe_controllers/pipe_condition/pipe_condition.pyindex acf5c8d6b..ffc36a013 100644--- a/tests/test_pipelines/pipe_controllers/pipe_condition/pipe_condition.py+++ b/tests/test_pipelines/pipe_controllers/pipe_condition/pipe_condition.py@@ -1,6 +1,6 @@from pydantic import Field-from pipelex.core.stuff_content import StructuredContent+from pipelex.core.stuffs.stuff_content import StructuredContentclass CategoryInput(StructuredContent):diff --git a/tests/test_pipelines/pipe_controllers/pipe_condition/pipe_condition_complex.py b/tests/test_pipelines/pipe_controllers/pipe_condition/pipe_condition_complex.pyindex 41527c4a0..791c1ffe1 100644--- a/tests/test_pipelines/pipe_controllers/pipe_condition/pipe_condition_complex.py+++ b/tests/test_pipelines/pipe_controllers/pipe_condition/pipe_condition_complex.py@@ -2,7 +2,7 @@ from typing import Literalfrom pydantic import Field-from pipelex.core.stuff_content import StructuredContent+from pipelex.core.stuffs.stuff_content import StructuredContentclass DocumentRequest(StructuredContent):diff --git a/tests/test_pipelines/pipe_controllers/pipe_parallel/pipe_parallel.py b/tests/test_pipelines/pipe_controllers/pipe_parallel/pipe_parallel.pyindex 7f8b63358..fea75b4e8 100644--- a/tests/test_pipelines/pipe_controllers/pipe_parallel/pipe_parallel.py+++ b/tests/test_pipelines/pipe_controllers/pipe_parallel/pipe_parallel.py@@ -1,6 +1,6 @@from pydantic import Field-from pipelex.core.stuff_content import StructuredContent+from pipelex.core.stuffs.stuff_content import StructuredContentclass DocumentInput(StructuredContent):diff --git a/tests/test_pipelines/pipe_controllers/pipe_sequence/pipe_sequence.py b/tests/test_pipelines/pipe_controllers/pipe_sequence/pipe_sequence.pyindex b7c7fb4b3..d1f6dcdaf 100644--- a/tests/test_pipelines/pipe_controllers/pipe_sequence/pipe_sequence.py+++ b/tests/test_pipelines/pipe_controllers/pipe_sequence/pipe_sequence.py@@ -1,6 +1,6 @@from pydantic import Field-from pipelex.core.stuff_content import StructuredContent+from pipelex.core.stuffs.stuff_content import StructuredContentclass Document(StructuredContent):diff --git a/tests/test_pipelines/pipe_llm_prompt.py b/tests/test_pipelines/pipe_llm_prompt.pyindex d920a22a6..79920a4f1 100644--- a/tests/test_pipelines/pipe_llm_prompt.py+++ b/tests/test_pipelines/pipe_llm_prompt.py@@ -2,7 +2,7 @@ from typing import List, Literalfrom pydantic import Field-from pipelex.core.stuff_content import StructuredContent+from pipelex.core.stuffs.stuff_content import StructuredContentfrom pipelex.types import StrEnumdiff --git a/tests/test_pipelines/test_content_factory.py b/tests/test_pipelines/test_content_factory.pyindex e6a11cd34..d40a2cb5e 100644--- a/tests/test_pipelines/test_content_factory.py+++ b/tests/test_pipelines/test_content_factory.py@@ -1,4 +1,4 @@-from pipelex.core.stuff_content import StructuredContent+from pipelex.core.stuffs.stuff_content import StructuredContentclass MockRegisteredContent(StructuredContent):diff --git a/tests/test_pipelines/tricky_questions.py b/tests/test_pipelines/tricky_questions.pyindex c72be0fdd..b591b8c21 100644--- a/tests/test_pipelines/tricky_questions.py+++ b/tests/test_pipelines/tricky_questions.py@@ -1,4 +1,4 @@-from pipelex.core.stuff_content import StructuredContent+from pipelex.core.stuffs.stuff_content import StructuredContentclass ThoughtfulAnswer(StructuredContent):diff --git a/tests/unit/pipelex/client/test_api_serialization.py b/tests/unit/pipelex/client/test_api_serialization.pyindex 504f6e3d0..e3d69a0a4 100644--- a/tests/unit/pipelex/client/test_api_serialization.py+++ b/tests/unit/pipelex/client/test_api_serialization.py@@ -8,11 +8,11 @@ import pytestfrom pydantic import BaseModelfrom pipelex.client.api_serializer import ApiSerializer-from pipelex.core.concept_native import NativeConcept-from pipelex.core.stuff_content import NumberContent-from pipelex.core.stuff_factory import StuffFactory-from pipelex.core.working_memory import WorkingMemory-from pipelex.core.working_memory_factory import WorkingMemoryFactory+from pipelex.core.concepts.concept_native import NativeConcept+from pipelex.core.memory.working_memory import WorkingMemory+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory+from pipelex.core.stuffs.stuff_content import NumberContent+from pipelex.core.stuffs.stuff_factory import StuffFactoryfrom tests.test_pipelines.datetime import DateTimeEventdiff --git a/tests/unit/pipelex/core/bundles/test_pipelex_bundle.py b/tests/unit/pipelex/core/bundles/test_pipelex_bundle.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/unit/pipelex/core/test_concept_library.py b/tests/unit/pipelex/core/concepts/test_concept_library.pysimilarity index 98%rename from tests/unit/pipelex/core/test_concept_library.pyrename to tests/unit/pipelex/core/concepts/test_concept_library.pyindex 58603661b..a719a4b89 100644--- a/tests/unit/pipelex/core/test_concept_library.py+++ b/tests/unit/pipelex/core/concepts/test_concept_library.py@@ -1,6 +1,6 @@-from pipelex.core.concept import Concept-from pipelex.core.concept_native import NativeConcept-from pipelex.core.concept_provider_abstract import ConceptProviderAbstract+from pipelex.core.concepts.concept import Concept+from pipelex.core.concepts.concept_native import NativeConcept+from pipelex.core.concepts.concept_provider_abstract import ConceptProviderAbstractclass Testget_concept_providerIsNativeConcept:diff --git a/tests/unit/pipelex/core/test_concept_pydantic_validation.py b/tests/unit/pipelex/core/concepts/test_concept_pydantic_validation.pysimilarity index 98%rename from tests/unit/pipelex/core/test_concept_pydantic_validation.pyrename to tests/unit/pipelex/core/concepts/test_concept_pydantic_validation.pyindex e170387a3..023df56f6 100644--- a/tests/unit/pipelex/core/test_concept_pydantic_validation.py+++ b/tests/unit/pipelex/core/concepts/test_concept_pydantic_validation.py@@ -1,7 +1,7 @@import pytest-from pipelex.core.concept import Concept-from pipelex.core.concept_native import NativeConcept+from pipelex.core.concepts.concept import Concept+from pipelex.core.concepts.concept_native import NativeConceptfrom pipelex.exceptions import ConceptCodeError, ConceptDomainErrordiff --git a/tests/unit/pipelex/core/test_concept_refines.py b/tests/unit/pipelex/core/concepts/test_concept_refines.pysimilarity index 97%rename from tests/unit/pipelex/core/test_concept_refines.pyrename to tests/unit/pipelex/core/concepts/test_concept_refines.pyindex c5b3cc696..c5acb9377 100644--- a/tests/unit/pipelex/core/test_concept_refines.py+++ b/tests/unit/pipelex/core/concepts/test_concept_refines.py@@ -1,7 +1,7 @@import pytest-from pipelex.core.concept import Concept-from pipelex.core.concept_native import NativeConcept+from pipelex.core.concepts.concept import Concept+from pipelex.core.concepts.concept_native import NativeConceptfrom pipelex.exceptions import ConceptCodeError, ConceptDomainError, ConceptErrordiff --git a/tests/unit/pipelex/core/test_dry_working_memory.py b/tests/unit/pipelex/core/memory/test_dry_working_memory.pysimilarity index 97%rename from tests/unit/pipelex/core/test_dry_working_memory.pyrename to tests/unit/pipelex/core/memory/test_dry_working_memory.pyindex 17b76a39d..96e100e61 100644--- a/tests/unit/pipelex/core/test_dry_working_memory.py+++ b/tests/unit/pipelex/core/memory/test_dry_working_memory.py@@ -6,10 +6,10 @@ import pytestfrom pytest import FixtureRequestfrom pipelex import log-from pipelex.core.concept_native import NativeConcept-from pipelex.core.pipe_input_spec import TypedNamedInputRequirement-from pipelex.core.stuff_content import PageContent, TextContent-from pipelex.core.working_memory_factory import WorkingMemoryFactory+from pipelex.core.concepts.concept_native import NativeConcept+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory+from pipelex.core.pipes.pipe_input_spec import TypedNamedInputRequirement+from pipelex.core.stuffs.stuff_content import PageContent, TextContentfrom tests.test_pipelines.tricky_questions import ThoughtfulAnswerdiff --git a/tests/unit/pipelex/core/test_working_memory.py b/tests/unit/pipelex/core/memory/test_working_memory.pysimilarity index 95%rename from tests/unit/pipelex/core/test_working_memory.pyrename to tests/unit/pipelex/core/memory/test_working_memory.pyindex 0e023b08b..10fca9b4d 100644--- a/tests/unit/pipelex/core/test_working_memory.py+++ b/tests/unit/pipelex/core/memory/test_working_memory.py@@ -2,11 +2,11 @@ from typing import ClassVar, List, Tupleimport pytest-from pipelex.core.concept_native import NativeConcept-from pipelex.core.stuff_content import HtmlContent, ImageContent, ListContent, NumberContent, TextAndImagesContent, TextContent-from pipelex.core.stuff_factory import StuffFactory-from pipelex.core.working_memory import WorkingMemory-from pipelex.core.working_memory_factory import WorkingMemoryFactory+from pipelex.core.concepts.concept_native import NativeConcept+from pipelex.core.memory.working_memory import WorkingMemory+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory+from pipelex.core.stuffs.stuff_content import HtmlContent, ImageContent, ListContent, NumberContent, TextAndImagesContent, TextContent+from pipelex.core.stuffs.stuff_factory import StuffFactoryclass TestWorkingMemoryData:diff --git a/tests/unit/pipelex/core/test_working_memory_factory.py b/tests/unit/pipelex/core/memory/test_working_memory_factory.pysimilarity index 96%rename from tests/unit/pipelex/core/test_working_memory_factory.pyrename to tests/unit/pipelex/core/memory/test_working_memory_factory.pyindex cde6aa5e6..2082ae354 100644--- a/tests/unit/pipelex/core/test_working_memory_factory.py+++ b/tests/unit/pipelex/core/memory/test_working_memory_factory.py@@ -1,7 +1,7 @@from pipelex.client.protocol import CompactMemory-from pipelex.core.concept_native import NativeConcept-from pipelex.core.stuff_content import ImageContent, PageContent, TextAndImagesContent, TextContent-from pipelex.core.working_memory_factory import WorkingMemoryFactory+from pipelex.core.concepts.concept_native import NativeConcept+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory+from pipelex.core.stuffs.stuff_content import ImageContent, PageContent, TextAndImagesContent, TextContentclass TestWorkingMemoryFactory:diff --git a/tests/unit/pipelex/core/test_pipe_condition_validation.py b/tests/unit/pipelex/core/pipes/test_pipe_condition_validation.pysimilarity index 74%rename from tests/unit/pipelex/core/test_pipe_condition_validation.pyrename to tests/unit/pipelex/core/pipes/test_pipe_condition_validation.pyindex 18116b8e8..6f3efde60 100644--- a/tests/unit/pipelex/core/test_pipe_condition_validation.py+++ b/tests/unit/pipelex/core/pipes/test_pipe_condition_validation.py@@ -1,4 +1,4 @@-from pipelex.core.pipe_input_spec import PipeInputSpec+from pipelex.core.pipes.pipe_input_spec import InputRequirementBlueprint, PipeInputSpecfrom pipelex.pipe_controllers.pipe_condition import PipeCondition@@ -10,7 +10,9 @@ class TestPipeConditionValidation:pipe_condition = PipeCondition(domain="test_domain",code="test_condition",- inputs=PipeInputSpec.make_from_dict(concepts_dict={"input_var": "test_domain.Text"}),+ inputs=PipeInputSpec.make_from_blueprint(+ domain="test_domain", blueprint={"input_var": InputRequirementBlueprint(concept_code="test_domain.Text")}+ ),output_concept_code="test_domain.ProcessedText",expression="input_var",pipe_map={"value1": "pipe_a", "value2": "pipe_b"},@@ -29,7 +31,9 @@ class TestPipeConditionValidation:pipe_condition_template = PipeCondition(domain="test_domain",code="test_condition_template",- inputs=PipeInputSpec.make_from_dict(concepts_dict={"var": "test_domain.Text"}),+ inputs=PipeInputSpec.make_from_blueprint(+ domain="test_domain", blueprint={"var": InputRequirementBlueprint(concept_code="test_domain.Text")}+ ),output_concept_code="test_domain.Result",expression_template="{{ var }}",pipe_map={"value": "target_pipe"},@@ -39,7 +43,9 @@ class TestPipeConditionValidation:pipe_condition_expr = PipeCondition(domain="test_domain",code="test_condition_expr",- inputs=PipeInputSpec.make_from_dict(concepts_dict={"var": "test_domain.Text"}),+ inputs=PipeInputSpec.make_from_blueprint(+ domain="test_domain", blueprint={"var": InputRequirementBlueprint(concept_code="test_domain.Text")}+ ),output_concept_code="test_domain.Result",expression="var",pipe_map={"value": "target_pipe"},diff --git a/tests/unit/pipelex/core/test_pipe_parallel_validation.py b/tests/unit/pipelex/core/pipes/test_pipe_parallel_validation.pysimilarity index 81%rename from tests/unit/pipelex/core/test_pipe_parallel_validation.pyrename to tests/unit/pipelex/core/pipes/test_pipe_parallel_validation.pyindex 32186496a..3d8e2826c 100644--- a/tests/unit/pipelex/core/test_pipe_parallel_validation.py+++ b/tests/unit/pipelex/core/pipes/test_pipe_parallel_validation.py@@ -1,4 +1,4 @@-from pipelex.core.pipe_input_spec import PipeInputSpec+from pipelex.core.pipes.pipe_input_spec import InputRequirementBlueprint, PipeInputSpecfrom pipelex.pipe_controllers.pipe_parallel import PipeParallelfrom pipelex.pipe_controllers.sub_pipe import SubPipefrom pipelex.pipe_operators.pipe_llm import PipeLLM@@ -32,7 +32,13 @@ class TestPipeParallelValidation:pipe_parallel = PipeParallel(domain="test_domain",code="parallel_document_processor",- inputs=PipeInputSpec.make_from_dict(concepts_dict={"document": "test_domain.document", "context": "test_domain.context"}),+ inputs=PipeInputSpec.make_from_blueprint(+ domain="test_domain",+ blueprint={+ "document": InputRequirementBlueprint(concept_code="test_domain.document"),+ "context": InputRequirementBlueprint(concept_code="test_domain.context"),+ },+ ),output_concept_code="test_domain.ProcessedAnalysis",parallel_sub_pipes=[SubPipe(pipe_code="analyze_document", output_name="analysis_result")],add_each_output=True,@@ -55,7 +61,9 @@ class TestPipeParallelValidation:pipe_parallel = PipeParallel(domain="test_domain",code="test_parallel",- inputs=PipeInputSpec.make_from_dict(concepts_dict={"input_var": "test_domain.Text"}),+ inputs=PipeInputSpec.make_from_blueprint(+ domain="test_domain", blueprint={"input_var": InputRequirementBlueprint(concept_code="test_domain.Text")}+ ),output_concept_code="test_domain.ProcessedText",parallel_sub_pipes=[SubPipe(pipe_code="test_pipe_1", output_name="result_1")],add_each_output=True,@@ -78,7 +86,13 @@ class TestPipeParallelValidation:pipe_parallel = PipeParallel(domain="test_domain",code="parallel_document_processor",- inputs=PipeInputSpec.make_from_dict(concepts_dict={"document": "test_domain.Document", "context": "test_domain.Context"}),+ inputs=PipeInputSpec.make_from_blueprint(+ domain="test_domain",+ blueprint={+ "document": InputRequirementBlueprint(concept_code="test_domain.Document"),+ "context": InputRequirementBlueprint(concept_code="test_domain.Context"),+ },+ ),output_concept_code="test_domain.ProcessedAnalysis",parallel_sub_pipes=[], # No sub-pipes to avoid dependency issuesadd_each_output=True,diff --git a/tests/unit/pipelex/core/test_pipe_sequence_validation.py b/tests/unit/pipelex/core/pipes/test_pipe_sequence_validation.pysimilarity index 77%rename from tests/unit/pipelex/core/test_pipe_sequence_validation.pyrename to tests/unit/pipelex/core/pipes/test_pipe_sequence_validation.pyindex c92816f2a..eac0444bf 100644--- a/tests/unit/pipelex/core/test_pipe_sequence_validation.py+++ b/tests/unit/pipelex/core/pipes/test_pipe_sequence_validation.py@@ -1,4 +1,4 @@-from pipelex.core.pipe_input_spec import PipeInputSpec+from pipelex.core.pipes.pipe_input_spec import InputRequirementBlueprint, PipeInputSpecfrom pipelex.pipe_controllers.pipe_sequence import PipeSequencefrom pipelex.pipe_controllers.sub_pipe import SubPipe@@ -11,7 +11,9 @@ class TestPipeSequenceValidation:pipe_sequence = PipeSequence(domain="test_domain",code="test_sequence",- inputs=PipeInputSpec.make_from_dict(concepts_dict={"text": "test_domain.Text"}),+ inputs=PipeInputSpec.make_from_blueprint(+ domain="test_domain", blueprint={"text": InputRequirementBlueprint(concept_code="test_domain.Text")}+ ),output_concept_code="test_domain.ProcessedText",sequential_sub_pipes=[SubPipe(pipe_code="test_pipe_1", output_name="intermediate_result")],)@@ -27,7 +29,9 @@ class TestPipeSequenceValidation:pipe_sequence = PipeSequence(domain="test_domain",code="test_sequence",- inputs=PipeInputSpec.make_from_dict(concepts_dict={"initial_input": "test_domain.Text"}),+ inputs=PipeInputSpec.make_from_blueprint(+ domain="test_domain", blueprint={"initial_input": InputRequirementBlueprint(concept_code="test_domain.Text")}+ ),output_concept_code="test_domain.FinalOutput",sequential_sub_pipes=[SubPipe(pipe_code="step_1", output_name="intermediate"), SubPipe(pipe_code="step_2", output_name="final_output")],)diff --git a/tests/unit/pipelex/core/test_stuff_content_factory.py b/tests/unit/pipelex/core/stuffs/test_stuff_content_factory.pysimilarity index 98%rename from tests/unit/pipelex/core/test_stuff_content_factory.pyrename to tests/unit/pipelex/core/stuffs/test_stuff_content_factory.pyindex 38a4a4970..06e907e25 100644--- a/tests/unit/pipelex/core/test_stuff_content_factory.py+++ b/tests/unit/pipelex/core/stuffs/test_stuff_content_factory.py@@ -2,8 +2,8 @@ from typing import Any, ClassVar, Dict, List, Tupleimport pytest-from pipelex.core.stuff_content import StructuredContent, TextContent-from pipelex.core.stuff_factory import StuffContentFactory+from pipelex.core.stuffs.stuff_content import StructuredContent, TextContent+from pipelex.core.stuffs.stuff_factory import StuffContentFactoryclass TestCases:diff --git a/tests/unit/pipelex/core/test_data/__init__.py b/tests/unit/pipelex/core/test_data/__init__.pynew file mode 100644index 000000000..d63ec8f8f--- /dev/null+++ b/tests/unit/pipelex/core/test_data/__init__.py@@ -0,0 +1,52 @@+"""Organized test data for PipelexInterpreter tests."""++from typing import ClassVar, List, Tuple++from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint++# Import all test data from organized modules+from .complex.multi_feature import COMPLEX_TEST_CASES+from .concepts.refining_concepts import REFINING_CONCEPT_TEST_CASES+from .concepts.simple_concepts import SIMPLE_CONCEPT_TEST_CASES+from .concepts.structured_concepts import STRUCTURED_CONCEPT_TEST_CASES+from .domain.simple_domains import DOMAIN_TEST_CASES+from .errors.invalid_toml import ERROR_TEST_CASES+from .pipes.controllers.batch.pipe_batch import PIPE_BATCH_TEST_CASES+from .pipes.controllers.condition.pipe_condition import PIPE_CONDITION_TEST_CASES+from .pipes.controllers.parallel.pipe_parallel import PIPE_PARALLEL_TEST_CASES+from .pipes.controllers.sequence.pipe_sequence import PIPE_SEQUENCE_TEST_CASES+from .pipes.operators.func.pipe_func import PIPE_FUNC_TEST_CASES+from .pipes.operators.img_gen.pipe_img_gen import PIPE_IMG_GEN_TEST_CASES+from .pipes.operators.jinja2.pipe_jinja2 import PIPE_JINJA2_TEST_CASES+from .pipes.operators.llm.pipe_llm import PIPE_LLM_TEST_CASES+from .pipes.operators.ocr.pipe_ocr import PIPE_OCR_TEST_CASES+++class InterpreterTestCases:+ """Test cases for PipelexInterpreter with TOML content and expected blueprints."""++ # Aggregate all valid test cases from organized modules+ VALID_TEST_CASES: ClassVar[List[Tuple[str, str, PipelexBundleBlueprint]]] = [+ # Domain tests+ *DOMAIN_TEST_CASES,+ # Concept tests+ *STRUCTURED_CONCEPT_TEST_CASES,+ *REFINING_CONCEPT_TEST_CASES,+ *SIMPLE_CONCEPT_TEST_CASES,+ # Pipe operator tests+ *PIPE_LLM_TEST_CASES,+ *PIPE_OCR_TEST_CASES,+ *PIPE_FUNC_TEST_CASES,+ *PIPE_IMG_GEN_TEST_CASES,+ *PIPE_JINJA2_TEST_CASES,+ # Pipe controller tests+ *PIPE_SEQUENCE_TEST_CASES,+ *PIPE_CONDITION_TEST_CASES,+ *PIPE_PARALLEL_TEST_CASES,+ *PIPE_BATCH_TEST_CASES,+ # Complex tests+ *COMPLEX_TEST_CASES,+ ]++ # Error test cases+ ERROR_TEST_CASES: ClassVar[List[Tuple[str, str]]] = ERROR_TEST_CASESdiff --git a/tests/unit/pipelex/core/test_data/complex/__init__.py b/tests/unit/pipelex/core/test_data/complex/__init__.pynew file mode 100644index 000000000..bbba9d0b6--- /dev/null+++ b/tests/unit/pipelex/core/test_data/complex/__init__.py@@ -0,0 +1 @@+"""Complex test scenarios for PipelexInterpreter tests."""diff --git a/tests/unit/pipelex/core/test_data/complex/multi_feature.py b/tests/unit/pipelex/core/test_data/complex/multi_feature.pynew file mode 100644index 000000000..fac28db5b--- /dev/null+++ b/tests/unit/pipelex/core/test_data/complex/multi_feature.py@@ -0,0 +1,67 @@+"""Complex test cases combining multiple features."""++from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint+from pipelex.core.pipes.pipe_input_spec import InputRequirementBlueprint+from pipelex.pipe_controllers.pipe_sequence_factory import PipeSequenceBlueprint+from pipelex.pipe_controllers.sub_pipe_factory import SubPipeBlueprint+from pipelex.pipe_operators.pipe_llm_factory import PipeLLMBlueprint++COMPLEX_PIPES = (+ "complex_pipes",+ """domain = "complex_domain"+definition = "Domain with multiple pipe types"++[concept]+InputData = "Input data concept"+ProcessedData = "Processed data concept"++[pipe.llm_pipe]+type = "PipeLLM"+definition = "Generate content"+output = "ProcessedData"+inputs = { data = "InputData" }+prompt_template = "Process this data: @data"++[pipe.sequence_pipe]+type = "PipeSequence"+definition = "Sequential processing"+output = "ProcessedData"+inputs = { input_data = "InputData" }+steps = [+ { pipe = "llm_pipe", result = "llm_result" },+ { pipe = "final_step", result = "final_output" },+]+""",+ PipelexBundleBlueprint(+ domain="complex_domain",+ definition="Domain with multiple pipe types",+ concept={+ "InputData": "Input data concept",+ "ProcessedData": "Processed data concept",+ },+ pipe={+ "llm_pipe": PipeLLMBlueprint(+ type="PipeLLM",+ definition="Generate content",+ inputs={"data": InputRequirementBlueprint(concept_code="InputData")},+ output="ProcessedData",+ prompt_template="Process this data: @data",+ ),+ "sequence_pipe": PipeSequenceBlueprint(+ type="PipeSequence",+ definition="Sequential processing",+ inputs={"input_data": InputRequirementBlueprint(concept_code="InputData")},+ output="ProcessedData",+ steps=[+ SubPipeBlueprint(pipe="llm_pipe", result="llm_result"),+ SubPipeBlueprint(pipe="final_step", result="final_output"),+ ],+ ),+ },+ ),+)++# Export all complex test cases+COMPLEX_TEST_CASES = [+ COMPLEX_PIPES,+]diff --git a/tests/unit/pipelex/core/test_data/concepts/__init__.py b/tests/unit/pipelex/core/test_data/concepts/__init__.pynew file mode 100644index 000000000..7d431fc5a--- /dev/null+++ b/tests/unit/pipelex/core/test_data/concepts/__init__.py@@ -0,0 +1 @@+"""Concept-related test data for PipelexInterpreter tests."""diff --git a/tests/unit/pipelex/core/test_data/concepts/refining_concepts.py b/tests/unit/pipelex/core/test_data/concepts/refining_concepts.pynew file mode 100644index 000000000..7f486319d--- /dev/null+++ b/tests/unit/pipelex/core/test_data/concepts/refining_concepts.py@@ -0,0 +1,33 @@+"""Refining concept test cases."""++from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint+from pipelex.core.concepts.concept_blueprint import ConceptBlueprint++CONCEPTS_WITH_REFINES = (+ "concepts_with_refines",+ """domain = "refining_concepts"+definition = "Domain with concepts that refine others"++[concept]+Concept1 = "A concept1"+Concept2 = "A concept2"++[concept.Concept3]+definition = "A concept3"+refines = "Concept2"+""",+ PipelexBundleBlueprint(+ domain="refining_concepts",+ definition="Domain with concepts that refine others",+ concept={+ "Concept1": "A concept1",+ "Concept2": "A concept2",+ "Concept3": ConceptBlueprint(definition="A concept3", refines="Concept2"),+ },+ ),+)++# Export all refining concept test cases+REFINING_CONCEPT_TEST_CASES = [+ CONCEPTS_WITH_REFINES,+]diff --git a/tests/unit/pipelex/core/test_data/concepts/simple_concepts.py b/tests/unit/pipelex/core/test_data/concepts/simple_concepts.pynew file mode 100644index 000000000..ff4f54df1--- /dev/null+++ b/tests/unit/pipelex/core/test_data/concepts/simple_concepts.py@@ -0,0 +1,36 @@+"""Simple concept test cases."""++from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint++SIMPLE_CONCEPTS = (+ "simple_concept",+ """domain = "simple_concept"+definition = "A simple concept"++[concept]+Concept1 = "A concept"+Concept2 = "A concept"+""",+ PipelexBundleBlueprint(+ domain="simple_concept",+ definition="A simple concept",+ concept={+ "Concept1": "A concept",+ "Concept2": "A concept",+ },+ ),+)++EMPTY_CONCEPTS = (+ "empty_concepts",+ """domain = "empty_concepts"+definition = "Domain with empty concept section"+""",+ PipelexBundleBlueprint(domain="empty_concepts", definition="Domain with empty concept section", concept={}),+)++# Export all simple concept test cases+SIMPLE_CONCEPT_TEST_CASES = [+ SIMPLE_CONCEPTS,+ EMPTY_CONCEPTS,+]diff --git a/tests/unit/pipelex/core/test_data/concepts/structured_concepts.py b/tests/unit/pipelex/core/test_data/concepts/structured_concepts.pynew file mode 100644index 000000000..61c1f6fc4--- /dev/null+++ b/tests/unit/pipelex/core/test_data/concepts/structured_concepts.py@@ -0,0 +1,80 @@+"""Structured concept test cases."""++from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint+from pipelex.core.concepts.concept_blueprint import ConceptBlueprint, ConceptStructureBlueprint, ConceptStructureBlueprintFieldType++CONCEPTS_WITH_STRUCTURES = (+ "concepts_with_structures",+ """domain = "structured_concepts"+definition = "Domain with structured concepts"++[concept]+SimpleData = "Simple data concept"++[concept.PersonInfo]+definition = "Information about a person"++[concept.PersonInfo.structure]+name = "The name of the person"+age = { type = "number", definition = "The age of the person", required = true }+birthdate = { type = "date", definition = "The birthdate of the person", required = true }+""",+ PipelexBundleBlueprint(+ domain="structured_concepts",+ definition="Domain with structured concepts",+ concept={+ "SimpleData": "Simple data concept",+ "PersonInfo": ConceptBlueprint(+ definition="Information about a person",+ structure={+ "name": "The name of the person",+ "age": ConceptStructureBlueprint(+ type=ConceptStructureBlueprintFieldType.NUMBER, definition="The age of the person", required=True+ ),+ "birthdate": ConceptStructureBlueprint(+ type=ConceptStructureBlueprintFieldType.DATE, definition="The birthdate of the person", required=True+ ),+ },+ ),+ },+ ),+)++CONCEPTS_WITH_NAMED_STRUCTURES = (+ "concepts_with_named_structures",+ """domain = "named_structures"+definition = "Domain with concepts using named structure references"++[concept]+BasicInfo = "Basic information concept"++[concept.ProductInfo]+definition = "Information about a product"+structure = "ProductData"++[concept.OrderInfo]+definition = "Information about an order"+structure = "OrderData"+""",+ PipelexBundleBlueprint(+ domain="named_structures",+ definition="Domain with concepts using named structure references",+ concept={+ "BasicInfo": "Basic information concept",+ "ProductInfo": ConceptBlueprint(+ definition="Information about a product",+ structure="ProductData",+ ),+ "OrderInfo": ConceptBlueprint(+ definition="Information about an order",+ structure="OrderData",+ ),+ },+ ),+)++# Export all structured concept test cases+STRUCTURED_CONCEPT_TEST_CASES = [+ CONCEPTS_WITH_STRUCTURES,+ CONCEPTS_WITH_NAMED_STRUCTURES,+]diff --git a/tests/unit/pipelex/core/test_data/domain/__init__.py b/tests/unit/pipelex/core/test_data/domain/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/unit/pipelex/core/test_data/domain/simple_domains.py b/tests/unit/pipelex/core/test_data/domain/simple_domains.pynew file mode 100644index 000000000..3dc8c2fd5--- /dev/null+++ b/tests/unit/pipelex/core/test_data/domain/simple_domains.py@@ -0,0 +1,37 @@+"""Simple domain test cases."""++from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint++SIMPLE_DOMAIN = (+ "simple_domain",+ """domain = "simple_test"+definition = "A simple test domain"+""",+ PipelexBundleBlueprint(+ domain="simple_test",+ definition="A simple test domain",+ ),+)++DOMAIN_WITH_SYSTEM_PROMPTS = (+ "domain_with_system_prompts",+ """domain = "system_prompt_test"+definition = "A domain with system prompts"+system_prompt = "You are an expert assistant"+system_prompt_to_structure = "Generate structured output"+prompt_template_to_structure = "Structure the following data: ..."+""",+ PipelexBundleBlueprint(+ domain="system_prompt_test",+ definition="A domain with system prompts",+ system_prompt="You are an expert assistant",+ system_prompt_to_structure="Generate structured output",+ prompt_template_to_structure="Structure the following data: ...",+ ),+)++# Export all domain test cases+DOMAIN_TEST_CASES = [+ SIMPLE_DOMAIN,+ DOMAIN_WITH_SYSTEM_PROMPTS,+]diff --git a/tests/unit/pipelex/core/test_data/errors/__init__.py b/tests/unit/pipelex/core/test_data/errors/__init__.pynew file mode 100644index 000000000..011b80c50--- /dev/null+++ b/tests/unit/pipelex/core/test_data/errors/__init__.py@@ -0,0 +1 @@+"""Error test cases for PipelexInterpreter tests."""diff --git a/tests/unit/pipelex/core/test_data/errors/invalid_toml.py b/tests/unit/pipelex/core/test_data/errors/invalid_toml.pynew file mode 100644index 000000000..26fc011a8--- /dev/null+++ b/tests/unit/pipelex/core/test_data/errors/invalid_toml.py@@ -0,0 +1,30 @@+"""Invalid TOML test cases."""++from typing import List, Tuple++INVALID_TOML_SYNTAX = (+ "invalid_toml_syntax",+ """domain = "invalid_syntax"+definition = "Domain with invalid TOML syntax"++[concept]+InvalidConcept = "This is missing a closing quote""",+ None, # Should raise an exception+)++MISSING_DOMAIN = (+ "missing_domain",+ """# Missing domain field+definition = "Domain without domain field"++[concept]+TestConcept = "A test concept"+""",+ None, # Should raise an exception+)++# Export all error test cases+ERROR_TEST_CASES: List[Tuple[str, str]] = [+ (INVALID_TOML_SYNTAX[0], INVALID_TOML_SYNTAX[1]),+ (MISSING_DOMAIN[0], MISSING_DOMAIN[1]),+]diff --git a/tests/unit/pipelex/core/test_data/pipes/__init__.py b/tests/unit/pipelex/core/test_data/pipes/__init__.pynew file mode 100644index 000000000..44aeec3e0--- /dev/null+++ b/tests/unit/pipelex/core/test_data/pipes/__init__.py@@ -0,0 +1 @@+"""Pipe-related test data for PipelexInterpreter tests."""diff --git a/tests/unit/pipelex/core/test_data/pipes/controllers/__init__.py b/tests/unit/pipelex/core/test_data/pipes/controllers/__init__.pynew file mode 100644index 000000000..b9b684798--- /dev/null+++ b/tests/unit/pipelex/core/test_data/pipes/controllers/__init__.py@@ -0,0 +1 @@+"""Pipe controller test data for PipelexInterpreter tests."""diff --git a/tests/unit/pipelex/core/test_data/pipes/controllers/batch/__init__.py b/tests/unit/pipelex/core/test_data/pipes/controllers/batch/__init__.pynew file mode 100644index 000000000..22a23e3cb--- /dev/null+++ b/tests/unit/pipelex/core/test_data/pipes/controllers/batch/__init__.py@@ -0,0 +1 @@+"""PipeBatch test data for PipelexInterpreter tests."""diff --git a/tests/unit/pipelex/core/test_data/pipes/controllers/batch/pipe_batch.py b/tests/unit/pipelex/core/test_data/pipes/controllers/batch/pipe_batch.pynew file mode 100644index 000000000..b493cb218--- /dev/null+++ b/tests/unit/pipelex/core/test_data/pipes/controllers/batch/pipe_batch.py@@ -0,0 +1,38 @@+"""PipeBatch test cases."""++from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint+from pipelex.pipe_controllers.pipe_batch_factory import PipeBatchBlueprint++PIPE_BATCH = (+ "pipe_batch",+ """domain = "test_pipes"+definition = "Domain with batch pipe"++[pipe.batch_process]+type = "PipeBatch"+definition = "Process items in batch"+output = "ProcessedData"+branch_pipe_code = "process_item"+input_list_name = "items"+input_item_name = "current_item"+""",+ PipelexBundleBlueprint(+ domain="test_pipes",+ definition="Domain with batch pipe",+ pipe={+ "batch_process": PipeBatchBlueprint(+ type="PipeBatch",+ definition="Process items in batch",+ output="ProcessedData",+ branch_pipe_code="process_item",+ input_list_name="items",+ input_item_name="current_item",+ ),+ },+ ),+)++# Export all PipeBatch test cases+PIPE_BATCH_TEST_CASES = [+ PIPE_BATCH,+]diff --git a/tests/unit/pipelex/core/test_data/pipes/controllers/condition/__init__.py b/tests/unit/pipelex/core/test_data/pipes/controllers/condition/__init__.pynew file mode 100644index 000000000..55ed0effd--- /dev/null+++ b/tests/unit/pipelex/core/test_data/pipes/controllers/condition/__init__.py@@ -0,0 +1 @@+"""PipeCondition test data for PipelexInterpreter tests."""diff --git a/tests/unit/pipelex/core/test_data/pipes/controllers/condition/pipe_condition.py b/tests/unit/pipelex/core/test_data/pipes/controllers/condition/pipe_condition.pynew file mode 100644index 000000000..973bb8f9e--- /dev/null+++ b/tests/unit/pipelex/core/test_data/pipes/controllers/condition/pipe_condition.py@@ -0,0 +1,39 @@+"""PipeCondition test cases."""++from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint+from pipelex.pipe_controllers.pipe_condition_factory import PipeConditionBlueprint++PIPE_CONDITION = (+ "pipe_condition",+ """domain = "test_pipes"+definition = "Domain with conditional pipe"++[pipe.conditional_process]+type = "PipeCondition"+definition = "Process based on condition"+output = "ProcessedData"+pipe_map = { small = "process_small", large = "process_large" }+expression = "input_data.category"+""",+ PipelexBundleBlueprint(+ domain="test_pipes",+ definition="Domain with conditional pipe",+ pipe={+ "conditional_process": PipeConditionBlueprint(+ type="PipeCondition",+ definition="Process based on condition",+ output="ProcessedData",+ expression="input_data.category",+ pipe_map={+ "small": "process_small",+ "large": "process_large",+ },+ ),+ },+ ),+)++# Export all PipeCondition test cases+PIPE_CONDITION_TEST_CASES = [+ PIPE_CONDITION,+]diff --git a/tests/unit/pipelex/core/test_data/pipes/controllers/parallel/__init__.py b/tests/unit/pipelex/core/test_data/pipes/controllers/parallel/__init__.pynew file mode 100644index 000000000..e15534b6c--- /dev/null+++ b/tests/unit/pipelex/core/test_data/pipes/controllers/parallel/__init__.py@@ -0,0 +1 @@+"""PipeParallel test data for PipelexInterpreter tests."""diff --git a/tests/unit/pipelex/core/test_data/pipes/controllers/parallel/pipe_parallel.py b/tests/unit/pipelex/core/test_data/pipes/controllers/parallel/pipe_parallel.pynew file mode 100644index 000000000..bbce89315--- /dev/null+++ b/tests/unit/pipelex/core/test_data/pipes/controllers/parallel/pipe_parallel.py@@ -0,0 +1,41 @@+"""PipeParallel test cases."""++from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint+from pipelex.pipe_controllers.pipe_parallel_factory import PipeParallelBlueprint+from pipelex.pipe_controllers.sub_pipe_factory import SubPipeBlueprint++PIPE_PARALLEL = (+ "pipe_parallel",+ """domain = "test_pipes"+definition = "Domain with parallel pipe"++[pipe.parallel_process]+type = "PipeParallel"+definition = "Process data in parallel"+output = "ProcessedData"+parallels = [+ { pipe = "process_a", result = "result_a" },+ { pipe = "process_b", result = "result_b" },+]+""",+ PipelexBundleBlueprint(+ domain="test_pipes",+ definition="Domain with parallel pipe",+ pipe={+ "parallel_process": PipeParallelBlueprint(+ type="PipeParallel",+ definition="Process data in parallel",+ output="ProcessedData",+ parallels=[+ SubPipeBlueprint(pipe="process_a", result="result_a"),+ SubPipeBlueprint(pipe="process_b", result="result_b"),+ ],+ ),+ },+ ),+)++# Export all PipeParallel test cases+PIPE_PARALLEL_TEST_CASES = [+ PIPE_PARALLEL,+]diff --git a/tests/unit/pipelex/core/test_data/pipes/controllers/sequence/__init__.py b/tests/unit/pipelex/core/test_data/pipes/controllers/sequence/__init__.pynew file mode 100644index 000000000..c0ec60837--- /dev/null+++ b/tests/unit/pipelex/core/test_data/pipes/controllers/sequence/__init__.py@@ -0,0 +1 @@+"""PipeSequence test data for PipelexInterpreter tests."""diff --git a/tests/unit/pipelex/core/test_data/pipes/controllers/sequence/pipe_sequence.py b/tests/unit/pipelex/core/test_data/pipes/controllers/sequence/pipe_sequence.pynew file mode 100644index 000000000..4bb4d95f8--- /dev/null+++ b/tests/unit/pipelex/core/test_data/pipes/controllers/sequence/pipe_sequence.py@@ -0,0 +1,41 @@+"""PipeSequence test cases."""++from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint+from pipelex.pipe_controllers.pipe_sequence_factory import PipeSequenceBlueprint+from pipelex.pipe_controllers.sub_pipe_factory import SubPipeBlueprint++PIPE_SEQUENCE = (+ "pipe_sequence",+ """domain = "test_pipes"+definition = "Domain with sequence pipe"++[pipe.process_sequence]+type = "PipeSequence"+definition = "Process data in sequence"+output = "ProcessedData"+steps = [+ { pipe = "step1", result = "intermediate1" },+ { pipe = "step2", result = "final_result" },+]+""",+ PipelexBundleBlueprint(+ domain="test_pipes",+ definition="Domain with sequence pipe",+ pipe={+ "process_sequence": PipeSequenceBlueprint(+ type="PipeSequence",+ definition="Process data in sequence",+ output="ProcessedData",+ steps=[+ SubPipeBlueprint(pipe="step1", result="intermediate1"),+ SubPipeBlueprint(pipe="step2", result="final_result"),+ ],+ ),+ },+ ),+)++# Export all PipeSequence test cases+PIPE_SEQUENCE_TEST_CASES = [+ PIPE_SEQUENCE,+]diff --git a/tests/unit/pipelex/core/test_data/pipes/operators/__init__.py b/tests/unit/pipelex/core/test_data/pipes/operators/__init__.pynew file mode 100644index 000000000..d9191620f--- /dev/null+++ b/tests/unit/pipelex/core/test_data/pipes/operators/__init__.py@@ -0,0 +1 @@+"""Pipe operator test data for PipelexInterpreter tests."""diff --git a/tests/unit/pipelex/core/test_data/pipes/operators/func/__init__.py b/tests/unit/pipelex/core/test_data/pipes/operators/func/__init__.pynew file mode 100644index 000000000..dac31fc83--- /dev/null+++ b/tests/unit/pipelex/core/test_data/pipes/operators/func/__init__.py@@ -0,0 +1 @@+"""PipeFunc test data for PipelexInterpreter tests."""diff --git a/tests/unit/pipelex/core/test_data/pipes/operators/func/pipe_func.py b/tests/unit/pipelex/core/test_data/pipes/operators/func/pipe_func.pynew file mode 100644index 000000000..cd395b0eb--- /dev/null+++ b/tests/unit/pipelex/core/test_data/pipes/operators/func/pipe_func.py@@ -0,0 +1,34 @@+"""PipeFunc test cases."""++from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint+from pipelex.pipe_operators.pipe_func_factory import PipeFuncBlueprint++PIPE_FUNC = (+ "pipe_func",+ """domain = "test_pipes"+definition = "Domain with function pipe"++[pipe.process_data]+type = "PipeFunc"+definition = "Process data with function"+output = "ProcessedData"+function_name = "process_data_function"+""",+ PipelexBundleBlueprint(+ domain="test_pipes",+ definition="Domain with function pipe",+ pipe={+ "process_data": PipeFuncBlueprint(+ type="PipeFunc",+ definition="Process data with function",+ output="ProcessedData",+ function_name="process_data_function",+ ),+ },+ ),+)++# Export all PipeFunc test cases+PIPE_FUNC_TEST_CASES = [+ PIPE_FUNC,+]diff --git a/tests/unit/pipelex/core/test_data/pipes/operators/img_gen/__init__.py b/tests/unit/pipelex/core/test_data/pipes/operators/img_gen/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/unit/pipelex/core/test_data/pipes/operators/img_gen/pipe_img_gen.py b/tests/unit/pipelex/core/test_data/pipes/operators/img_gen/pipe_img_gen.pynew file mode 100644index 000000000..f9a1e2d6b--- /dev/null+++ b/tests/unit/pipelex/core/test_data/pipes/operators/img_gen/pipe_img_gen.py@@ -0,0 +1,34 @@+"""PipeImgGen test cases."""++from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint+from pipelex.pipe_operators.pipe_img_gen_factory import PipeImgGenBlueprint++PIPE_IMG_GEN = (+ "pipe_img_gen",+ """domain = "test_pipes"+definition = "Domain with image generation pipe"++[pipe.generate_image]+type = "PipeImgGen"+definition = "Generate an image from a prompt"+output = "Image"+img_gen_prompt = "A beautiful landscape"+""",+ PipelexBundleBlueprint(+ domain="test_pipes",+ definition="Domain with image generation pipe",+ pipe={+ "generate_image": PipeImgGenBlueprint(+ type="PipeImgGen",+ definition="Generate an image from a prompt",+ output="Image",+ img_gen_prompt="A beautiful landscape",+ ),+ },+ ),+)++# Export all PipeImgGen test cases+PIPE_IMG_GEN_TEST_CASES = [+ PIPE_IMG_GEN,+]diff --git a/tests/unit/pipelex/core/test_data/pipes/operators/jinja2/__init__.py b/tests/unit/pipelex/core/test_data/pipes/operators/jinja2/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/unit/pipelex/core/test_data/pipes/operators/jinja2/pipe_jinja2.py b/tests/unit/pipelex/core/test_data/pipes/operators/jinja2/pipe_jinja2.pynew file mode 100644index 000000000..af5642070--- /dev/null+++ b/tests/unit/pipelex/core/test_data/pipes/operators/jinja2/pipe_jinja2.py@@ -0,0 +1,34 @@+"""PipeJinja2 test cases."""++from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint+from pipelex.pipe_operators.pipe_jinja2_factory import PipeJinja2Blueprint++PIPE_JINJA2 = (+ "pipe_jinja2",+ """domain = "test_pipes"+definition = "Domain with template processing pipe"++[pipe.process_template]+type = "PipeJinja2"+definition = "Process a Jinja2 template"+output = "Text"+jinja2 = "Hello {{ name }}!"+""",+ PipelexBundleBlueprint(+ domain="test_pipes",+ definition="Domain with template processing pipe",+ pipe={+ "process_template": PipeJinja2Blueprint(+ type="PipeJinja2",+ definition="Process a Jinja2 template",+ output="Text",+ jinja2="Hello {{ name }}!",+ ),+ },+ ),+)++# Export all PipeJinja2 test cases+PIPE_JINJA2_TEST_CASES = [+ PIPE_JINJA2,+]diff --git a/tests/unit/pipelex/core/test_data/pipes/operators/llm/__init__.py b/tests/unit/pipelex/core/test_data/pipes/operators/llm/__init__.pynew file mode 100644index 000000000..a91704a7e--- /dev/null+++ b/tests/unit/pipelex/core/test_data/pipes/operators/llm/__init__.py@@ -0,0 +1 @@+"""PipeLLM test data for PipelexInterpreter tests."""diff --git a/tests/unit/pipelex/core/test_data/pipes/operators/llm/pipe_llm.py b/tests/unit/pipelex/core/test_data/pipes/operators/llm/pipe_llm.pynew file mode 100644index 000000000..9daf44c54--- /dev/null+++ b/tests/unit/pipelex/core/test_data/pipes/operators/llm/pipe_llm.py@@ -0,0 +1,34 @@+"""PipeLLM test cases."""++from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint+from pipelex.pipe_operators.pipe_llm_factory import PipeLLMBlueprint++PIPE_LLM = (+ "pipe_llm",+ """domain = "test_pipes"+definition = "Domain with pipe definitions"++[pipe.generate_text]+type = "PipeLLM"+definition = "Generate text using LLM"+output = "Text"+prompt_template = "Generate a story about a programmer"+""",+ PipelexBundleBlueprint(+ domain="test_pipes",+ definition="Domain with pipe definitions",+ pipe={+ "generate_text": PipeLLMBlueprint(+ type="PipeLLM",+ definition="Generate text using LLM",+ output="Text",+ prompt_template="Generate a story about a programmer",+ ),+ },+ ),+)++# Export all PipeLLM test cases+PIPE_LLM_TEST_CASES = [+ PIPE_LLM,+]diff --git a/tests/unit/pipelex/core/test_data/pipes/operators/ocr/__init__.py b/tests/unit/pipelex/core/test_data/pipes/operators/ocr/__init__.pynew file mode 100644index 000000000..a5ef8b7a3--- /dev/null+++ b/tests/unit/pipelex/core/test_data/pipes/operators/ocr/__init__.py@@ -0,0 +1 @@+"""PipeOcr test data for PipelexInterpreter tests."""diff --git a/tests/unit/pipelex/core/test_data/pipes/operators/ocr/pipe_ocr.py b/tests/unit/pipelex/core/test_data/pipes/operators/ocr/pipe_ocr.pynew file mode 100644index 000000000..c654f42f3--- /dev/null+++ b/tests/unit/pipelex/core/test_data/pipes/operators/ocr/pipe_ocr.py@@ -0,0 +1,32 @@+"""PipeOcr test cases."""++from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint+from pipelex.pipe_operators.pipe_ocr_factory import PipeOcrBlueprint++PIPE_OCR = (+ "pipe_ocr",+ """domain = "test_pipes"+definition = "Domain with OCR pipe"++[pipe.extract_text]+type = "PipeOcr"+definition = "Extract text from document"+output = "Page"+""",+ PipelexBundleBlueprint(+ domain="test_pipes",+ definition="Domain with OCR pipe",+ pipe={+ "extract_text": PipeOcrBlueprint(+ type="PipeOcr",+ definition="Extract text from document",+ output="Page",+ ),+ },+ ),+)++# Export all PipeOcr test cases+PIPE_OCR_TEST_CASES = [+ PIPE_OCR,+]diff --git a/tests/unit/pipelex/core/test_interpreter.py b/tests/unit/pipelex/core/test_interpreter.pynew file mode 100644index 000000000..6a6861d1e--- /dev/null+++ b/tests/unit/pipelex/core/test_interpreter.py@@ -0,0 +1,42 @@+from pathlib import Path++import pytest++from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint+from pipelex.core.interpreter import PipelexInterpreter+from tests.unit.pipelex.core.test_data import InterpreterTestCases+++class TestPipelexInterpreter:+ """Test the PipelexInterpreter class with various TOML configurations."""++ def test_init_with_both_file_path_and_content(self, tmp_path: Path):+ """Test initialization with both file_path and file_content."""+ test_file = tmp_path / "test.toml"+ test_file.write_text("domain = 'test'\n[concept]\n")+ content = "domain = 'other'\n[concept]\n"++ converter = PipelexInterpreter(file_path=test_file, file_content=content)+ assert converter.file_path == test_file+ assert converter.file_content == content++ @pytest.mark.parametrize("test_name,toml_content,expected_blueprint", InterpreterTestCases.VALID_TEST_CASES)+ def test_make_pipelex_bundle_blueprint(self, test_name: str, toml_content: str, expected_blueprint: PipelexBundleBlueprint):+ """Test making blueprint from various valid TOML content."""+ converter = PipelexInterpreter(file_content=toml_content)+ from pipelex import pretty_print++ blueprint = converter.make_pipelex_bundle_blueprint()+ pretty_print(blueprint, title=f"Blueprint {test_name}")+ pretty_print(expected_blueprint, title=f"Expected blueprint {test_name}")+ assert blueprint == expected_blueprint++ @pytest.mark.parametrize("test_name,toml_content,expected_blueprint", InterpreterTestCases.VALID_TEST_CASES)+ def test_make_toml_content(self, test_name: str, toml_content: str, expected_blueprint: PipelexBundleBlueprint):+ """Test making blueprint from various valid TOML content."""+ get_toml_content = PipelexInterpreter.make_toml_content(blueprint=expected_blueprint)+ from pipelex import pretty_print++ pretty_print(get_toml_content, title=f"TOML content {test_name}")+ pretty_print(toml_content, title=f"Expected TOML content {test_name}")+ assert get_toml_content == toml_contentdiff --git a/tests/unit/pipelex/create/test_structured_output_generator.py b/tests/unit/pipelex/create/test_structured_output_generator.pyindex e7838c1d5..635f88a4d 100644--- a/tests/unit/pipelex/create/test_structured_output_generator.py+++ b/tests/unit/pipelex/create/test_structured_output_generator.py@@ -31,7 +31,7 @@ active = { type = "boolean", definition = "Active status", default = true }# Check that basic structure is correctassert "from typing import Optional" in result- assert "from pipelex.core.stuff_content import StructuredContent" in result+ assert "from pipelex.core.stuffs.stuff_content import StructuredContent" in resultassert "from pydantic import Field" in resultassert "class TestModel(StructuredContent):" in resultassert '"""A test model"""' in resultdiff --git a/tests/unit/pipelex/tools/misc/test_toml_utils.py b/tests/unit/pipelex/tools/misc/test_toml_utils.pyindex b9edca0c5..7edcdcda1 100644--- a/tests/unit/pipelex/tools/misc/test_toml_utils.py+++ b/tests/unit/pipelex/tools/misc/test_toml_utils.py@@ -13,8 +13,7 @@ from pipelex.tools.misc.toml_utils import (class TestTomlUtils:def test_load_toml_from_path_valid_file(self, tmp_path: Path) -> None:"""Test loading a valid TOML file without issues."""- toml_content = """-domain = "test_domain"+ toml_content = """domain = "test_domain"definition = "Test definition"[concept]@@ -22,10 +21,11 @@ TestConcept = "A test concept"[pipe][pipe.test_pipe]-PipeLLM = "Test pipe definition"+type = "PipeLLM"+definition = "Test pipe definition"prompt_template = '''This is a test prompt-'''+'''"""toml_file = tmp_path / "valid.toml"toml_file.write_text(toml_content)@@ -194,7 +194,8 @@ CategoryInput = "Input with a category field"[pipe][pipe.basic_condition_by_category_2]-PipeCondition = "Route based on category field using expression"+type = "PipeCondition"+definition = "Route based on category field using expression"inputs = { input_data = "CategoryInput" }output = "native.Text"expression = "input_data.category"@@ -205,7 +206,8 @@ medium = "process_medium_2"large = "process_large_2"[pipe.process_large_2]-PipeLLM = "Generate random text for large items"+type = "PipeLLM"+definition = "Generate random text for large items"output = "native.Text"prompt_template = """Output this only: "large"@@ -218,7 +220,7 @@ Output this only: "large"error_msg = str(exc_info.value)assert "Trailing whitespace after triple quotes" in error_msg- assert "Line 24" in error_msg # The line with """ followed by space+ assert "Line 26" in error_msg # The line with """ followed by spacedef test_validate_toml_file_actual_problematic_file(self) -> None:"""Test validation on the actual problematic file from the codebase."""diff --git a/tests/unit/pipelex/tools/typing/test_type_inspector.py b/tests/unit/pipelex/tools/typing/test_type_inspector.pyindex 399c195d9..5ea82b6c8 100644--- a/tests/unit/pipelex/tools/typing/test_type_inspector.py+++ b/tests/unit/pipelex/tools/typing/test_type_inspector.py@@ -4,7 +4,7 @@ from typing import List, Literal, Optionalfrom pydantic import BaseModel, Field, field_validatorfrom pytest import FixtureRequest-from pipelex.core.stuff_content import ListContent, StructuredContent, TextContent+from pipelex.core.stuffs.stuff_content import ListContent, StructuredContent, TextContentfrom pipelex.tools.typing.type_inspector import get_type_structurefrom pipelex.types import StrEnumdiff --git a/uv.lock b/uv.lockindex d7b5a32d9..6f6a5a622 100644--- a/uv.lock+++ b/uv.lock@@ -1748,7 +1748,7 @@ wheels = [[[package]]name = "pipelex"-version = "0.6.10"+version = "0.7.0"source = { editable = "." }dependencies = [{ name = "aiofiles" },