Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

Refactor/epic library blueprint 5 (#224)

Thomas Hebrard0318270ed6c6
diff --git a/.cursor/rules/pipe-llm.mdc b/.cursor/rules/pipe-llm.mdc
index 0721d4968..3a740fbf6 100644
--- a/.cursor/rules/pipe-llm.mdc
+++ b/.cursor/rules/pipe-llm.mdc
@@ -39,7 +39,7 @@ Extract person information from this text:
Place your Pydantic models in `pipelex_libraries/pipelines/your_models.py`:
```python
-from pipelex.core.stuff_content import StructuredContent
+from pipelex.core.stuffs.stuff_content import StructuredContent
class PersonInfo(StructuredContent): # The output models always have to be subclass of StructuredContent
name: str
diff --git a/.cursor/rules/pipelex.mdc b/.cursor/rules/pipelex.mdc
index ae078a3f4..c98f83fca 100644
--- a/.cursor/rules/pipelex.mdc
+++ b/.cursor/rules/pipelex.mdc
@@ -99,7 +99,7 @@ from datetime import datetime
from typing import List, Optional
from pydantic import Field
-from pipelex.core.stuff_content import StructuredContent
+from pipelex.core.stuffs.stuff_content import StructuredContent
# IMPORTANT: THE CLASS MUST BE A SUBCLASS OF StructuredContent
class YourModel(StructuredContent): # Always be a subclass of StructuredContent
diff --git a/.cursor/rules/pytest.mdc b/.cursor/rules/pytest.mdc
index fd5a199bd..09994dd3b 100644
--- a/.cursor/rules/pytest.mdc
+++ b/.cursor/rules/pytest.mdc
@@ -69,16 +69,16 @@ Sometimes it can be convenient to access the test's name in its body, for instan
import pytest
from pytest import FixtureRequest
from pipelex import log, pretty_print
-from pipelex.core.stuff_factory import StuffBlueprint, StuffFactory
-from pipelex.core.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.stuffs.stuff_factory import StuffBlueprint, StuffFactory
+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
from pipelex.hub import get_report_delegate
from pipelex_libraries.pipelines.base_library.retrieve import RetrievedExcerpt
from pipelex.config_pipelex import get_config
from pipelex.core.pipe import PipeAbstract, update_job_metadata_for_pipe
-from pipelex.core.pipe_output import PipeOutput, PipeOutputType
-from pipelex.core.pipe_run_params import PipeRunParams
-from pipelex.core.pipe_run_params import PipeRunParams
+from pipelex.core.pipes.pipe_output import PipeOutput, PipeOutputType
+from pipelex.core.pipes.pipe_run_params import PipeRunParams
+from pipelex.core.pipes.pipe_run_params import PipeRunParams
from pipelex.pipe_works.pipe_router_protocol import PipeRouterProtocol
```
diff --git a/.cursor/rules/structures.mdc b/.cursor/rules/structures.mdc
index c01303cbe..534a97deb 100644
--- a/.cursor/rules/structures.mdc
+++ b/.cursor/rules/structures.mdc
@@ -23,7 +23,7 @@ from datetime import datetime
from typing import List, Optional
from pydantic import BaseModel, Field
-from pipelex.core.stuff_content import StructuredContent
+from pipelex.core.stuffs.stuff_content import StructuredContent
class YourModel(StructuredContent): # Always be a subclass of StructuredContent
# Required fields
diff --git a/.pipelex/create/build_pipelines.md b/.pipelex/create/build_pipelines.md
index 1dc666313..717c735f7 100644
--- a/.pipelex/create/build_pipelines.md
+++ b/.pipelex/create/build_pipelines.md
@@ -1,6 +1,6 @@
# Building Pipelex Pipelines from Drafts
-This guide explains how to translate a markdown pipeline draft into a formal `PipelineBlueprint` structure that can be validated and executed by Pipelex.
+This guide explains how to translate a markdown pipeline draft into a formal `PipelexBundleBlueprint` structure that can be validated and executed by Pipelex.
## Overview
diff --git a/.pipelex/create/design_pipelines.md b/.pipelex/create/design_pipelines.md
index 2a3fcfb80..9d560dafd 100644
--- a/.pipelex/create/design_pipelines.md
+++ b/.pipelex/create/design_pipelines.md
@@ -6,7 +6,7 @@ This section explains how to design pipelines for Pipelex.
- Plan the pipeline in natural language (goal, inputs, steps, outputs).
-- Translate the plan into a `PipelineBlueprint` object graph: domain, concepts, pipes.
+- Translate the plan into a `PipelexBundleBlueprint` object graph: domain, concepts, pipes.
- Define controllers first (`PipeSequence`, `PipeParallel`, `PipeCondition`), then operators (`PipeLLM`, `PipeOcr`, etc.).
@@ -16,9 +16,9 @@ This section explains how to design pipelines for Pipelex.
- Choose appropriate native or structured output concepts.
-### Root blueprint object: PipelineBlueprint
+### Root blueprint object: PipelexBundleBlueprint
-`PipelineBlueprint` fields:
+`PipelexBundleBlueprint` fields:
- `domain: str` — required. Single word or snake_case code for the domain.
diff --git a/README.md b/README.md
index 280af8968..ed965193a 100644
--- a/README.md
+++ b/README.md
@@ -22,7 +22,7 @@ Stop reinventing AI workflows from scratch. With Pipelex, your proven methods be
alt="PyPI – latest release">
<br/>
<br/>
- <a href="https://go.pipelex.com/discord"><img src="https://img.shields.io/badge/Discord-5865F2?logo=discord&logoColor=white" alt="Discord"></a>
+ <!-- <a href="https://go.pipelex.com/discord"><img src="https://img.shields.io/badge/Discord-5865F2?logo=discord&logoColor=white" alt="Discord"></a> -->
<a href="https://www.youtube.com/@PipelexAI"><img src="https://img.shields.io/badge/YouTube-FF0000?logo=youtube&logoColor=white" alt="YouTube"></a>
<a href="https://pipelex.com"><img src="https://img.shields.io/badge/Homepage-03bb95?logo=google-chrome&logoColor=white&style=flat" alt="Website"></a>
<a href="https://github.com/Pipelex/pipelex-cookbook"><img src="https://img.shields.io/badge/Cookbook-5a0dad?logo=github&logoColor=white&style=flat" alt="Cookbook"></a>
@@ -276,11 +276,11 @@ class optimize_tweet_sequence sub_a;
We welcome contributions! Please see our [Contributing Guidelines](CONTRIBUTING.md) for details on how to get started, including development setup and testing information.
-## 👥 Join the Community
+<!-- ## 👥 Join the Community
-Join our vibrant Discord community to connect with other developers, share your experiences, and get help with your Pipelex projects!
+Join our vibrant Discord community to connect with other developers, share your experiences, and get help with your Pipelex projects! -->
-[![Discord](https://img.shields.io/badge/Discord-5865F2?logo=discord&logoColor=white)](https://go.pipelex.com/discord)
+<!-- [![Discord](https://img.shields.io/badge/Discord-5865F2?logo=discord&logoColor=white)](https://go.pipelex.com/discord) -->
## 💬 Support
diff --git a/docs/pages/build-reliable-ai-workflows-with-pipelex/define_your_concepts.md b/docs/pages/build-reliable-ai-workflows-with-pipelex/define_your_concepts.md
index 0beec74a0..fcc7e34e8 100644
--- a/docs/pages/build-reliable-ai-workflows-with-pipelex/define_your_concepts.md
+++ b/docs/pages/build-reliable-ai-workflows-with-pipelex/define_your_concepts.md
@@ -78,7 +78,7 @@ For each concept that needs structured output, create a corresponding Python cla
from datetime import datetime
from typing import List, Optional
from pydantic import Field
-from pipelex.core.stuff_content import StructuredContent
+from pipelex.core.stuffs.stuff_content import StructuredContent
class Invoice(StructuredContent):
invoice_number: str
@@ -99,7 +99,7 @@ Use Pydantic's validation features to ensure data quality:
```python
from pydantic import field_validator
-from pipelex.core.stuff_content import StructuredContent
+from pipelex.core.stuffs.stuff_content import StructuredContent
class Employee(StructuredContent):
name: str
@@ -129,7 +129,7 @@ Not all data is always available. Use Optional fields with sensible defaults:
```python
from typing import Optional
from datetime import datetime
-from pipelex.core.stuff_content import StructuredContent
+from pipelex.core.stuffs.stuff_content import StructuredContent
class Meeting(StructuredContent):
title: str
@@ -158,7 +158,7 @@ Department = "An organizational unit within a company" # No Python model => tex
```python
# pipelex_libraries/pipelines/hr.py
-from pipelex.core.stuff_content import StructuredContent
+from pipelex.core.stuffs.stuff_content import StructuredContent
from datetime import datetime
from typing import List, Optional
@@ -266,7 +266,7 @@ While TOML refinement is primarily semantic, you can mirror these relationships
```python
# pipelex_libraries/pipelines/content.py
-from pipelex.core.stuff_content import StructuredContent
+from pipelex.core.stuffs.stuff_content import StructuredContent
from datetime import datetime
from typing import Optional, List
diff --git a/docs/pages/build-reliable-ai-workflows-with-pipelex/kick-off-a-knowledge-pipeline-project.md b/docs/pages/build-reliable-ai-workflows-with-pipelex/kick-off-a-knowledge-pipeline-project.md
index afeca0bff..60c06d921 100644
--- a/docs/pages/build-reliable-ai-workflows-with-pipelex/kick-off-a-knowledge-pipeline-project.md
+++ b/docs/pages/build-reliable-ai-workflows-with-pipelex/kick-off-a-knowledge-pipeline-project.md
@@ -69,7 +69,7 @@ A domain consists of:
2. **Python Models** (`.py`)
```python
- from pipelex.core.stuff_content import StructuredContent
+ from pipelex.core.stuffs.stuff_content import StructuredContent
from pydantic import Field
from typing import List
from datetime import date
@@ -110,7 +110,7 @@ A domain consists of:
When using a domain in your code, you refer to concepts with `domain.ConceptName`:
```python
-from pipelex.core.stuff_factory import StuffFactory
+from pipelex.core.stuffs.stuff_factory import StuffFactory
# The concept_code combines domain and concept names
invoice_stuff = StuffFactory.make_stuff(
diff --git a/docs/pages/installation/index.md b/docs/pages/installation/index.md
index 183ed10ef..906ae2301 100644
--- a/docs/pages/installation/index.md
+++ b/docs/pages/installation/index.md
@@ -61,4 +61,5 @@ Learn more about pipelex_libraries in our [Libraries documentation](../build-rel
- `pipelex init config`: This cli command will create a `pipelex.toml` file at the root of the project, with basic configuration. This configuration file gathers all configuration for feature flags, logging, cost reporting, and so on... Learn more in our [Configuration documentation](../configuration/index.md)
-💡 _Any troubles? Have a look at our [Cookbook](https://github.com/Pipelex/pipelex-cookbook)! and come ask for help on our [Discord](https://go.pipelex.com/discord)_
+💡 _Any troubles? Have a look at our [Cookbook](https://github.com/Pipelex/pipelex-cookbook)!
+<!-- and come ask for help on our [Discord](https://go.pipelex.com/discord)_ -->
diff --git a/docs/pages/quick-start/index.md b/docs/pages/quick-start/index.md
index 4da126632..fa3d92815 100644
--- a/docs/pages/quick-start/index.md
+++ b/docs/pages/quick-start/index.md
@@ -104,7 +104,7 @@ Using the [Pydantic BaseModel](https://docs.pydantic.dev/latest/) syntax, define
`pipelex_libraries/pipelines/characters.py`
```python
-from pipelex.core.stuff_content import StructuredContent
+from pipelex.core.stuffs.stuff_content import StructuredContent
# Define the structure of your output here
# This class must inherit from StructuredContent
@@ -160,7 +160,7 @@ We want to extract structured information from the description field. Thus we ha
```python
# pipelex_libraries/pipelines/character_model.py
-from pipelex.core.stuff_content import StructuredContent
+from pipelex.core.stuffs.stuff_content import StructuredContent
# input class
class Character(StructuredContent):
@@ -208,8 +208,8 @@ Learn more about how we use Jinja in the [PipeLLM documentation](../build-reliab
```python
import asyncio
-from pipelex.core.stuff_factory import StuffFactory
-from pipelex.core.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.stuffs.stuff_factory import StuffFactory
+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
from pipelex.pipelex import Pipelex
from pipelex.pipeline.execute import execute_pipeline
diff --git a/mkdocs.yml b/mkdocs.yml
index 357eb6e0e..c56c4d6c3 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -44,9 +44,9 @@ extra:
- icon: fontawesome/brands/twitter
link: https://x.com/PipelexAI
name: Pipelex on X
- - icon: fontawesome/brands/discord
- link: https://go.pipelex.com/discord
- name: Pipelex on Discord
+ # - icon: fontawesome/brands/discord
+ # link: https://go.pipelex.com/discord
+ # name: Pipelex on Discord
generator: false
plugins:
diff --git a/pipelex/cli/commands/init_cmd.py b/pipelex/cli/commands/init_cmd.py
index 22a942796..b7441b739 100644
--- a/pipelex/cli/commands/init_cmd.py
+++ b/pipelex/cli/commands/init_cmd.py
@@ -14,16 +14,16 @@ from pipelex.tools.config.manager import config_manager
def do_init_libraries(directory: str = ".", overwrite: bool = False) -> None:
"""Initialize pipelex libraries in a pipelex_libraries folder in the specified directory."""
try:
- target_path = os.path.join(directory, "pipelex_libraries")
+ target_dir = os.path.join(directory, "pipelex_libraries")
os.makedirs(directory, exist_ok=True)
- library_config = LibraryConfig(config_folder_path=target_path)
+ library_config = LibraryConfig(config_dir_path=target_dir)
library_config.export_libraries(overwrite=overwrite)
if overwrite:
- typer.echo(f"✅ Successfully initialized pipelex libraries at '{target_path}' (all files overwritten)")
+ typer.echo(f"✅ Successfully initialized pipelex libraries at '{target_dir}' (all files overwritten)")
else:
- typer.echo(f"✅ Successfully initialized pipelex libraries at '{target_path}' (only created non-existing files)")
+ typer.echo(f"✅ Successfully initialized pipelex libraries at '{target_dir}' (only created non-existing files)")
except Exception as exc:
raise PipelexCLIError(f"Failed to initialize libraries at '{directory}': {exc}") from exc
diff --git a/pipelex/cli/commands/validate_cmd.py b/pipelex/cli/commands/validate_cmd.py
index b56ee1cc9..3458d83b2 100644
--- a/pipelex/cli/commands/validate_cmd.py
+++ b/pipelex/cli/commands/validate_cmd.py
@@ -4,18 +4,12 @@ import asyncio
from typing import Annotated
import typer
-from pydantic import ValidationError
from pipelex import log
from pipelex.cli.commands.common import is_pipelex_libraries_folder
-from pipelex.create.validate_blueprint import validate_blueprint
-from pipelex.exceptions import PipelexCLIError
from pipelex.hub import get_pipeline_tracker
-from pipelex.libraries.pipeline_blueprint import PipelineBlueprint
from pipelex.pipe_works.pipe_dry import dry_run_all_pipes, dry_run_single_pipe
from pipelex.pipelex import Pipelex
-from pipelex.tools.misc.toml_utils import load_toml_from_path
-from pipelex.tools.typing.pydantic_utils import format_pydantic_validation_error
def do_validate_all_libraries_and_dry_run(relative_config_folder_path: str = "./pipelex_libraries") -> None:
@@ -48,32 +42,6 @@ def do_dry_run_pipe(pipe_code: str, relative_config_folder_path: str = "./pipele
raise typer.Exit(1)
-async def do_validate_blueprint_toml_file(blueprint_path: str, is_error_fixing_enabled: bool) -> None:
- """Validate an already generated pipeline blueprint from a TOML file.
-
- Args:
- blueprint_path: Path to the blueprint TOML file to validate
-
- Raises:
- PipelexCLIError: If validation fails
- ValidationError: If the TOML structure doesn't match PipelineBlueprint
- """
- # Load the TOML file
- log.info(f"Loading blueprint from '{blueprint_path}'...")
- blueprint_dict = load_toml_from_path(path=blueprint_path)
-
- # Validate and create PipelineBlueprint model
- try:
- blueprint = PipelineBlueprint.model_validate(blueprint_dict)
- except ValidationError as exc:
- error_msg = format_pydantic_validation_error(exc=exc)
- raise PipelexCLIError(f"Invalid blueprint structure in '{blueprint_path}': {error_msg}") from exc
-
- log.info(f"Blueprint loaded successfully: domain='{blueprint.domain}'")
-
- await validate_blueprint(blueprint=blueprint, is_error_fixing_enabled=is_error_fixing_enabled)
-
-
# Typer group for validation commands
validate_app = typer.Typer(help="Validation and dry-run commands")
@@ -95,34 +63,3 @@ def dry_run_pipe_cmd(
] = "./pipelex_libraries",
) -> None:
do_dry_run_pipe(pipe_code=pipe_code, relative_config_folder_path=relative_config_folder_path)
-
-
-@validate_app.command("blueprint")
-def validate_blueprint_cmd(
- blueprint_path: Annotated[str, typer.Argument(help="Path to the blueprint TOML file to validate")],
- relative_config_folder_path: Annotated[
- str, typer.Option("--config-folder-path", "-c", help="Relative path to the config folder path")
- ] = "./pipelex_libraries",
- is_error_fixing_enabled: Annotated[bool, typer.Option("--fix/--no-fix", "-f/-F", help="Enable or disable error fixing during validation")] = True,
-) -> None:
- """Validate an already generated pipeline blueprint TOML file.
-
- This command loads the TOML file, validates it against the `PipelineBlueprint` model,
- loads the defined pipes into memory, and performs a dry-run validation of those pipes.
- """
- if not is_pipelex_libraries_folder(relative_config_folder_path):
- typer.echo(f"❌ No pipelex libraries folder found at '{relative_config_folder_path}'")
- typer.echo("To create a pipelex libraries folder, run: pipelex init-libraries")
- raise typer.Exit(1)
-
- # Initialize Pipelex (loads libraries and pipes)
- Pipelex.make(relative_config_folder_path=relative_config_folder_path, from_file=False)
-
- # Run validation
- asyncio.run(
- do_validate_blueprint_toml_file(
- blueprint_path=blueprint_path,
- is_error_fixing_enabled=is_error_fixing_enabled,
- )
- )
- log.info("Blueprint validation completed successfully.")
diff --git a/pipelex/client/api_serializer.py b/pipelex/client/api_serializer.py
index 8de04fea0..7390dbcb8 100644
--- a/pipelex/client/api_serializer.py
+++ b/pipelex/client/api_serializer.py
@@ -5,10 +5,10 @@ from pathlib import Path
from typing import Any, Dict, List, Optional, cast
from pipelex.client.protocol import CompactMemory
-from pipelex.core.concept_native import NativeConcept
-from pipelex.core.pipe_output import PipeOutput
-from pipelex.core.stuff_content import TextContent
-from pipelex.core.working_memory import WorkingMemory
+from pipelex.core.concepts.concept_native import NativeConcept
+from pipelex.core.memory.working_memory import WorkingMemory
+from pipelex.core.pipes.pipe_output import PipeOutput
+from pipelex.core.stuffs.stuff_content import TextContent
class ApiSerializer:
diff --git a/pipelex/client/client.py b/pipelex/client/client.py
index f5217315d..7da3480ed 100644
--- a/pipelex/client/client.py
+++ b/pipelex/client/client.py
@@ -6,9 +6,9 @@ from typing_extensions import override
from pipelex.client.pipeline_request_factory import PipelineRequestFactory
from pipelex.client.pipeline_response_factory import PipelineResponseFactory
from pipelex.client.protocol import CompactMemory, PipelexProtocol, PipelineResponse
-from pipelex.core.pipe_run_params import PipeOutputMultiplicity
-from pipelex.core.working_memory import WorkingMemory
-from pipelex.core.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.memory.working_memory import WorkingMemory
+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.pipes.pipe_run_params import PipeOutputMultiplicity
from pipelex.exceptions import ClientAuthenticationError
from pipelex.tools.environment import get_required_env
diff --git a/pipelex/client/pipeline_request_factory.py b/pipelex/client/pipeline_request_factory.py
index 6c198361b..51b732806 100644
--- a/pipelex/client/pipeline_request_factory.py
+++ b/pipelex/client/pipeline_request_factory.py
@@ -2,8 +2,8 @@ from typing import Any, Dict, Optional
from pipelex.client.api_serializer import ApiSerializer
from pipelex.client.protocol import COMPACT_MEMORY_KEY, PipelineRequest
-from pipelex.core.pipe_run_params import PipeOutputMultiplicity
-from pipelex.core.working_memory import WorkingMemory
+from pipelex.core.memory.working_memory import WorkingMemory
+from pipelex.core.pipes.pipe_run_params import PipeOutputMultiplicity
class PipelineRequestFactory:
diff --git a/pipelex/client/pipeline_response_factory.py b/pipelex/client/pipeline_response_factory.py
index 5014fc153..f0e70db84 100644
--- a/pipelex/client/pipeline_response_factory.py
+++ b/pipelex/client/pipeline_response_factory.py
@@ -2,7 +2,7 @@ from typing import Any, Dict, Optional
from pipelex.client.api_serializer import ApiSerializer
from pipelex.client.protocol import PipelineResponse, PipelineState
-from pipelex.core.pipe_output import PipeOutput
+from pipelex.core.pipes.pipe_output import PipeOutput
class PipelineResponseFactory:
diff --git a/pipelex/client/protocol.py b/pipelex/client/protocol.py
index 7dcf84246..cddce3300 100644
--- a/pipelex/client/protocol.py
+++ b/pipelex/client/protocol.py
@@ -4,9 +4,9 @@ from typing import Any, Dict, List, Optional, Protocol
from pydantic import BaseModel
from typing_extensions import runtime_checkable
-from pipelex.core.pipe_run_params import PipeOutputMultiplicity
-from pipelex.core.stuff_content import StuffContent
-from pipelex.core.working_memory import WorkingMemory
+from pipelex.core.memory.working_memory import WorkingMemory
+from pipelex.core.pipes.pipe_run_params import PipeOutputMultiplicity
+from pipelex.core.stuffs.stuff_content import StuffContent
from pipelex.types import StrEnum
StuffContentOrData = Dict[str, Any] | StuffContent | List[Any] | str
diff --git a/pipelex/cogt/content_generation/content_generator.py b/pipelex/cogt/content_generation/content_generator.py
index e5e4e088c..da1353f12 100644
--- a/pipelex/cogt/content_generation/content_generator.py
+++ b/pipelex/cogt/content_generation/content_generator.py
@@ -46,7 +46,7 @@ class ContentGenerator(ContentGeneratorProtocol):
llm_prompt_for_text: LLMPrompt,
) -> str:
log.verbose(f"{self.__class__.__name__} make_llm_text: {llm_prompt_for_text}")
- log.info(f"llm_setting_main: {llm_setting_main}")
+ log.verbose(f"llm_setting_main: {llm_setting_main}")
llm_assignment = LLMAssignment(
job_metadata=job_metadata,
llm_setting=llm_setting_main,
diff --git a/pipelex/cogt/llm/llm_models/llm_model_library.py b/pipelex/cogt/llm/llm_models/llm_model_library.py
index 589668427..0511c190a 100644
--- a/pipelex/cogt/llm/llm_models/llm_model_library.py
+++ b/pipelex/cogt/llm/llm_models/llm_model_library.py
@@ -35,8 +35,8 @@ class LLMModelLibrary(LLMModelProviderAbstract, RootModel[LLMModelLibraryRoot]):
library_config: ClassVar[LibraryConfig]
@classmethod
- def make_empty(cls, config_folder_path: str) -> "LLMModelLibrary":
- cls.library_config = LibraryConfig(config_folder_path=config_folder_path)
+ def make_empty(cls, config_dir_path: str) -> "LLMModelLibrary":
+ cls.library_config = LibraryConfig(config_dir_path=config_dir_path)
return cls()
@override
@@ -69,7 +69,7 @@ class LLMModelLibrary(LLMModelProviderAbstract, RootModel[LLMModelLibraryRoot]):
@classmethod
def load_llm_model_library_dict(cls) -> LLMModelLibraryDict:
- libraries_path = cls.library_config.llm_integrations_path
+ libraries_path = cls.library_config.llm_integrations_dir_path
if not os.path.exists(libraries_path):
raise LLMModelLibraryError(f"LLM model library path `{libraries_path}` not found. Please run `pipelex init-libraries` to create it.")
llm_library: LLMModelLibraryDict = {}
diff --git a/pipelex/core/bundles/pipelex_bundle.py b/pipelex/core/bundles/pipelex_bundle.py
new file mode 100644
index 000000000..d5d56a6a9
--- /dev/null
+++ b/pipelex/core/bundles/pipelex_bundle.py
@@ -0,0 +1,15 @@
+from typing import Dict
+
+from pydantic import BaseModel
+
+from pipelex.core.concepts.concept import Concept
+from pipelex.core.domains.domain import Domain
+from pipelex.core.pipes.pipe_abstract import PipeAbstract
+
+
+class PipelexBundle(BaseModel):
+ """Complete bundle of a pipelex bundle."""
+
+ domain: Domain
+ concepts: Dict[str, Concept]
+ pipes: Dict[str, PipeAbstract]
diff --git a/pipelex/core/bundles/pipelex_bundle_blueprint.py b/pipelex/core/bundles/pipelex_bundle_blueprint.py
new file mode 100644
index 000000000..1b7dfb705
--- /dev/null
+++ b/pipelex/core/bundles/pipelex_bundle_blueprint.py
@@ -0,0 +1,45 @@
+from typing import Annotated, Dict, Optional, Union
+
+from pydantic import BaseModel, Field
+
+from pipelex.core.concepts.concept_blueprint import ConceptBlueprint
+from pipelex.pipe_controllers.pipe_batch_factory import PipeBatchBlueprint
+from pipelex.pipe_controllers.pipe_condition_factory import PipeConditionBlueprint
+from pipelex.pipe_controllers.pipe_parallel_factory import PipeParallelBlueprint
+from pipelex.pipe_controllers.pipe_sequence_factory import PipeSequenceBlueprint
+from pipelex.pipe_operators.pipe_func_factory import PipeFuncBlueprint
+from pipelex.pipe_operators.pipe_img_gen_factory import PipeImgGenBlueprint
+from pipelex.pipe_operators.pipe_jinja2_factory import PipeJinja2Blueprint
+from pipelex.pipe_operators.pipe_llm_factory import PipeLLMBlueprint
+from pipelex.pipe_operators.pipe_ocr_factory import PipeOcrBlueprint
+
+PipeBlueprintUnion = Annotated[
+ Union[
+ # Pipe operators
+ PipeFuncBlueprint,
+ PipeImgGenBlueprint,
+ PipeJinja2Blueprint,
+ PipeLLMBlueprint,
+ PipeOcrBlueprint,
+ # Pipe controllers
+ PipeBatchBlueprint,
+ PipeConditionBlueprint,
+ PipeParallelBlueprint,
+ PipeSequenceBlueprint,
+ ],
+ Field(discriminator="type"),
+]
+
+
+class PipelexBundleBlueprint(BaseModel):
+ """Complete blueprint of a pipelex bundle TOML definition."""
+
+ domain: str
+ definition: Optional[str] = None
+ system_prompt: Optional[str] = None
+ system_prompt_to_structure: Optional[str] = None
+ prompt_template_to_structure: Optional[str] = None
+
+ concept: Optional[Dict[str, ConceptBlueprint | str]] = Field(default_factory=dict)
+
+ pipe: Optional[Dict[str, PipeBlueprintUnion]] = Field(default_factory=dict)
diff --git a/pipelex/core/bundles/pipelex_bundle_factory.py b/pipelex/core/bundles/pipelex_bundle_factory.py
new file mode 100644
index 000000000..bf45adbf6
--- /dev/null
+++ b/pipelex/core/bundles/pipelex_bundle_factory.py
@@ -0,0 +1,47 @@
+from typing import Dict
+
+from pydantic import BaseModel
+
+from pipelex.core.bundles.pipelex_bundle import PipelexBundle
+from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint
+from pipelex.core.concepts.concept import Concept
+from pipelex.core.concepts.concept_blueprint import ConceptBlueprint
+from pipelex.core.concepts.concept_factory import ConceptFactory
+from pipelex.core.domains.domain import Domain
+from pipelex.core.pipes.pipe_abstract import PipeAbstract
+from pipelex.core.pipes.pipe_factory import PipeFactory
+
+
+class PipelexBundleFactory(BaseModel):
+ @classmethod
+ def make_from_blueprint(cls, blueprint: PipelexBundleBlueprint) -> PipelexBundle:
+ """Make a PipelexBundle from a PipelexBundleBlueprint."""
+ domain = Domain(
+ code=blueprint.domain,
+ definition=blueprint.definition,
+ system_prompt=blueprint.system_prompt,
+ system_prompt_to_structure=blueprint.system_prompt_to_structure,
+ prompt_template_to_structure=blueprint.prompt_template_to_structure,
+ )
+ concepts: Dict[str, Concept] = {}
+ if blueprint.concept is not None:
+ for concept_name, concept_blueprint_or_str in blueprint.concept.items():
+ if isinstance(concept_blueprint_or_str, ConceptBlueprint):
+ concepts[concept_name] = ConceptFactory.make_concept_from_blueprint(
+ domain=blueprint.domain, code=concept_name, concept_blueprint=concept_blueprint_or_str
+ )
+ else:
+ # One line concepts
+ concepts[concept_name] = ConceptFactory.make_concept_from_definition_str(
+ domain_code=blueprint.domain, concept_str=concept_name, definition=concept_blueprint_or_str
+ )
+
+ pipes: Dict[str, PipeAbstract] = {}
+ if blueprint.pipe is not None:
+ for pipe_name, pipe_blueprint in blueprint.pipe.items():
+ pipes[pipe_name] = PipeFactory.make_pipe_from_blueprint(
+ domain_code=blueprint.domain,
+ pipe_code=pipe_name,
+ pipe_blueprint=pipe_blueprint,
+ )
+ return PipelexBundle(domain=domain, concepts=concepts, pipes=pipes)
diff --git a/pipelex/core/concept.py b/pipelex/core/concepts/concept.py
similarity index 89%
rename from pipelex/core/concept.py
rename to pipelex/core/concepts/concept.py
index e36b5e69b..4b98b9175 100644
--- a/pipelex/core/concept.py
+++ b/pipelex/core/concepts/concept.py
@@ -6,10 +6,10 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator, model_valida
from typing_extensions import Self
from pipelex import log
-from pipelex.core.concept_native import NativeConcept
-from pipelex.core.domain import SpecialDomain
-from pipelex.core.stuff_content import StuffContent
-from pipelex.exceptions import ConceptCodeError, ConceptDomainError, ConceptError, StructureClassError
+from pipelex.core.concepts.concept_native import NativeConcept
+from pipelex.core.domains.domain import SpecialDomain
+from pipelex.core.stuffs.stuff_content import StuffContent
+from pipelex.exceptions import ConceptCodeError, ConceptDomainError, ConceptError
from pipelex.tools.misc.string_utils import pascal_case_to_sentence
@@ -18,11 +18,12 @@ class Concept(BaseModel):
code: str
domain: str
- structure_class_name: str
definition: str
+ structure_class_name: str
refines: List[str] = Field(default_factory=list)
@model_validator(mode="after")
+ # TODO: Refacto, its not clean
def validate_code_domain(self) -> Self:
if not Concept.concept_str_contains_domain(self.code):
raise ConceptCodeError(f"Code must contain a dot (.) for concept with code '{self.code}' and domain '{self.domain}'")
@@ -83,24 +84,6 @@ class Concept(BaseModel):
return validated_refines
- @field_validator("structure_class_name")
- @classmethod
- def validate_structure_class_name(cls, value: str) -> str:
- if not cls.is_valid_structure_class(structure_class_name=value):
- raise StructureClassError(f"Could not validate concept because structure_class_name '{value}' is not in class registry. ")
- return value
-
- @classmethod
- def is_valid_structure_class(cls, structure_class_name: str) -> bool:
- # we get_class_registry directly from KajsonManager instead of pipelex hub to avoid circular import
- if KajsonManager.get_class_registry().has_subclass(name=structure_class_name, base_class=StuffContent):
- return True
- else:
- # we get_class_registry directly from KajsonManager instead of pipelex hub to avoid circular import
- if KajsonManager.get_class_registry().has_class(name=structure_class_name):
- log.warning(f"Concept class '{structure_class_name}' is registered but it's not a subclass of StuffContent")
- return False
-
@classmethod
def extract_domain_and_concept_from_str(cls, concept_str: str) -> Tuple[str, str]:
if "." in concept_str:
@@ -138,3 +121,14 @@ class Concept(BaseModel):
return domain == SpecialDomain.NATIVE.value
else:
return concept_str in NativeConcept.names()
+
+ @classmethod
+ def is_valid_structure_class(cls, structure_class_name: str) -> bool:
+ # We get_class_registry directly from KajsonManager instead of pipelex hub to avoid circular import
+ if KajsonManager.get_class_registry().has_subclass(name=structure_class_name, base_class=StuffContent):
+ return True
+ else:
+ # We get_class_registry directly from KajsonManager instead of pipelex hub to avoid circular import
+ if KajsonManager.get_class_registry().has_class(name=structure_class_name):
+ log.warning(f"Concept class '{structure_class_name}' is registered but it's not a subclass of StuffContent")
+ return False
diff --git a/pipelex/core/concepts/concept_blueprint.py b/pipelex/core/concepts/concept_blueprint.py
new file mode 100644
index 000000000..4012935f5
--- /dev/null
+++ b/pipelex/core/concepts/concept_blueprint.py
@@ -0,0 +1,81 @@
+from typing import Any, Dict, List, Optional, Union
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+from typing_extensions import Self
+
+from pipelex.exceptions import ConceptFactoryError
+from pipelex.types import StrEnum
+
+
+class ConceptStructureBlueprintError(ConceptFactoryError):
+ pass
+
+
+class ConceptStructureBlueprintFieldType(StrEnum):
+ TEXT = "text"
+ LIST = "list"
+ DICT = "dict"
+ INTEGER = "integer"
+ BOOLEAN = "boolean"
+ NUMBER = "number"
+ DATE = "date"
+
+
+class ConceptStructureBlueprint(BaseModel):
+ definition: str
+ type: ConceptStructureBlueprintFieldType | None = None
+ item_type: Optional[str] = None
+ key_type: Optional[str] = None
+ value_type: Optional[str] = None
+ choices: Optional[List[str]] = Field(default_factory=list)
+ required: Optional[bool] = Field(default=True)
+
+ @model_validator(mode="after")
+ def validate_structure_blueprint(self) -> Self:
+ """Validate the structure blueprint according to type rules."""
+ # If type is None (array), choices must not be None
+ if self.type is None and not self.choices:
+ raise ConceptStructureBlueprintError("When type is None (array), choices must not be empty")
+
+ # If type is "dict", key_type and value_type must not be empty
+ if self.type == ConceptStructureBlueprintFieldType.DICT:
+ if not self.key_type:
+ raise ConceptStructureBlueprintError(f"When type is '{ConceptStructureBlueprintFieldType.DICT}', key_type must not be empty")
+ if not self.value_type:
+ raise ConceptStructureBlueprintError(f"When type is '{ConceptStructureBlueprintFieldType.DICT}', value_type must not be empty")
+
+ return self
+
+
+ConceptStructureBlueprintType = Union[str, ConceptStructureBlueprint]
+
+
+class ConceptBlueprint(BaseModel):
+ model_config = ConfigDict(extra="forbid")
+
+ definition: str
+ structure: Optional[Union[str, Dict[str, ConceptStructureBlueprintType]]] = None
+ refines: Optional[Union[str, List[str]]] = Field(default_factory=list)
+
+ @model_validator(mode="after")
+ def model_validate_blueprint(self) -> Self:
+ return self
+
+ def structure_to_field_def(self) -> Dict[str, Any]:
+ if isinstance(self.structure, str):
+ raise ValueError("structure_to_field_def can only be called when structure is a dict in the blueprint")
+
+ if self.structure is None:
+ return {}
+
+ # Process the dict structure
+ result: Dict[str, Any] = {}
+ for key, value in self.structure.items():
+ if isinstance(value, ConceptStructureBlueprint):
+ # Use model_dump for ConceptStructureBlueprint instances
+ result[key] = value.model_dump()
+ else:
+ # This shouldn't happen based on the type hints, but handle it gracefully
+ result[key] = {"type": ConceptStructureBlueprintFieldType.TEXT, "definition": value}
+
+ return result
diff --git a/pipelex/core/concept_code_factory.py b/pipelex/core/concepts/concept_code_factory.py
similarity index 69%
rename from pipelex/core/concept_code_factory.py
rename to pipelex/core/concepts/concept_code_factory.py
index 4a533b39f..dbfd0cd9c 100644
--- a/pipelex/core/concept_code_factory.py
+++ b/pipelex/core/concepts/concept_code_factory.py
@@ -1,7 +1,7 @@
from typing import Optional
-from pipelex.core.concept import Concept
-from pipelex.core.concept_native import NativeConcept
+from pipelex.core.concepts.concept import Concept
+from pipelex.core.concepts.concept_native import NativeConcept
from pipelex.exceptions import ConceptFactoryError
@@ -13,12 +13,14 @@ class ConceptCodeFactory:
return f"{domain}.{code}"
@classmethod
- def make_concept_code_from_str(cls, concept_str: str, fallback_domain: Optional[str] = None) -> str:
+ def make_concept_code_from_str(cls, concept_str: str, domain: Optional[str] = None, fallback_domain: Optional[str] = None) -> str:
if Concept.concept_str_contains_domain(concept_str=concept_str):
return concept_str
elif concept_str in NativeConcept.names():
native_concept = NativeConcept(concept_str)
return native_concept.code
+ elif domain:
+ return cls.make_concept_code(domain=domain, code=concept_str)
elif fallback_domain:
return cls.make_concept_code(domain=fallback_domain, code=concept_str)
else:
diff --git a/pipelex/core/concept_factory.py b/pipelex/core/concepts/concept_factory.py
similarity index 55%
rename from pipelex/core/concept_factory.py
rename to pipelex/core/concepts/concept_factory.py
index 19d1e6a53..240e8b5ec 100644
--- a/pipelex/core/concept_factory.py
+++ b/pipelex/core/concepts/concept_factory.py
@@ -1,27 +1,18 @@
-from inspect import getsource
-from typing import Any, Dict, List, Optional, Type, Union
+from typing import Any, Dict, List, Union
-from pydantic import BaseModel, ConfigDict, Field, ValidationError
+from pydantic import ValidationError
-from pipelex.core.concept import Concept
-from pipelex.core.concept_code_factory import ConceptCodeFactory
-from pipelex.core.concept_native import NativeConcept, NativeConceptClass
-from pipelex.core.domain import SpecialDomain
-from pipelex.core.stuff_content import TextContent
+from pipelex.core.concepts.concept import Concept
+from pipelex.core.concepts.concept_blueprint import ConceptBlueprint
+from pipelex.core.concepts.concept_code_factory import ConceptCodeFactory
+from pipelex.core.concepts.concept_native import NativeConcept, NativeConceptClass
+from pipelex.core.domains.domain import SpecialDomain
+from pipelex.core.stuffs.stuff_content import TextContent
from pipelex.create.structured_output_generator import generate_structured_output_from_inline_definition
from pipelex.exceptions import ConceptFactoryError, StructureClassError
from pipelex.hub import get_class_registry
-class ConceptBlueprint(BaseModel):
- model_config = ConfigDict(extra="forbid")
-
- definition: str
- structure: Optional[Union[str, Dict[str, Any]]] = None
- refines: Union[str, List[str]] = Field(default_factory=list)
- domain: Optional[str] = None
-
-
class ConceptFactory:
@classmethod
def make_refines(cls, domain: str, refines: Union[str, List[str]]) -> List[str]:
@@ -31,57 +22,10 @@ class ConceptFactory:
concept_str_list = refines
new_refines: List[str] = []
for concept_str in concept_str_list:
- concept_code = ConceptCodeFactory.make_concept_code_from_str(concept_str=concept_str, fallback_domain=domain)
+ concept_code = ConceptCodeFactory.make_concept_code_from_str(concept_str=concept_str, domain=domain, fallback_domain=domain)
new_refines.append(concept_code)
return new_refines
- @classmethod
- def make_from_details_dict_if_possible(
- cls,
- domain: str,
- code: str,
- details_dict: Dict[str, Any],
- ) -> Optional[Concept]:
- if concept_definition := details_dict.pop("Concept", None):
- details_dict["definition"] = concept_definition
- details_dict["refines"] = ConceptFactory.make_refines(domain=domain, refines=details_dict.pop("refines", []))
- concept_blueprint = ConceptBlueprint.model_validate(details_dict)
- the_concept = ConceptFactory.make_concept_from_blueprint(domain=domain, code=code, concept_blueprint=concept_blueprint)
- return the_concept
- elif "definition" in details_dict:
- # legacy format
- details_dict["domain"] = domain
- details_dict["refines"] = ConceptFactory.make_refines(domain=domain, refines=details_dict.pop("refines", []))
- details_dict["code"] = ConceptCodeFactory.make_concept_code(domain, code)
- try:
- the_concept = Concept.model_validate(details_dict)
- except ValidationError as exc:
- raise ConceptFactoryError(f"Error validating concept: {exc}") from exc
- return the_concept
- else:
- return None
-
- @classmethod
- def make_from_details_dict(
- cls,
- domain_code: str,
- code: str,
- details_dict: Dict[str, Any],
- ) -> Concept:
- concept_definition = details_dict.pop("Concept", None)
- if not concept_definition:
- raise ConceptFactoryError(f"Concept '{code}' in domain '{domain_code}' has no definition")
- details_dict["definition"] = concept_definition
- details_dict["domain"] = domain_code
- refines = ConceptFactory.make_refines(domain=domain_code, refines=details_dict.pop("refines", []))
- if not refines and not details_dict.get("structure"):
- # No structure? this refines Text
- refines = [NativeConcept.TEXT.code]
- details_dict["refines"] = refines
- concept_blueprint = ConceptBlueprint.model_validate(details_dict)
- the_concept = ConceptFactory.make_concept_from_blueprint(domain=domain_code, code=code, concept_blueprint=concept_blueprint)
- return the_concept
-
@classmethod
def make_concept_from_definition_str(
cls,
@@ -122,23 +66,29 @@ class ConceptFactory:
code: str,
concept_blueprint: ConceptBlueprint,
) -> Concept:
- structure_class_name: str
- if structure := concept_blueprint.structure:
- if isinstance(structure, str):
- # structure is set explicitly as a class name reference
- if not Concept.is_valid_structure_class(structure_class_name=structure):
+ current_refines: List[str]
+ if concept_blueprint.refines:
+ current_refines = cls.make_refines(domain=domain, refines=concept_blueprint.refines)
+ else:
+ current_refines = []
+
+ structure_class_name: str = code
+ if concept_blueprint.structure:
+ if isinstance(concept_blueprint.structure, str):
+ # Structure is defined inline - generate Python class dynamically
+ if not Concept.is_valid_structure_class(structure_class_name=concept_blueprint.structure):
raise StructureClassError(
- f"Structure class '{structure}' set for concept '{code}' in domain '{domain}' is not a registered subclass of StuffContent"
+ f"Structure class '{concept_blueprint.structure}' set for concept '{code}' in domain '{domain}' "
+ "is not a registered subclass of StuffContent"
)
- structure_class_name = structure
+ structure_class_name = concept_blueprint.structure
else:
- # structure is defined inline - generate Python class dynamically
- structure_class_name = code
+ # Structure is defined as a ConceptStructureBlueprint
try:
# Generate Python class from inline definition
python_code = generate_structured_output_from_inline_definition(
class_name=code,
- fields_def=structure, # type: ignore[arg-type]
+ fields_def=concept_blueprint.structure_to_field_def(),
enums=None, # TODO: Handle enums if needed in the future
)
@@ -153,19 +103,24 @@ class ConceptFactory:
except Exception as exc:
raise ConceptFactoryError(f"Error generating structure class for concept '{code}' in domain '{domain}': {exc}") from exc
elif Concept.is_valid_structure_class(structure_class_name=code):
- # structure is set implicitly, by the concept's code
- structure_class_name = code
+ # No structure defined on the blueprint, but the concept code is a valid structure class
+ pass
else:
- structure_class_name = TextContent.__name__
-
- refines_list = cls.make_refines(domain=domain, refines=concept_blueprint.refines)
+ if concept_blueprint.refines:
+ # Has a refining element
+ pass
+ else:
+ # Fallback to Text structure
+ structure_class_name = TextContent.__name__
+ current_refines = [NativeConcept.TEXT.code]
+ refines = cls.make_refines(domain=domain, refines=current_refines)
return Concept(
code=ConceptCodeFactory.make_concept_code(domain, code),
domain=domain,
definition=concept_blueprint.definition,
structure_class_name=structure_class_name,
- refines=refines_list,
+ refines=refines,
)
@classmethod
@@ -211,11 +166,3 @@ class ConceptFactory:
continue
concepts.append(cls.make_native_concept(native_concept=native_concept))
return concepts
-
- @classmethod
- def get_concept_class_source_code(cls, concept_name: str, base_class: Type[Any]) -> str:
- if not get_class_registry().has_class(concept_name):
- raise RuntimeError(f"Class '{concept_name}' not found in registry")
-
- cls = get_class_registry().get_required_subclass(name=concept_name, base_class=base_class)
- return getsource(cls)
diff --git a/pipelex/core/concept_library.py b/pipelex/core/concepts/concept_library.py
similarity index 94%
rename from pipelex/core/concept_library.py
rename to pipelex/core/concepts/concept_library.py
index 99b6f3ded..a79a4d8d1 100644
--- a/pipelex/core/concept_library.py
+++ b/pipelex/core/concepts/concept_library.py
@@ -4,13 +4,13 @@ from pydantic import Field, RootModel
from typing_extensions import override
from pipelex import log
-from pipelex.core.concept import Concept
-from pipelex.core.concept_code_factory import ConceptCodeFactory
-from pipelex.core.concept_factory import ConceptFactory
-from pipelex.core.concept_native import NativeConcept
-from pipelex.core.concept_provider_abstract import ConceptProviderAbstract
-from pipelex.core.domain import SpecialDomain
-from pipelex.core.stuff_content import ImageContent
+from pipelex.core.concepts.concept import Concept
+from pipelex.core.concepts.concept_code_factory import ConceptCodeFactory
+from pipelex.core.concepts.concept_factory import ConceptFactory
+from pipelex.core.concepts.concept_native import NativeConcept
+from pipelex.core.concepts.concept_provider_abstract import ConceptProviderAbstract
+from pipelex.core.domains.domain import SpecialDomain
+from pipelex.core.stuffs.stuff_content import ImageContent
from pipelex.exceptions import ConceptLibraryConceptNotFoundError, ConceptLibraryError
from pipelex.hub import get_class_registry
@@ -138,14 +138,13 @@ class ConceptLibrary(RootModel[ConceptLibraryRoot], ConceptProviderAbstract):
concept_code = f"{domain}.{concept_code}"
else:
concept_code = f"{SpecialDomain.NATIVE.value}.{concept_code}"
-
the_concept = self.get_concept(concept_code=concept_code)
if not the_concept:
if self.is_concept_implicit(concept_code=concept_code):
# The implicit concept is obviously coming with a domain (the one it is used in)
# TODO: replace this with a concept factory method make_implicit_concept
return ConceptFactory.make_concept_from_definition_str(
- domain_code="implicit",
+ domain_code=SpecialDomain.IMPLICIT,
concept_str=Concept.extract_domain_and_concept_from_str(concept_str=concept_code)[1],
definition=concept_code,
)
diff --git a/pipelex/core/concept_native.py b/pipelex/core/concepts/concept_native.py
similarity index 98%
rename from pipelex/core/concept_native.py
rename to pipelex/core/concepts/concept_native.py
index efc50c3e2..b9c5a349c 100644
--- a/pipelex/core/concept_native.py
+++ b/pipelex/core/concepts/concept_native.py
@@ -1,7 +1,7 @@
from enum import Enum
from typing import List
-from pipelex.core.domain import SpecialDomain
+from pipelex.core.domains.domain import SpecialDomain
from pipelex.types import StrEnum
diff --git a/pipelex/core/concept_provider_abstract.py b/pipelex/core/concepts/concept_provider_abstract.py
similarity index 96%
rename from pipelex/core/concept_provider_abstract.py
rename to pipelex/core/concepts/concept_provider_abstract.py
index 379b96490..5f2f33f4e 100644
--- a/pipelex/core/concept_provider_abstract.py
+++ b/pipelex/core/concepts/concept_provider_abstract.py
@@ -1,7 +1,7 @@
from abc import ABC, abstractmethod
from typing import Any, Dict, List, Optional, Type
-from pipelex.core.concept import Concept
+from pipelex.core.concepts.concept import Concept
ConceptLibraryRoot = Dict[str, Concept]
diff --git a/pipelex/core/domain.py b/pipelex/core/domains/domain.py
similarity index 100%
rename from pipelex/core/domain.py
rename to pipelex/core/domains/domain.py
diff --git a/pipelex/core/domain_library.py b/pipelex/core/domains/domain_library.py
similarity index 93%
rename from pipelex/core/domain_library.py
rename to pipelex/core/domains/domain_library.py
index e2b3b4005..372ab0536 100644
--- a/pipelex/core/domain_library.py
+++ b/pipelex/core/domains/domain_library.py
@@ -4,8 +4,8 @@ from pydantic import RootModel
from typing_extensions import override
from pipelex import log
-from pipelex.core.domain import Domain
-from pipelex.core.domain_provider_abstract import DomainProviderAbstract
+from pipelex.core.domains.domain import Domain
+from pipelex.core.domains.domain_provider_abstract import DomainProviderAbstract
from pipelex.exceptions import DomainLibraryError
from pipelex.tools.misc.file_utils import save_text_to_path
@@ -23,7 +23,7 @@ class DomainLibrary(RootModel[DomainLibraryRoot], DomainProviderAbstract):
def make_empty(cls):
return cls(root={})
- def add_domain_details(self, domain: Domain):
+ def add_domain(self, domain: Domain):
domain_code = domain.code
if existing_domain := self.root.get(domain_code):
# merge the new domain with the existing one
diff --git a/pipelex/core/domain_provider_abstract.py b/pipelex/core/domains/domain_provider_abstract.py
similarity index 91%
rename from pipelex/core/domain_provider_abstract.py
rename to pipelex/core/domains/domain_provider_abstract.py
index 0711431f1..08e37bcba 100644
--- a/pipelex/core/domain_provider_abstract.py
+++ b/pipelex/core/domains/domain_provider_abstract.py
@@ -1,7 +1,7 @@
from abc import ABC, abstractmethod
from typing import Dict, List, Optional
-from pipelex.core.domain import Domain
+from pipelex.core.domains.domain import Domain
class DomainProviderAbstract(ABC):
diff --git a/pipelex/core/interpreter.py b/pipelex/core/interpreter.py
new file mode 100644
index 000000000..a67c2b748
--- /dev/null
+++ b/pipelex/core/interpreter.py
@@ -0,0 +1,457 @@
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+import toml
+from pydantic import BaseModel, model_validator
+from typing_extensions import Self
+
+from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint
+from pipelex.core.concepts.concept_blueprint import ConceptBlueprint
+from pipelex.pipe_controllers.pipe_batch_factory import PipeBatchBlueprint
+from pipelex.pipe_controllers.pipe_condition_factory import PipeConditionBlueprint
+from pipelex.pipe_controllers.pipe_parallel_factory import PipeParallelBlueprint
+from pipelex.pipe_controllers.pipe_sequence_factory import PipeSequenceBlueprint
+from pipelex.pipe_operators.pipe_func_factory import PipeFuncBlueprint
+from pipelex.pipe_operators.pipe_img_gen_factory import PipeImgGenBlueprint
+from pipelex.pipe_operators.pipe_jinja2_factory import PipeJinja2Blueprint
+from pipelex.pipe_operators.pipe_llm_factory import PipeLLMBlueprint
+from pipelex.pipe_operators.pipe_ocr_factory import PipeOcrBlueprint
+from pipelex.tools.misc.toml_utils import clean_trailing_whitespace, dict_to_toml, validate_toml_content, validate_toml_file
+
+
+class PipelexInterpreter(BaseModel):
+ """TOML -> PipelexBundleBlueprint"""
+
+ file_path: Optional[Path] = None
+ file_content: Optional[str] = None
+
+ @model_validator(mode="after")
+ def check_file_path_or_file_content(self) -> Self:
+ """Need to check if there is at least one of file_path or file_content"""
+ if self.file_path is None and self.file_content is None:
+ raise ValueError("Either file_path or file_content must be provided")
+ return self
+
+ @model_validator(mode="after")
+ def validate_file_path(self) -> Self:
+ if self.file_path:
+ validate_toml_file(path=str(self.file_path))
+ if self.file_content:
+ validate_toml_content(content=self.file_content, file_path=str(self.file_path))
+ return self
+
+ @staticmethod
+ def is_pipelex_file(file_path: Path) -> bool:
+ """Check if a file is a valid Pipelex TOML file.
+
+ Args:
+ file_path: Path to the file to check
+
+ Returns:
+ True if the file is a Pipelex file, False otherwise
+
+ Criteria:
+ - Has .toml extension
+ - Starts with "domain =" (ignoring leading whitespace)
+ """
+ # Check if it has .toml extension
+ if file_path.suffix != ".toml":
+ return False
+
+ # Check if file exists
+ if not file_path.exists() or not file_path.is_file():
+ return False
+
+ try:
+ # Read the first few lines to check for "domain ="
+ with open(file_path, "r", encoding="utf-8") as f:
+ # Read first 100 characters (should be enough to find domain)
+ content = f.read(100)
+ # Remove leading whitespace and check if it starts with "domain ="
+ stripped_content = content.lstrip()
+ return stripped_content.startswith("domain =")
+ except Exception:
+ # If we can't read the file, it's not a valid Pipelex file
+ return False
+
+ def _load_toml_content(self) -> str:
+ """Load TOML content from file_path or use file_content directly."""
+ if self.file_path:
+ try:
+ with open(self.file_path, "r", encoding="utf-8") as file:
+ file_content = file.read()
+
+ # Clean trailing whitespace and write back if needed
+ cleaned_content = clean_trailing_whitespace(file_content)
+ if file_content != cleaned_content:
+ with open(self.file_path, "w", encoding="utf-8") as file:
+ file.write(cleaned_content)
+ return cleaned_content
+
+ return file_content
+
+ except Exception as exc:
+ raise ValueError(f"Failed to read TOML file '{self.file_path}': {exc}") from exc
+ else:
+ if self.file_content is None:
+ raise ValueError("file_content must be provided if file_path is not provided")
+ return self.file_content
+
+ def _parse_toml_content(self, content: str) -> Dict[str, Any]:
+ """Parse TOML content and return the dictionary."""
+ try:
+ return toml.loads(content)
+ except toml.TomlDecodeError as exc:
+ file_path_str = str(self.file_path) if self.file_path else "content"
+ raise toml.TomlDecodeError(f"TOML parsing error in '{file_path_str}': {exc}", exc.doc, exc.pos) from exc
+
+ def make_pipelex_bundle_blueprint(self) -> PipelexBundleBlueprint:
+ """Make a PipelexBundleBlueprint from the file_path or file_content"""
+ file_content = self._load_toml_content()
+ toml_data = self._parse_toml_content(file_content)
+ return PipelexBundleBlueprint.model_validate(toml_data)
+
+ @staticmethod
+ def make_toml_content(blueprint: PipelexBundleBlueprint) -> str:
+ """Convert a PipelexBundleBlueprint to properly formatted TOML content."""
+ toml_data: Dict[str, Any] = {}
+
+ # Domain-level fields
+ toml_data["domain"] = blueprint.domain
+ if blueprint.definition:
+ toml_data["definition"] = blueprint.definition
+ if blueprint.system_prompt:
+ toml_data["system_prompt"] = blueprint.system_prompt
+ if blueprint.system_prompt_to_structure:
+ toml_data["system_prompt_to_structure"] = blueprint.system_prompt_to_structure
+ if blueprint.prompt_template_to_structure:
+ toml_data["prompt_template_to_structure"] = blueprint.prompt_template_to_structure
+
+ # Concepts section - always include if concepts is not None
+ if blueprint.concept is not None:
+ toml_data["concept"] = PipelexInterpreter._serialize_concepts(blueprint.concept, blueprint.domain)
+
+ # Pipes section
+ if blueprint.pipe:
+ toml_data["pipe"] = PipelexInterpreter._serialize_pipes(blueprint.pipe, blueprint.domain)
+
+ return dict_to_toml(toml_data)
+
+ @staticmethod
+ def _serialize_concepts(concepts: Optional[Dict[str, ConceptBlueprint | str]], domain: str) -> Dict[str, Any]:
+ """Serialize concepts section with domain context."""
+ result: Dict[str, Any] = {}
+ if concepts is None:
+ return {}
+
+ for concept_name, concept_blueprint in concepts.items():
+ if isinstance(concept_blueprint, str):
+ # Simple string concept
+ result[concept_name] = concept_blueprint
+ else:
+ # Complex ConceptBlueprint
+ if hasattr(concept_blueprint, "structure") and concept_blueprint.structure:
+ # Structured concept - create nested structure
+ concept_data: Dict[str, Any] = {}
+ if concept_blueprint.definition:
+ concept_data["definition"] = concept_blueprint.definition
+ if concept_blueprint.structure:
+ concept_data["structure"] = {}
+ if isinstance(concept_blueprint.structure, str):
+ concept_data["structure"] = concept_blueprint.structure
+ else:
+ for field_name, field_value in concept_blueprint.structure.items():
+ if isinstance(field_value, str):
+ concept_data["structure"][field_name] = field_value
+ else:
+ # ConceptStructureBlueprint
+ field_data: Dict[str, Any] = {}
+ field_data["type"] = field_value.type
+ field_data["definition"] = field_value.definition
+ field_data["required"] = field_value.required
+ concept_data["structure"][field_name] = field_data
+ result[concept_name] = concept_data
+ elif hasattr(concept_blueprint, "refines") and concept_blueprint.refines:
+ # Concept with refines
+ concept_data = {}
+ if concept_blueprint.definition:
+ concept_data["definition"] = concept_blueprint.definition
+ concept_data["refines"] = concept_blueprint.refines
+ result[concept_name] = concept_data
+ else:
+ # Simple concept with just definition
+ result[concept_name] = concept_blueprint.definition if concept_blueprint.definition else f"Concept {concept_name}"
+ return result
+
+ @staticmethod
+ def _serialize_pipes(pipes: Dict[str, Any], domain: str) -> Dict[str, Any]:
+ """Serialize pipes section with domain context."""
+ result: Dict[str, Any] = {}
+ for pipe_name, pipe_blueprint in pipes.items():
+ result[pipe_name] = PipelexInterpreter._serialize_pipe(pipe_blueprint, domain)
+ return result
+
+ @staticmethod
+ def _serialize_pipe(pipe_blueprint: Any, domain: str) -> Dict[str, Any]:
+ """Serialize a single pipe blueprint with domain context."""
+
+ if isinstance(pipe_blueprint, PipeLLMBlueprint):
+ return PipelexInterpreter._serialize_llm_pipe(pipe_blueprint, domain)
+ elif isinstance(pipe_blueprint, PipeOcrBlueprint):
+ return PipelexInterpreter._serialize_ocr_pipe(pipe_blueprint, domain)
+ elif isinstance(pipe_blueprint, PipeFuncBlueprint):
+ return PipelexInterpreter._serialize_func_pipe(pipe_blueprint, domain)
+ elif isinstance(pipe_blueprint, PipeImgGenBlueprint):
+ return PipelexInterpreter._serialize_img_gen_pipe(pipe_blueprint, domain)
+ elif isinstance(pipe_blueprint, PipeJinja2Blueprint):
+ return PipelexInterpreter._serialize_jinja2_pipe(pipe_blueprint, domain)
+ elif isinstance(pipe_blueprint, PipeSequenceBlueprint):
+ return PipelexInterpreter._serialize_sequence_pipe(pipe_blueprint, domain)
+ elif isinstance(pipe_blueprint, PipeConditionBlueprint):
+ return PipelexInterpreter._serialize_condition_pipe(pipe_blueprint, domain)
+ elif isinstance(pipe_blueprint, PipeParallelBlueprint):
+ return PipelexInterpreter._serialize_parallel_pipe(pipe_blueprint, domain)
+ elif isinstance(pipe_blueprint, PipeBatchBlueprint):
+ return PipelexInterpreter._serialize_batch_pipe(pipe_blueprint, domain)
+ else:
+ raise ValueError(f"Unknown pipe blueprint type: {type(pipe_blueprint)}")
+
+ @staticmethod
+ def _serialize_llm_pipe(pipe: PipeLLMBlueprint, domain: str) -> Dict[str, Any]:
+ """Serialize PipeLLM blueprint."""
+ result: Dict[str, Any] = {
+ "type": pipe.type,
+ "definition": pipe.definition,
+ "output": pipe.output,
+ }
+
+ # Add optional fields only if they have values
+ if pipe.inputs:
+ result["inputs"] = PipelexInterpreter._serialize_inputs(pipe.inputs)
+ if pipe.prompt_template:
+ result["prompt_template"] = pipe.prompt_template
+ if pipe.system_prompt:
+ result["system_prompt"] = pipe.system_prompt
+
+ return result
+
+ @staticmethod
+ def _serialize_ocr_pipe(pipe: PipeOcrBlueprint, domain: str) -> Dict[str, Any]:
+ """Serialize PipeOcr blueprint."""
+ result: Dict[str, Any] = {
+ "type": pipe.type,
+ "definition": pipe.definition,
+ "output": pipe.output,
+ }
+
+ # Add optional fields only if they have values
+ if pipe.inputs:
+ result["inputs"] = PipelexInterpreter._serialize_inputs(pipe.inputs)
+
+ return result
+
+ @staticmethod
+ def _serialize_func_pipe(pipe: PipeFuncBlueprint, domain: str) -> Dict[str, Any]:
+ """Serialize PipeFunc blueprint."""
+ result: Dict[str, Any] = {
+ "type": pipe.type,
+ "definition": pipe.definition,
+ "output": pipe.output,
+ "function_name": pipe.function_name,
+ }
+
+ # Add optional fields only if they have values
+ if pipe.inputs:
+ result["inputs"] = PipelexInterpreter._serialize_inputs(pipe.inputs)
+
+ return result
+
+ @staticmethod
+ def _serialize_img_gen_pipe(pipe: PipeImgGenBlueprint, domain: str) -> Dict[str, Any]:
+ """Serialize PipeImgGen blueprint."""
+ result: Dict[str, Any] = {
+ "type": pipe.type,
+ "definition": pipe.definition,
+ "output": pipe.output,
+ }
+
+ # Add optional fields only if they have values
+ if pipe.inputs:
+ result["inputs"] = PipelexInterpreter._serialize_inputs(pipe.inputs)
+ if pipe.img_gen_prompt:
+ result["img_gen_prompt"] = pipe.img_gen_prompt
+ if pipe.imgg_handle:
+ result["imgg_handle"] = pipe.imgg_handle
+ if pipe.aspect_ratio:
+ result["aspect_ratio"] = pipe.aspect_ratio
+ if pipe.quality:
+ result["quality"] = pipe.quality
+ if pipe.nb_steps:
+ result["nb_steps"] = pipe.nb_steps
+ if pipe.guidance_scale:
+ result["guidance_scale"] = pipe.guidance_scale
+ if pipe.is_moderated is not None:
+ result["is_moderated"] = pipe.is_moderated
+ if pipe.safety_tolerance:
+ result["safety_tolerance"] = pipe.safety_tolerance
+ if pipe.is_raw is not None:
+ result["is_raw"] = pipe.is_raw
+ if pipe.seed:
+ result["seed"] = pipe.seed
+ if pipe.nb_output:
+ result["nb_output"] = pipe.nb_output
+
+ return result
+
+ @staticmethod
+ def _serialize_jinja2_pipe(pipe: PipeJinja2Blueprint, domain: str) -> Dict[str, Any]:
+ """Serialize PipeJinja2 blueprint."""
+ result: Dict[str, Any] = {
+ "type": pipe.type,
+ "definition": pipe.definition,
+ "output": pipe.output,
+ }
+
+ # Add optional fields only if they have values
+ if pipe.inputs:
+ result["inputs"] = PipelexInterpreter._serialize_inputs(pipe.inputs)
+ if pipe.jinja2_name:
+ result["jinja2_name"] = pipe.jinja2_name
+ if pipe.jinja2:
+ result["jinja2"] = pipe.jinja2
+ if pipe.prompting_style:
+ result["prompting_style"] = pipe.prompting_style
+ # Only include template_category if it's not the default value
+ if pipe.template_category and pipe.template_category.value != "llm_prompt":
+ result["template_category"] = pipe.template_category
+
+ return result
+
+ @staticmethod
+ def _serialize_sequence_pipe(pipe: PipeSequenceBlueprint, domain: str) -> Dict[str, Any]:
+ """Serialize PipeSequence blueprint."""
+ result: Dict[str, Any] = {
+ "type": pipe.type,
+ "definition": pipe.definition,
+ "output": pipe.output,
+ }
+
+ # Add optional fields only if they have values
+ if pipe.inputs:
+ result["inputs"] = PipelexInterpreter._serialize_inputs(pipe.inputs)
+ if pipe.steps:
+ # Serialize steps, but only include non-default values
+ serialized_steps: List[Dict[str, Any]] = []
+ for step in pipe.steps:
+ step_data: Dict[str, Any] = {
+ "pipe": step.pipe,
+ "result": step.result,
+ }
+ # Only include optional fields if they have non-default values
+ if step.nb_output is not None:
+ step_data["nb_output"] = step.nb_output
+ if step.multiple_output is not None:
+ step_data["multiple_output"] = step.multiple_output
+ if step.batch_over is not False: # Only include if not default False
+ step_data["batch_over"] = step.batch_over
+ if step.batch_as is not None:
+ step_data["batch_as"] = step.batch_as
+ serialized_steps.append(step_data)
+ result["steps"] = serialized_steps
+
+ return result
+
+ @staticmethod
+ def _serialize_condition_pipe(pipe: PipeConditionBlueprint, domain: str) -> Dict[str, Any]:
+ """Serialize PipeCondition blueprint."""
+ result: Dict[str, Any] = {
+ "type": pipe.type,
+ "definition": pipe.definition,
+ "output": pipe.output,
+ "pipe_map": pipe.pipe_map,
+ }
+
+ # Add optional fields only if they have values
+ if pipe.inputs:
+ result["inputs"] = PipelexInterpreter._serialize_inputs(pipe.inputs)
+ if pipe.expression_template:
+ result["expression_template"] = pipe.expression_template
+ if pipe.expression:
+ result["expression"] = pipe.expression
+ if pipe.default_pipe_code:
+ result["default_pipe_code"] = pipe.default_pipe_code
+ if pipe.add_alias_from_expression_to:
+ result["add_alias_from_expression_to"] = pipe.add_alias_from_expression_to
+
+ return result
+
+ @staticmethod
+ def _serialize_parallel_pipe(pipe: PipeParallelBlueprint, domain: str) -> Dict[str, Any]:
+ """Serialize PipeParallel blueprint."""
+ result: Dict[str, Any] = {
+ "type": pipe.type,
+ "definition": pipe.definition,
+ "output": pipe.output,
+ }
+
+ # Add optional fields only if they have values
+ if pipe.inputs:
+ result["inputs"] = PipelexInterpreter._serialize_inputs(pipe.inputs)
+ if pipe.parallels:
+ # Serialize parallels, but only include non-default values
+ serialized_parallels: List[Dict[str, Any]] = []
+ for parallel in pipe.parallels:
+ parallel_data: Dict[str, Any] = {
+ "pipe": parallel.pipe,
+ "result": parallel.result,
+ }
+ # Only include optional fields if they have non-default values
+ if parallel.nb_output is not None:
+ parallel_data["nb_output"] = parallel.nb_output
+ if parallel.multiple_output is not None:
+ parallel_data["multiple_output"] = parallel.multiple_output
+ if parallel.batch_over is not False: # Only include if not default False
+ parallel_data["batch_over"] = parallel.batch_over
+ if parallel.batch_as is not None:
+ parallel_data["batch_as"] = parallel.batch_as
+ serialized_parallels.append(parallel_data)
+ result["parallels"] = serialized_parallels
+ if pipe.add_each_output is not True: # Only include if not default True
+ result["add_each_output"] = pipe.add_each_output
+ if pipe.combined_output:
+ result["combined_output"] = pipe.combined_output
+
+ return result
+
+ @staticmethod
+ def _serialize_batch_pipe(pipe: PipeBatchBlueprint, domain: str) -> Dict[str, Any]:
+ """Serialize PipeBatch blueprint."""
+ result: Dict[str, Any] = {
+ "type": pipe.type,
+ "definition": pipe.definition,
+ "output": pipe.output,
+ "branch_pipe_code": pipe.branch_pipe_code,
+ }
+
+ # Add optional fields only if they have values
+ if pipe.inputs:
+ result["inputs"] = PipelexInterpreter._serialize_inputs(pipe.inputs)
+ if pipe.input_list_name:
+ result["input_list_name"] = pipe.input_list_name
+ if pipe.input_item_name:
+ result["input_item_name"] = pipe.input_item_name
+
+ return result
+
+ @staticmethod
+ def _serialize_inputs(inputs: Dict[str, Any]) -> Dict[str, str]:
+ """Convert InputRequirementBlueprint objects to concept code strings for TOML serialization."""
+ result: Dict[str, str] = {}
+ for key, value in inputs.items():
+ if hasattr(value, "concept_code"):
+ # InputRequirementBlueprint object
+ result[key] = value.concept_code
+ else:
+ # Already a string
+ result[key] = str(value)
+ return result
diff --git a/pipelex/core/working_memory.py b/pipelex/core/memory/working_memory.py
similarity index 98%
rename from pipelex/core/working_memory.py
rename to pipelex/core/memory/working_memory.py
index dbee03c2e..d7c54fcca 100644
--- a/pipelex/core/working_memory.py
+++ b/pipelex/core/memory/working_memory.py
@@ -5,10 +5,10 @@ from pydantic import BaseModel, Field, model_validator
from typing_extensions import Self
from pipelex import log, pretty_print
-from pipelex.core.concept_native import NativeConcept
-from pipelex.core.stuff import Stuff
-from pipelex.core.stuff_artefact import StuffArtefact
-from pipelex.core.stuff_content import (
+from pipelex.core.concepts.concept_native import NativeConcept
+from pipelex.core.stuffs.stuff import Stuff
+from pipelex.core.stuffs.stuff_artefact import StuffArtefact
+from pipelex.core.stuffs.stuff_content import (
HtmlContent,
ImageContent,
ListContent,
diff --git a/pipelex/core/working_memory_factory.py b/pipelex/core/memory/working_memory_factory.py
similarity index 95%
rename from pipelex/core/working_memory_factory.py
rename to pipelex/core/memory/working_memory_factory.py
index e42ab4a1b..0cda7d5cf 100644
--- a/pipelex/core/working_memory_factory.py
+++ b/pipelex/core/memory/working_memory_factory.py
@@ -6,12 +6,12 @@ from pydantic import BaseModel
from pipelex import log
from pipelex.client.protocol import CompactMemory, ImplicitMemory
-from pipelex.core.concept_native import NativeConcept
-from pipelex.core.pipe_input_spec import TypedNamedInputRequirement
-from pipelex.core.stuff import Stuff
-from pipelex.core.stuff_content import ImageContent, ListContent, PDFContent, StuffContent, TextContent
-from pipelex.core.stuff_factory import StuffBlueprint, StuffFactory
-from pipelex.core.working_memory import MAIN_STUFF_NAME, StuffDict, WorkingMemory
+from pipelex.core.concepts.concept_native import NativeConcept
+from pipelex.core.memory.working_memory import MAIN_STUFF_NAME, StuffDict, WorkingMemory
+from pipelex.core.pipes.pipe_input_spec import TypedNamedInputRequirement
+from pipelex.core.stuffs.stuff import Stuff
+from pipelex.core.stuffs.stuff_content import ImageContent, ListContent, PDFContent, StuffContent, TextContent
+from pipelex.core.stuffs.stuff_factory import StuffBlueprint, StuffFactory
from pipelex.exceptions import WorkingMemoryFactoryError
from pipelex.tools.misc.json_utils import load_json_dict_from_path
diff --git a/pipelex/core/pipe_blueprint.py b/pipelex/core/pipe_blueprint.py
deleted file mode 100644
index 60488a4a0..000000000
--- a/pipelex/core/pipe_blueprint.py
+++ /dev/null
@@ -1,55 +0,0 @@
-from typing import Any, Dict, Optional, Protocol, TypeVar
-
-from pydantic import ConfigDict, model_validator
-from typing_extensions import Self, runtime_checkable
-
-from pipelex.core.concept_code_factory import ConceptCodeFactory
-from pipelex.core.pipe_abstract import PipeAbstract
-from pipelex.core.stuff_content import StructuredContent
-
-
-class PipeBlueprint(StructuredContent):
- model_config = ConfigDict(extra="forbid")
-
- definition: Optional[str] = None
- inputs: Optional[Dict[str, str]] = None
- output: str
- domain: str
-
- @model_validator(mode="after")
- def add_domain_prefix(self) -> Self:
- if self.inputs:
- for input_name, input_concept_code in self.inputs.items():
- self.inputs[input_name] = ConceptCodeFactory.make_concept_code_from_str(
- concept_str=input_concept_code,
- fallback_domain=self.domain,
- )
- self.output = ConceptCodeFactory.make_concept_code_from_str(
- concept_str=self.output,
- fallback_domain=self.domain,
- )
- return self
-
-
-PipeBlueprintType = TypeVar("PipeBlueprintType", bound="PipeBlueprint", contravariant=True)
-
-PipeType = TypeVar("PipeType", bound="PipeAbstract", covariant=True)
-
-
-@runtime_checkable
-class PipeSpecificFactoryProtocol(Protocol[PipeBlueprintType, PipeType]):
- @classmethod
- def make_pipe_from_blueprint(
- cls,
- domain_code: str,
- pipe_code: str,
- pipe_blueprint: PipeBlueprintType,
- ) -> PipeType: ...
-
- @classmethod
- def make_pipe_from_details_dict(
- cls,
- domain_code: str,
- pipe_code: str,
- details_dict: Dict[str, Any],
- ) -> PipeType: ...
diff --git a/pipelex/core/pipe_input_details.py b/pipelex/core/pipe_input_details.py
deleted file mode 100644
index 6e5bc96a0..000000000
--- a/pipelex/core/pipe_input_details.py
+++ /dev/null
@@ -1,76 +0,0 @@
-from typing import Callable, Dict, List, Set, Tuple
-
-from pydantic import Field, RootModel, field_validator
-
-from pipelex import log
-from pipelex.core.concept_code_factory import ConceptCodeFactory
-
-PipeInputDetailsRoot = Dict[str, str]
-
-
-class PipeInputDetails(RootModel[PipeInputDetailsRoot]):
- root: PipeInputDetailsRoot = Field(default_factory=dict)
-
- @field_validator("root", mode="wrap")
- @classmethod
- def validate_concept_codes(cls, input_value: Dict[str, str], handler: Callable[[Dict[str, str]], Dict[str, str]]) -> Dict[str, str]:
- # First let Pydantic handle the basic type validation
- validated_dict: Dict[str, str] = handler(input_value)
-
- # Now we can transform and validate the keys and values
- transformed_dict: Dict[str, str] = {}
- for required_input, concept_str in validated_dict.items():
- # in case of sub-attribute, the variable name is the object name, before the 1st dot
- transformed_key: str = required_input.split(".", 1)[0]
- if transformed_key != required_input:
- log.verbose(f"Sub-attribute {required_input} detected, using {transformed_key} as variable name")
-
- # Validate concept_code
- concept_code = ConceptCodeFactory.make_concept_code_from_str(concept_str=concept_str)
-
- if transformed_key in transformed_dict and transformed_dict[transformed_key] != concept_code:
- log.verbose(
- f"Variable {transformed_key} already exists with a different concept code: {transformed_dict[transformed_key]} -> {concept_str}"
- )
- transformed_dict[transformed_key] = concept_code
-
- return transformed_dict
-
- def set_default_domain(self, domain: str):
- for input_name, input_concept_code in self.root.items():
- if "." not in input_concept_code:
- self.root[input_name] = f"{domain}.{input_concept_code}"
-
- def get(self, variable_name: str) -> str:
- return self.root[variable_name]
-
- def add_requirement(self, variable_name: str, concept_code: str):
- self.root[variable_name] = concept_code
-
- @property
- def items(self) -> List[Tuple[str, str]]:
- return list(self.root.items())
-
- @property
- def concepts(self) -> Set[str]:
- return set(self.root.values())
-
- # @property
- # def variables(self) -> List[str]:
- # return list(self.root.keys())
-
- @property
- def required_names(self) -> List[str]:
- the_required_names: List[str] = []
- for requirement_expression in self.root.keys():
- required_variable_name = requirement_expression.split(".", 1)[0]
- the_required_names.append(required_variable_name)
- return the_required_names
-
- @property
- def detailed_requirements(self) -> List[Tuple[str, str, str]]:
- the_requirements: List[Tuple[str, str, str]] = []
- for requirement_expression, concept_code in self.root.items():
- required_variable_name = requirement_expression.split(".", 1)[0]
- the_requirements.append((required_variable_name, requirement_expression, concept_code))
- return the_requirements
diff --git a/pipelex/core/pipe_abstract.py b/pipelex/core/pipes/pipe_abstract.py
similarity index 60%
rename from pipelex/core/pipe_abstract.py
rename to pipelex/core/pipes/pipe_abstract.py
index 2caef096c..76e8192e0 100644
--- a/pipelex/core/pipe_abstract.py
+++ b/pipelex/core/pipes/pipe_abstract.py
@@ -1,12 +1,14 @@
from abc import ABC, abstractmethod
-from typing import Optional, Set, Type
+from typing import Any, Dict, Optional, Set, Type
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import BaseModel, ConfigDict, Field, model_validator
-from pipelex.core.pipe_input_spec import PipeInputSpec
-from pipelex.core.pipe_output import PipeOutput
-from pipelex.core.pipe_run_params import PipeRunParams
-from pipelex.core.working_memory import WorkingMemory
+from pipelex.core.concepts.concept_code_factory import ConceptCodeFactory
+from pipelex.core.domains.domain import SpecialDomain
+from pipelex.core.memory.working_memory import WorkingMemory
+from pipelex.core.pipes.pipe_input_spec import PipeInputSpec
+from pipelex.core.pipes.pipe_output import PipeOutput
+from pipelex.core.pipes.pipe_run_params import PipeRunParams
from pipelex.exceptions import PipeStackOverflowError
from pipelex.pipeline.job_metadata import JobMetadata
@@ -21,6 +23,32 @@ class PipeAbstract(ABC, BaseModel):
inputs: PipeInputSpec = Field(default_factory=PipeInputSpec)
output_concept_code: str
+ @model_validator(mode="before")
+ @classmethod
+ # TODO: This is a hack to add the domain prefix to the concept code.
+ # Fix this with a more simple Concept Factory approach.
+ def add_domain_prefix(cls, data: Dict[str, Any]) -> Dict[str, Any]:
+ # Process output_concept_code - always present
+ if "output_concept_code" in data:
+ data["output_concept_code"] = ConceptCodeFactory.make_concept_code_from_str(
+ domain=data["domain"],
+ concept_str=data["output_concept_code"],
+ fallback_domain=SpecialDomain.IMPLICIT,
+ )
+
+ # Process inputs - always present
+ if "inputs" in data:
+ inputs = data["inputs"]
+ if isinstance(inputs, PipeInputSpec):
+ for _, input_requirement in inputs.root.items():
+ input_requirement.concept_code = ConceptCodeFactory.make_concept_code_from_str(
+ concept_str=input_requirement.concept_code,
+ domain=data["domain"],
+ fallback_domain=SpecialDomain.IMPLICIT,
+ )
+
+ return data
+
@property
def class_name(self) -> str:
return self.__class__.__name__
diff --git a/pipelex/core/pipes/pipe_blueprint.py b/pipelex/core/pipes/pipe_blueprint.py
new file mode 100644
index 000000000..962143c38
--- /dev/null
+++ b/pipelex/core/pipes/pipe_blueprint.py
@@ -0,0 +1,48 @@
+from typing import Any, Dict, Optional
+
+from pydantic import BaseModel, model_validator
+from typing_extensions import Self
+
+from pipelex.core.pipes.pipe_input_spec import InputRequirementBlueprint
+from pipelex.types import StrEnum
+
+
+class PipeBlueprintError(Exception):
+ """Exception raised for errors in the PipeBlueprint class."""
+
+ pass
+
+
+class AllowedPipeTypes(StrEnum):
+ # Pipe Operators
+ PIPE_FUNC = "PipeFunc"
+ PIPE_IMG_GEN = "PipeImgGen"
+ PIPE_JINJA2 = "PipeJinja2"
+ PIPE_LLM = "PipeLLM"
+ PIPE_OCR = "PipeOcr"
+ # Pipe Controller
+ PIPE_BATCH = "PipeBatch"
+ PIPE_CONDITION = "PipeCondition"
+ PIPE_PARALLEL = "PipeParallel"
+ PIPE_SEQUENCE = "PipeSequence"
+
+
+class PipeBlueprint(BaseModel):
+ """Simple data container for pipe blueprint information.
+
+ The 'type' field uses Any to avoid type override conflicts but is validated
+ at runtime to ensure only valid pipe type values are allowed.
+ """
+
+ type: Any # TODO: Find a better way to handle this.
+ definition: Optional[str] = None
+ inputs: Optional[Dict[str, InputRequirementBlueprint]] = None
+ output: str
+
+ @model_validator(mode="after")
+ def validate_pipe_type(self) -> Self:
+ """Validate that the pipe type is one of the allowed values."""
+ allowed_types = [_type.value for _type in AllowedPipeTypes]
+ if self.type not in allowed_types:
+ raise PipeBlueprintError(f"Invalid pipe type '{self.type}'. Must be one of: {sorted(allowed_types)}")
+ return self
diff --git a/pipelex/core/pipes/pipe_factory.py b/pipelex/core/pipes/pipe_factory.py
new file mode 100644
index 000000000..eff1b405c
--- /dev/null
+++ b/pipelex/core/pipes/pipe_factory.py
@@ -0,0 +1,56 @@
+from typing import Any, Protocol, Type, TypeVar
+
+from kajson.exceptions import ClassRegistryInheritanceError, ClassRegistryNotFoundError
+from kajson.kajson_manager import KajsonManager
+from typing_extensions import override, runtime_checkable
+
+from pipelex.core.pipes.pipe_abstract import PipeAbstract
+from pipelex.core.pipes.pipe_blueprint import PipeBlueprint
+from pipelex.exceptions import PipeFactoryError
+
+PipeBlueprintType = TypeVar("PipeBlueprintType", bound="PipeBlueprint", contravariant=True)
+PipeType = TypeVar("PipeType", bound="PipeAbstract", covariant=True)
+
+
+@runtime_checkable
+class PipeFactoryProtocol(Protocol[PipeBlueprintType, PipeType]):
+ @classmethod
+ def make_pipe_from_blueprint(
+ cls,
+ domain_code: str,
+ pipe_code: str,
+ pipe_blueprint: PipeBlueprintType,
+ ) -> PipeType: ...
+
+
+class PipeFactory(PipeFactoryProtocol[PipeBlueprint, PipeAbstract]):
+ @classmethod
+ @override
+ def make_pipe_from_blueprint(
+ cls,
+ domain_code: str,
+ pipe_code: str,
+ pipe_blueprint: PipeBlueprint,
+ ) -> PipeAbstract:
+ # The factory class name for that specific type of Pipe is the pipe class name with "Factory" suffix
+ factory_class_name = f"{pipe_blueprint.type}Factory"
+ try:
+ pipe_factory: Type[PipeFactoryProtocol[Any, Any]] = KajsonManager.get_class_registry().get_required_subclass(
+ name=factory_class_name,
+ base_class=PipeFactoryProtocol,
+ )
+ except ClassRegistryNotFoundError as factory_not_found_error:
+ raise PipeFactoryError(
+ f"Pipe '{pipe_code}' couldn't be created: factory '{factory_class_name}' not found: {factory_not_found_error}"
+ ) from factory_not_found_error
+ except ClassRegistryInheritanceError as factory_inheritance_error:
+ raise PipeFactoryError(
+ f"Pipe '{pipe_code}' couldn't be created: factory '{factory_class_name}' is not a subclass of {type(PipeFactoryProtocol)}."
+ ) from factory_inheritance_error
+
+ pipe_from_blueprint: PipeAbstract = pipe_factory.make_pipe_from_blueprint(
+ domain_code=domain_code,
+ pipe_code=pipe_code,
+ pipe_blueprint=pipe_blueprint,
+ )
+ return pipe_from_blueprint
diff --git a/pipelex/core/pipe_input_spec.py b/pipelex/core/pipes/pipe_input_spec.py
similarity index 78%
rename from pipelex/core/pipe_input_spec.py
rename to pipelex/core/pipes/pipe_input_spec.py
index 5e489fb47..8b9244c36 100644
--- a/pipelex/core/pipe_input_spec.py
+++ b/pipelex/core/pipes/pipe_input_spec.py
@@ -1,15 +1,28 @@
-from typing import Callable, Dict, List, Optional, Set, Tuple, Type
+from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type
-from pydantic import BaseModel, Field, RootModel, field_validator
+from pydantic import BaseModel, Field, RootModel, field_validator, model_validator
from typing_extensions import Self
from pipelex import log
-from pipelex.core.concept_code_factory import ConceptCodeFactory
-from pipelex.core.pipe_run_params import PipeOutputMultiplicity
-from pipelex.core.stuff_content import StuffContent
+from pipelex.core.concepts.concept_code_factory import ConceptCodeFactory
+from pipelex.core.pipes.pipe_run_params import PipeOutputMultiplicity
+from pipelex.core.stuffs.stuff_content import StuffContent
from pipelex.exceptions import PipeInputNotFoundError
+class InputRequirementBlueprint(BaseModel):
+ concept_code: str
+ multiplicity: Optional[PipeOutputMultiplicity] = None
+
+ @model_validator(mode="before")
+ @classmethod
+ def validate_input(cls, data: Any) -> Any:
+ """If the incoming data is a string, convert it to a dict with concept_code."""
+ if isinstance(data, str):
+ return {"concept_code": data}
+ return data
+
+
class InputRequirement(BaseModel):
concept_code: str
multiplicity: Optional[PipeOutputMultiplicity] = None
@@ -93,11 +106,18 @@ class PipeInputSpec(RootModel[PipeInputSpecRoot]):
return cls(root={})
@classmethod
- def make_from_dict(cls, concepts_dict: Dict[str, str]) -> Self:
- the_dict: Dict[str, InputRequirement] = {
- var_name: InputRequirement(concept_code=concept_code) for var_name, concept_code in concepts_dict.items()
- }
- return cls(root=the_dict)
+ def make_from_blueprint(cls, domain: str, blueprint: Dict[str, InputRequirementBlueprint]) -> Self:
+ for var_name, input_requirement_blueprint in blueprint.items():
+ concept_code = ConceptCodeFactory.make_concept_code_from_str(domain=domain, concept_str=input_requirement_blueprint.concept_code)
+ blueprint[var_name].concept_code = concept_code
+ return cls(
+ root={
+ var_name: InputRequirement(
+ concept_code=input_requirement_blueprint.concept_code, multiplicity=input_requirement_blueprint.multiplicity
+ )
+ for var_name, input_requirement_blueprint in blueprint.items()
+ }
+ )
@property
def items(self) -> List[Tuple[str, InputRequirement]]:
diff --git a/pipelex/core/pipe_library.py b/pipelex/core/pipes/pipe_library.py
similarity index 95%
rename from pipelex/core/pipe_library.py
rename to pipelex/core/pipes/pipe_library.py
index ad05d4770..5df5f0419 100644
--- a/pipelex/core/pipe_library.py
+++ b/pipelex/core/pipes/pipe_library.py
@@ -7,8 +7,8 @@ from rich.table import Table
from typing_extensions import override
from pipelex import pretty_print
-from pipelex.core.pipe_abstract import PipeAbstract
-from pipelex.core.pipe_provider_abstract import PipeProviderAbstract
+from pipelex.core.pipes.pipe_abstract import PipeAbstract
+from pipelex.core.pipes.pipe_provider_abstract import PipeProviderAbstract
from pipelex.exceptions import ConceptError, ConceptLibraryConceptNotFoundError, PipeLibraryError, PipeLibraryPipeNotFoundError
from pipelex.hub import get_concept_provider
@@ -46,6 +46,10 @@ class PipeLibrary(RootModel[PipeLibraryRoot], PipeProviderAbstract):
raise PipeLibraryError(f"Pipe '{name}' already exists in the library")
self.root[pipe.code] = pipe
+ def add_pipes(self, pipes: List[PipeAbstract]):
+ for pipe in pipes:
+ self.add_new_pipe(pipe=pipe)
+
def add_or_update_pipe(self, pipe: PipeAbstract):
name = pipe.code
pipe.inputs.set_default_domain(domain=pipe.domain)
diff --git a/pipelex/core/pipe_output.py b/pipelex/core/pipes/pipe_output.py
similarity index 95%
rename from pipelex/core/pipe_output.py
rename to pipelex/core/pipes/pipe_output.py
index 3be2b90b2..1a5143d52 100644
--- a/pipelex/core/pipe_output.py
+++ b/pipelex/core/pipes/pipe_output.py
@@ -2,8 +2,9 @@ from typing import List, Type, TypeVar
from pydantic import BaseModel, Field
-from pipelex.core.stuff import Stuff
-from pipelex.core.stuff_content import (
+from pipelex.core.memory.working_memory import WorkingMemory
+from pipelex.core.stuffs.stuff import Stuff
+from pipelex.core.stuffs.stuff_content import (
HtmlContent,
ImageContent,
ListContent,
@@ -13,7 +14,6 @@ from pipelex.core.stuff_content import (
TextAndImagesContent,
TextContent,
)
-from pipelex.core.working_memory import WorkingMemory
from pipelex.pipeline.pipeline_models import SpecialPipelineId
diff --git a/pipelex/core/pipe_provider_abstract.py b/pipelex/core/pipes/pipe_provider_abstract.py
similarity index 92%
rename from pipelex/core/pipe_provider_abstract.py
rename to pipelex/core/pipes/pipe_provider_abstract.py
index 3f3665c42..ebeb52120 100644
--- a/pipelex/core/pipe_provider_abstract.py
+++ b/pipelex/core/pipes/pipe_provider_abstract.py
@@ -1,7 +1,7 @@
from abc import ABC, abstractmethod
from typing import Dict, List, Optional
-from pipelex.core.pipe_abstract import PipeAbstract
+from pipelex.core.pipes.pipe_abstract import PipeAbstract
PipeLibraryRoot = Dict[str, PipeAbstract]
diff --git a/pipelex/core/pipe_run_params.py b/pipelex/core/pipes/pipe_run_params.py
similarity index 98%
rename from pipelex/core/pipe_run_params.py
rename to pipelex/core/pipes/pipe_run_params.py
index 7c93ee3c2..35eb7872f 100644
--- a/pipelex/core/pipe_run_params.py
+++ b/pipelex/core/pipes/pipe_run_params.py
@@ -4,7 +4,7 @@ from pydantic import BaseModel, Field, field_validator
from typing_extensions import Self
from pipelex import log
-from pipelex.core.working_memory import BATCH_ITEM_STUFF_NAME, MAIN_STUFF_NAME
+from pipelex.core.memory.working_memory import BATCH_ITEM_STUFF_NAME, MAIN_STUFF_NAME
from pipelex.types import StrEnum
diff --git a/pipelex/core/pipe_run_params_factory.py b/pipelex/core/pipes/pipe_run_params_factory.py
similarity index 89%
rename from pipelex/core/pipe_run_params_factory.py
rename to pipelex/core/pipes/pipe_run_params_factory.py
index 79abb8f3e..65b006529 100644
--- a/pipelex/core/pipe_run_params_factory.py
+++ b/pipelex/core/pipes/pipe_run_params_factory.py
@@ -1,7 +1,7 @@
from typing import Any, Dict, Optional
from pipelex.config import get_config
-from pipelex.core.pipe_run_params import BatchParams, PipeOutputMultiplicity, PipeRunMode, PipeRunParams
+from pipelex.core.pipes.pipe_run_params import BatchParams, PipeOutputMultiplicity, PipeRunMode, PipeRunParams
class PipeRunParamsFactory:
diff --git a/pipelex/core/registry_models.py b/pipelex/core/registry_models.py
index c9b2bfd95..0b3d7843d 100644
--- a/pipelex/core/registry_models.py
+++ b/pipelex/core/registry_models.py
@@ -1,9 +1,9 @@
from typing import Any, ClassVar, List
-from pipelex.core.pipe_abstract import PipeAbstractType
-from pipelex.core.pipe_blueprint import PipeSpecificFactoryProtocol
-from pipelex.core.stuff import Stuff
-from pipelex.core.stuff_content import (
+from pipelex.core.pipes.pipe_abstract import PipeAbstractType
+from pipelex.core.pipes.pipe_factory import PipeFactoryProtocol
+from pipelex.core.stuffs.stuff import Stuff
+from pipelex.core.stuffs.stuff_content import (
DynamicContent,
HtmlContent,
ImageContent,
@@ -17,8 +17,7 @@ from pipelex.core.stuff_content import (
TextAndImagesContent,
TextContent,
)
-from pipelex.libraries.pipeline_blueprint import PipelineBlueprint
-from pipelex.libraries.pipelines.meta.pipeline_draft import PipelineDraft
+from pipelex.libraries.pipelines.meta.pipeline_draft import PipelexBundleBlueprint, PipelineDraft
from pipelex.pipe_controllers.pipe_batch import PipeBatch
from pipelex.pipe_controllers.pipe_batch_factory import PipeBatchFactory
from pipelex.pipe_controllers.pipe_condition import PipeCondition
@@ -53,7 +52,7 @@ class PipelexRegistryModels(RegistryModels):
PipeOcr,
]
- PIPE_OPERATORS_FACTORY: ClassVar[List[PipeSpecificFactoryProtocol[Any, Any]]] = [
+ PIPE_OPERATORS_FACTORY: ClassVar[List[PipeFactoryProtocol[Any, Any]]] = [
PipeFuncFactory,
PipeImgGenFactory,
PipeJinja2Factory,
@@ -68,7 +67,7 @@ class PipelexRegistryModels(RegistryModels):
PipeSequence,
]
- PIPE_CONTROLLERS_FACTORY: ClassVar[List[PipeSpecificFactoryProtocol[Any, Any]]] = [
+ PIPE_CONTROLLERS_FACTORY: ClassVar[List[PipeFactoryProtocol[Any, Any]]] = [
PipeBatchFactory,
PipeConditionFactory,
PipeParallelFactory,
@@ -88,7 +87,7 @@ class PipelexRegistryModels(RegistryModels):
PDFContent,
TextAndImagesContent,
PageContent,
- PipelineBlueprint,
+ PipelexBundleBlueprint,
PipelineDraft,
]
diff --git a/pipelex/core/stuff.py b/pipelex/core/stuffs/stuff.py
similarity index 97%
rename from pipelex/core/stuff.py
rename to pipelex/core/stuffs/stuff.py
index 208c51701..88659b56a 100644
--- a/pipelex/core/stuff.py
+++ b/pipelex/core/stuffs/stuff.py
@@ -4,9 +4,9 @@ from pydantic import ConfigDict, ValidationError
from typing_extensions import override
from pipelex import log
-from pipelex.core.concept import Concept
-from pipelex.core.stuff_artefact import StuffArtefact
-from pipelex.core.stuff_content import (
+from pipelex.core.concepts.concept import Concept
+from pipelex.core.stuffs.stuff_artefact import StuffArtefact
+from pipelex.core.stuffs.stuff_content import (
HtmlContent,
ImageContent,
ListContent,
diff --git a/pipelex/core/stuff_artefact.py b/pipelex/core/stuffs/stuff_artefact.py
similarity index 97%
rename from pipelex/core/stuff_artefact.py
rename to pipelex/core/stuffs/stuff_artefact.py
index 45b8d3b9f..e1cc2e771 100644
--- a/pipelex/core/stuff_artefact.py
+++ b/pipelex/core/stuffs/stuff_artefact.py
@@ -4,7 +4,7 @@ from jinja2.runtime import Context
from pydantic import RootModel
from typing_extensions import override
-from pipelex.core.stuff_content import StuffContent
+from pipelex.core.stuffs.stuff_content import StuffContent
from pipelex.exceptions import StuffArtefactError
from pipelex.tools.templating.jinja2_models import Jinja2ContextKey, Jinja2TaggableAbstract
from pipelex.tools.templating.templating_models import TextFormat
diff --git a/pipelex/core/stuff_content.py b/pipelex/core/stuffs/stuff_content.py
similarity index 100%
rename from pipelex/core/stuff_content.py
rename to pipelex/core/stuffs/stuff_content.py
diff --git a/pipelex/core/stuff_factory.py b/pipelex/core/stuffs/stuff_factory.py
similarity index 97%
rename from pipelex/core/stuff_factory.py
rename to pipelex/core/stuffs/stuff_factory.py
index f5e663611..4fb383025 100644
--- a/pipelex/core/stuff_factory.py
+++ b/pipelex/core/stuffs/stuff_factory.py
@@ -4,12 +4,12 @@ import shortuuid
from pydantic import BaseModel, ValidationError
from pipelex.client.protocol import StuffContentOrData
-from pipelex.core.concept import Concept
-from pipelex.core.concept_code_factory import ConceptCodeFactory
-from pipelex.core.concept_factory import ConceptFactory
-from pipelex.core.concept_native import NativeConcept, NativeConceptClass
-from pipelex.core.stuff import Stuff
-from pipelex.core.stuff_content import (
+from pipelex.core.concepts.concept import Concept
+from pipelex.core.concepts.concept_code_factory import ConceptCodeFactory
+from pipelex.core.concepts.concept_factory import ConceptFactory
+from pipelex.core.concepts.concept_native import NativeConcept, NativeConceptClass
+from pipelex.core.stuffs.stuff import Stuff
+from pipelex.core.stuffs.stuff_content import (
ListContent,
StuffContent,
StuffContentInitableFromStr,
diff --git a/pipelex/create/build_blueprint.py b/pipelex/create/build_blueprint.py
index feb0b3573..867fe91ce 100644
--- a/pipelex/create/build_blueprint.py
+++ b/pipelex/create/build_blueprint.py
@@ -3,10 +3,9 @@ from __future__ import annotations
from typing import Optional
from pipelex import log, pretty_print
+from pipelex.core.interpreter import PipelexInterpreter
from pipelex.create.helpers import get_support_file
-from pipelex.create.pipeline_toml import save_pipeline_blueprint_toml_to_path
-from pipelex.create.validate_blueprint import load_pipes_from_generated_blueprint, validate_blueprint
-from pipelex.libraries.pipeline_blueprint import PipelineBlueprint
+from pipelex.libraries.pipelines.meta.pipeline_draft import PipelexBundleBlueprintStuff
from pipelex.pipeline.execute import execute_pipeline
from pipelex.tools.misc.file_utils import save_text_to_path
from pipelex.tools.misc.json_utils import save_as_json_to_path
@@ -27,30 +26,20 @@ async def do_build_blueprint(
"requirements": requirements,
"draft_pipeline_rules": get_support_file(subpath="create/draft_pipelines.md"),
"build_pipeline_rules": get_support_file(subpath="create/build_pipelines.md"),
+ "create_structured_output_rules": get_support_file(subpath="create/structures.md"),
},
)
- blueprint = pipe_output.main_stuff_as(content_type=PipelineBlueprint)
- pretty_print(blueprint, title="Pipeline Blueprint")
+ pretty_print(pipe_output, title="Pipe Output")
+
+ blueprint = pipe_output.main_stuff_as(content_type=PipelexBundleBlueprintStuff)
pipeline_draft = pipe_output.working_memory.get_stuff_as_str(name="pipeline_draft")
- pretty_print(pipeline_draft, title="Pipeline Draft")
# Save or display result
output_path_base = output_path or "pipelex/libraries/pipelines/temp/gen_blueprint"
draft_path = f"{output_path_base}_draft.md"
- save_text_to_path(text=pipeline_draft, path=draft_path)
+ save_text_to_path(text=pipeline_draft, path=draft_path, create_directory=True)
rough_toml_path = f"{output_path_base}_rough.toml"
- save_pipeline_blueprint_toml_to_path(blueprint=blueprint, path=rough_toml_path)
+ save_text_to_path(text=PipelexInterpreter.make_toml_content(blueprint=blueprint), path=rough_toml_path)
rough_json_path = f"{output_path_base}_rough.json"
save_as_json_to_path(object_to_save=blueprint, path=rough_json_path)
log.info(f"✅ Rough blueprint saved to '{output_path_base}'")
-
- load_pipes_from_generated_blueprint(blueprint=blueprint)
- log.info("✅ Pipes loaded from blueprint")
-
- if validate:
- await validate_blueprint(blueprint=blueprint, is_error_fixing_enabled=True)
- fixed_toml_path = f"{output_path_base}_fixed.toml"
- save_pipeline_blueprint_toml_to_path(blueprint=blueprint, path=fixed_toml_path)
- fixed_json_path = f"{output_path_base}_fixed.json"
- save_as_json_to_path(object_to_save=blueprint, path=fixed_json_path)
- log.info(f"✅ Fixed blueprint saved to '{output_path_base}'")
diff --git a/pipelex/create/draft_pipeline.py b/pipelex/create/draft_pipeline.py
index f87dd736e..7b10aec56 100644
--- a/pipelex/create/draft_pipeline.py
+++ b/pipelex/create/draft_pipeline.py
@@ -6,10 +6,10 @@ import typer
from pipelex import pretty_print
from pipelex.create.helpers import get_support_file
-from pipelex.create.pipeline_toml import dict_to_toml, save_toml_to_path
from pipelex.libraries.pipelines.meta.pipeline_draft import PipelineDraft
from pipelex.pipeline.execute import execute_pipeline
from pipelex.tools.misc.file_utils import save_text_to_path
+from pipelex.tools.misc.toml_utils import dict_to_toml, save_toml_to_path
async def do_draft_pipeline_text(
diff --git a/pipelex/create/pipeline_toml.py b/pipelex/create/pipeline_toml.py
deleted file mode 100644
index 9e6bd8a5c..000000000
--- a/pipelex/create/pipeline_toml.py
+++ /dev/null
@@ -1,138 +0,0 @@
-from __future__ import annotations
-
-from typing import Any, Dict, List, Mapping, cast
-
-import tomlkit
-from tomlkit import array, document, inline_table, table
-from tomlkit import string as tk_string
-
-from pipelex.libraries.pipeline_blueprint import PipelineBlueprint
-from pipelex.tools.misc.file_utils import save_text_to_path
-from pipelex.tools.misc.json_utils import remove_none_values_from_dict
-from pipelex.tools.misc.toml_utils import clean_trailing_whitespace
-
-
-def make_toml_string(
- text: str,
- prefer_literal: bool = False,
- force_multiline: bool = False,
- ensure_trailing_newline: bool = True,
- ensure_leading_blank_line_in_value: bool = False,
-):
- """
- Build a tomlkit string node.
- - If `force_multiline` or the text contains '\\n', we emit a triple-quoted multiline string.
- - When multiline, `ensure_trailing_newline` puts the closing quotes on their own line.
- - When multiline, `ensure_leading_blank_line_in_value` inserts a real blank line at the start of the value.
- """
- needs_multiline = force_multiline or ("\n" in text)
- normalized = text
-
- if needs_multiline:
- if ensure_leading_blank_line_in_value and not normalized.startswith("\n"):
- normalized = "\n" + normalized
- if ensure_trailing_newline and not normalized.endswith("\n"):
- normalized = normalized + "\n"
-
- use_literal = prefer_literal and ("'''" not in normalized)
- return tk_string(normalized, multiline=needs_multiline, literal=use_literal)
-
-
-def _convert_to_inline(value: Any):
- """Recursively convert Python values; dicts -> inline tables; lists kept as arrays."""
- if isinstance(value, Mapping):
- value = cast(Mapping[str, Any], value)
- inline_table_obj = inline_table()
- for key, value_item in value.items():
- inline_table_obj[key] = _convert_to_inline(value_item)
- return inline_table_obj
-
- if isinstance(value, list):
- value = cast(List[Any], value)
- array_obj = array()
- array_obj.multiline(True) # set to False for single-line arrays
- for element in value:
- if isinstance(element, Mapping):
- element = cast(Mapping[str, Any], element)
- inline_element = inline_table()
- for inner_key, inner_value in element.items():
- inline_element[inner_key] = _convert_to_inline(inner_value)
- array_obj.append(inline_element) # pyright: ignore[reportUnknownMemberType]
- else:
- array_obj.append(_convert_to_inline(element)) # pyright: ignore[reportUnknownMemberType]
- return array_obj
-
- if isinstance(value, str):
- # Triple quotes if needed (or forced); closing quotes on their own line.
- return make_toml_string(
- value,
- prefer_literal=False, # flip to True for '''...'''
- force_multiline=False, # flip to True to force """...""" even without \n
- ensure_trailing_newline=True, # keep closing """ on its own line
- ensure_leading_blank_line_in_value=False, # flip to True to keep a blank first line
- )
- return value
-
-
-def _convert_mapping_to_table(mapping: Mapping[str, Any]) -> Any:
- """Convert a mapping into a TOML Table where any nested mappings (third level+)
- are converted to inline tables.
-
- This creates a second-level standard table, and only uses inline tables for
- third level and deeper structures.
- """
- tbl = table()
- for field_key, field_value in mapping.items():
- if isinstance(field_value, Mapping):
- # Third-level mapping -> inline table
- tbl.add(field_key, _convert_to_inline(field_value))
- else:
- tbl.add(field_key, _convert_to_inline(field_value))
- return tbl
-
-
-def dict_to_toml(data: Mapping[str, Any]) -> str:
- """Top-level keys become tables; second-level mappings become tables; inline tables start at third level."""
- data = remove_none_values_from_dict(data=data)
- document_root = document()
- for section_key, section_value in data.items():
- if isinstance(section_value, Mapping):
- section_value = cast(Mapping[str, Any], section_value)
- table_obj = table()
- for field_key, field_value in section_value.items():
- if isinstance(field_value, Mapping):
- # Second level mapping -> standard table [section.field]
- sub_table = _convert_mapping_to_table(cast(Mapping[str, Any], field_value))
- table_obj.add(field_key, sub_table)
- else:
- table_obj.add(field_key, _convert_to_inline(field_value))
- document_root.add(section_key, table_obj) # `[section_key]` section
- else:
- document_root.add(section_key, _convert_to_inline(section_value))
- dumped_content = tomlkit.dumps(document_root) # pyright: ignore[reportUnknownMemberType]
- return dumped_content
-
-
-def pipeline_blueprint_to_toml(blueprint: PipelineBlueprint) -> str:
- blueprint_dict = blueprint.smart_dump()
- return dict_to_toml(data=blueprint_dict)
-
-
-def save_toml_to_path(data: Dict[str, Any], path: str) -> None:
- """Save dictionary as TOML to file path.
-
- Args:
- data: Dictionary to save as TOML
- path: Path where to save the TOML file
- """
- # data_cleaned = remove_none_values_from_dict(data=data)
- data_cleaned = data
- with open(path, "w", encoding="utf-8") as file:
- toml_content: str = dict_to_toml(data=data_cleaned)
- cleaned_content = clean_trailing_whitespace(toml_content)
- file.write(cleaned_content)
-
-
-def save_pipeline_blueprint_toml_to_path(blueprint: PipelineBlueprint, path: str) -> None:
- pipeline_blueprint_toml = pipeline_blueprint_to_toml(blueprint=blueprint)
- save_text_to_path(text=pipeline_blueprint_toml, path=path)
diff --git a/pipelex/create/structured_output_generator.py b/pipelex/create/structured_output_generator.py
index cec834dcd..3c4bd0023 100644
--- a/pipelex/create/structured_output_generator.py
+++ b/pipelex/create/structured_output_generator.py
@@ -2,43 +2,21 @@
from __future__ import annotations
-from typing import Any, ClassVar, Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional, Union
import tomlkit
-from pipelex.types import StrEnum
-
-
-class FieldType(StrEnum):
- """Supported field types for structured outputs."""
-
- TEXT = "text"
- NUMBER = "number"
- INTEGER = "integer"
- BOOLEAN = "boolean"
- LIST = "list"
- DICT = "dict"
- ENUM = "enum" # For referencing defined enums
+from pipelex.core.concepts.concept_blueprint import ConceptStructureBlueprintFieldType
class StructuredOutputGenerator:
"""Generate Pydantic BaseModel classes from TOML structured output definitions."""
- # Map high-level type names to Python types
- TYPE_MAPPING: ClassVar[Dict[FieldType, str]] = {
- FieldType.TEXT: "str",
- FieldType.NUMBER: "float",
- FieldType.INTEGER: "int",
- FieldType.BOOLEAN: "bool",
- FieldType.LIST: "List",
- FieldType.DICT: "Dict",
- }
-
def __init__(self):
self.imports = {
"from typing import Optional, List, Dict, Any, Literal",
"from enum import Enum",
- "from pipelex.core.stuff_content import StructuredContent",
+ "from pipelex.core.stuffs.stuff_content import StructuredContent",
"from pydantic import Field",
}
self.enum_definitions: Dict[str, Dict[str, Any]] = {} # Store enum definitions
@@ -170,9 +148,9 @@ class StructuredOutputGenerator:
"""
# Handle simple string definitions (just the definition text)
if isinstance(field_def, str):
- field_def = {"type": FieldType.TEXT, "definition": field_def}
+ field_def = {"type": ConceptStructureBlueprintFieldType.TEXT, "definition": field_def}
- field_type = field_def.get("type", FieldType.TEXT)
+ field_type = field_def.get("type", ConceptStructureBlueprintFieldType.TEXT)
definition = field_def.get("definition", f"{field_name} field")
required = field_def.get("required", False)
default_value = field_def.get("default")
@@ -225,7 +203,7 @@ class StructuredOutputGenerator:
# Convert string to FieldType if needed
if isinstance(field_type, str):
try:
- field_type_enum = FieldType(field_type)
+ field_type_enum = ConceptStructureBlueprintFieldType(field_type)
except ValueError:
# Unknown type, assume it's a custom type or class reference
return field_type
@@ -233,15 +211,15 @@ class StructuredOutputGenerator:
# Use match/case for type handling
match field_type:
- case FieldType.TEXT:
+ case ConceptStructureBlueprintFieldType.TEXT:
return "str"
- case FieldType.NUMBER:
+ case ConceptStructureBlueprintFieldType.NUMBER:
return "float"
- case FieldType.INTEGER:
+ case ConceptStructureBlueprintFieldType.INTEGER:
return "int"
- case FieldType.BOOLEAN:
+ case ConceptStructureBlueprintFieldType.BOOLEAN:
return "bool"
- case FieldType.LIST:
+ case ConceptStructureBlueprintFieldType.LIST:
item_type = field_def.get("item_type", "Any")
# Check if item_type is an enum reference
if isinstance(item_type, str) and item_type in self.enum_definitions:
@@ -249,25 +227,25 @@ class StructuredOutputGenerator:
# Recursively handle item types
if isinstance(item_type, str):
try:
- item_type_enum = FieldType(item_type)
+ item_type_enum = ConceptStructureBlueprintFieldType(item_type)
item_type = self._get_python_type(item_type_enum, {})
except ValueError:
# Keep as string if not a known FieldType
pass
return f"List[{item_type}]"
- case FieldType.DICT:
+ case ConceptStructureBlueprintFieldType.DICT:
key_type = field_def.get("key_type", "str")
value_type = field_def.get("value_type", "Any")
# Recursively handle key and value types
if isinstance(key_type, str):
try:
- key_type_enum = FieldType(key_type)
+ key_type_enum = ConceptStructureBlueprintFieldType(key_type)
key_type = self._get_python_type(key_type_enum, {})
except ValueError:
pass
if isinstance(value_type, str):
try:
- value_type_enum = FieldType(value_type)
+ value_type_enum = ConceptStructureBlueprintFieldType(value_type)
value_type = self._get_python_type(value_type_enum, {})
except ValueError:
pass
diff --git a/pipelex/create/validate_blueprint.py b/pipelex/create/validate_blueprint.py
deleted file mode 100644
index 39fae3dbc..000000000
--- a/pipelex/create/validate_blueprint.py
+++ /dev/null
@@ -1,105 +0,0 @@
-from __future__ import annotations
-
-from typing import Any, Dict, cast
-
-from pipelex import log
-from pipelex.exceptions import PipeDefinitionError, PipelexCLIError, StaticValidationError, StaticValidationErrorType
-from pipelex.hub import get_library_manager, get_required_pipe
-from pipelex.libraries.library_manager import LibraryManager
-from pipelex.libraries.pipeline_blueprint import PipelineBlueprint
-from pipelex.pipe_controllers.pipe_controller import PipeController
-from pipelex.pipe_works.pipe_dry import dry_run_pipe_codes
-from pipelex.tools.misc.dict_utils import insert_before
-
-
-async def validate_blueprint(blueprint: PipelineBlueprint, is_error_fixing_enabled: bool) -> None:
- # Load pipes from the blueprint
- log.dev("Loading pipes from blueprint")
- try:
- load_pipes_from_generated_blueprint(blueprint=blueprint, updates_allowed=True)
- except PipeDefinitionError as exc:
- raise PipelexCLIError(f"Failed to load pipes from blueprint for domain '{blueprint.desc}': {exc}") from exc
-
- try:
- await validate_loaded_blueprint(blueprint=blueprint)
- except StaticValidationError as static_validation_error:
- match static_validation_error.error_type:
- case StaticValidationErrorType.MISSING_INPUT_VARIABLE:
- if is_error_fixing_enabled:
- pipe_code = static_validation_error.pipe_code
- if not pipe_code:
- raise PipelexCLIError(f"No pipe code found for static validation error: {static_validation_error}")
- pipe = get_required_pipe(pipe_code=pipe_code)
- if isinstance(pipe, PipeController):
- variable_names = static_validation_error.variable_names
- required_concept_codes = static_validation_error.required_concept_codes
- if not variable_names:
- raise PipelexCLIError(f"No variable names found for static validation error: {static_validation_error}")
- if not required_concept_codes:
- raise PipelexCLIError(f"No required concept codes found for static validation error: {static_validation_error}")
- log.error(f"❌ Missing: {variable_names} / concepts: {required_concept_codes}")
- new_inputs: Dict[str, Any] = blueprint.pipe[pipe_code].get("inputs") or {}
- for variable_name, concept_code in zip(variable_names, required_concept_codes):
- new_inputs[variable_name] = concept_code
- log.info(f"✅ Added: {variable_name} / {concept_code}")
- if "inputs" in blueprint.pipe[pipe_code]:
- blueprint.pipe[pipe_code]["inputs"] = new_inputs
- else:
- blueprint.pipe[pipe_code] = insert_before(
- blueprint.pipe[pipe_code],
- "output",
- "inputs",
- new_inputs,
- )
- log.dev(f"✅ Fixed missing inputs for pipe '{pipe_code}'")
- await validate_blueprint(blueprint=blueprint, is_error_fixing_enabled=True)
- else:
- raise PipelexCLIError(f"Unexpected validation error:\n{static_validation_error}") from static_validation_error
- else:
- raise static_validation_error
- case (
- StaticValidationErrorType.EXTRANEOUS_INPUT_VARIABLE
- | StaticValidationErrorType.INADEQUATE_INPUT_CONCEPT
- | StaticValidationErrorType.TOO_MANY_CANDIDATE_INPUTS
- ):
- raise static_validation_error
- log.info("✅ Validation passed")
-
-
-async def validate_loaded_blueprint(blueprint: PipelineBlueprint) -> None:
- # Run validation
- log.dev("Validating blueprint")
- get_library_manager().validate_libraries()
- generated_pipe_codes = list(blueprint.pipe.keys())
- if not generated_pipe_codes:
- raise PipelexCLIError("No pipe found in blueprint to validate")
- await dry_run_pipe_codes(pipe_codes=generated_pipe_codes)
- log.info(f"✅ All pipes validated successfully: {generated_pipe_codes}")
-
-
-def load_pipes_from_generated_blueprint(blueprint: PipelineBlueprint, updates_allowed: bool = False) -> None:
- """Instantiate and register all pipes from a generated PipelineBlueprint.
-
- This constructs the appropriate typed PipeBlueprint for each pipe entry and
- uses LibraryManager.load_pipe_from_blueprint to create the Pipe, then adds it
- to the in-memory PipeLibrary so that validation can run against them.
- """
- library_manager = cast(LibraryManager, get_library_manager())
- domain_code = blueprint.domain
-
- for pipe_code, details in blueprint.pipe.items():
- details_dict: Dict[str, Any] = details.copy()
-
- pipe_blueprint = LibraryManager.make_pipe_blueprint_from_details(
- domain_code=domain_code,
- details_dict=details_dict,
- )
- # Create pipe and register in library
- pipe = LibraryManager.load_pipe_from_blueprint(
- pipe_code=pipe_code,
- pipe_blueprint=pipe_blueprint,
- )
- if updates_allowed:
- library_manager.pipe_library.add_or_update_pipe(pipe=pipe)
- else:
- library_manager.pipe_library.add_new_pipe(pipe=pipe)
diff --git a/pipelex/hub.py b/pipelex/hub.py
index b3d965f68..55e36ac81 100644
--- a/pipelex/hub.py
+++ b/pipelex/hub.py
@@ -1,5 +1,4 @@
-from collections import defaultdict
-from typing import ClassVar, Dict, List, Optional, Type
+from typing import ClassVar, List, Optional, Type
from kajson.class_registry_abstract import ClassRegistryAbstract
@@ -15,12 +14,12 @@ from pipelex.cogt.llm.llm_models.llm_model_provider_abstract import (
)
from pipelex.cogt.llm.llm_worker_abstract import LLMWorkerAbstract
from pipelex.cogt.ocr.ocr_worker_abstract import OcrWorkerAbstract
-from pipelex.core.concept import Concept
-from pipelex.core.concept_provider_abstract import ConceptProviderAbstract
-from pipelex.core.domain import Domain
-from pipelex.core.domain_provider_abstract import DomainProviderAbstract
-from pipelex.core.pipe_abstract import PipeAbstract
-from pipelex.core.pipe_provider_abstract import PipeProviderAbstract
+from pipelex.core.concepts.concept import Concept
+from pipelex.core.concepts.concept_provider_abstract import ConceptProviderAbstract
+from pipelex.core.domains.domain import Domain
+from pipelex.core.domains.domain_provider_abstract import DomainProviderAbstract
+from pipelex.core.pipes.pipe_abstract import PipeAbstract
+from pipelex.core.pipes.pipe_provider_abstract import PipeProviderAbstract
from pipelex.libraries.library_manager_abstract import LibraryManagerAbstract
from pipelex.pipe_works.pipe_router_protocol import PipeRouterProtocol
from pipelex.pipeline.activity.activity_manager_protocol import ActivityManagerProtocol
@@ -391,10 +390,6 @@ def get_secret(secret_id: str) -> str:
return get_secrets_provider().get_secret(secret_id=secret_id)
-def get_domain_provider() -> DomainProviderAbstract:
- return get_pipelex_hub().get_required_domain_provider()
-
-
def get_domains(excluded_domains: Optional[List[str]] = None) -> List[Domain]:
domains = get_pipelex_hub().get_required_domain_provider().get_domains()
if excluded_domains:
@@ -417,18 +412,6 @@ def get_pipe_provider() -> PipeProviderAbstract:
return get_pipelex_hub().get_required_pipe_provider()
-def get_pipes_by_domain(
- excluded_domains: Optional[List[str]] = None,
-) -> Dict[str, List[str]]:
- pipes = get_pipe_provider().get_pipes()
- pipes_by_domain: Dict[str, List[str]] = defaultdict(list)
- for pipe in pipes:
- if excluded_domains and pipe.domain in excluded_domains:
- continue
- pipes_by_domain[pipe.domain].append(pipe.code)
- return pipes_by_domain
-
-
def get_required_pipe(pipe_code: str) -> PipeAbstract:
return get_pipelex_hub().get_required_pipe_provider().get_required_pipe(pipe_code=pipe_code)
diff --git a/pipelex/libraries/library_config.py b/pipelex/libraries/library_config.py
index 4af9468e1..eb10ec107 100644
--- a/pipelex/libraries/library_config.py
+++ b/pipelex/libraries/library_config.py
@@ -8,55 +8,55 @@ PIPELEX_LIBRARIES_PATH = "libraries"
class LibraryConfig(ConfigModel):
package_name: ClassVar[str] = "pipelex"
- config_folder_path: str = "pipelex_libraries"
+ config_dir_path: str = "pipelex_libraries"
@property
- def config_file(self) -> str:
- return f"{self.config_folder_path}/config.toml"
+ def config_file_path(self) -> str:
+ return f"{self.config_dir_path}/config.toml"
@property
- def pipelines_path(self) -> str:
- return f"{self.config_folder_path}/pipelines"
+ def pipelines_dir_path(self) -> str:
+ return f"{self.config_dir_path}/pipelines"
@property
- def base_pipelines_path(self) -> str:
- return f"{self.config_folder_path}/pipelines/base_library"
+ def base_pipelines_dir_path(self) -> str:
+ return f"{self.config_dir_path}/pipelines/base_library"
@property
- def llm_integrations_path(self) -> str:
- return f"{self.config_folder_path}/llm_integrations"
+ def llm_integrations_dir_path(self) -> str:
+ return f"{self.config_dir_path}/llm_integrations"
@property
- def llm_deck_path(self) -> str:
- return f"{self.config_folder_path}/llm_deck"
+ def llm_deck_dir_path(self) -> str:
+ return f"{self.config_dir_path}/llm_deck"
@property
- def templates_path(self) -> str:
- return f"{self.config_folder_path}/templates"
+ def templates_dir_path(self) -> str:
+ return f"{self.config_dir_path}/templates"
@property
- def plugins_path(self) -> str:
- return f"{self.config_folder_path}/plugins"
+ def plugins_dir_path(self) -> str:
+ return f"{self.config_dir_path}/plugins"
@property
- def test_pipelines_path(self) -> str:
+ def test_pipelines_dir_path(self) -> str:
return "tests/test_pipelines"
@property
- def failing_pipelines_path(self) -> str:
- return "tests/test_pipelines/failing_pipelines.toml"
+ def failing_pipelines_file_paths(self) -> List[str]:
+ return ["tests/test_pipelines/failing_pipelines.toml"]
def get_llm_deck_paths(self) -> List[str]:
- llm_deck_paths = [str(path) for path in find_files_in_dir(dir_path=self.llm_deck_path, pattern="*.toml", is_recursive=True)]
+ llm_deck_paths = [str(path) for path in find_files_in_dir(dir_path=self.llm_deck_dir_path, pattern="*.toml", is_recursive=True)]
llm_deck_paths.sort()
return llm_deck_paths
def get_templates_paths(self) -> List[str]:
- return [str(path) for path in find_files_in_dir(dir_path=self.templates_path, pattern="*.toml", is_recursive=True)]
+ return [str(path) for path in find_files_in_dir(dir_path=self.templates_dir_path, pattern="*.toml", is_recursive=True)]
def get_default_plugin_config_path(self) -> str:
"""Get the default plugin config path."""
- return f"{self.plugins_path}/plugin_config.toml"
+ return f"{self.plugins_dir_path}/plugin_config.toml"
def export_libraries(self, overwrite: bool = False) -> None:
"""Duplicate pipelex libraries files in the client project, preserving directory structure."""
@@ -64,32 +64,32 @@ class LibraryConfig(ConfigModel):
copy_folder_from_package(
package_name=self.package_name,
folder_path_in_package=f"{PIPELEX_LIBRARIES_PATH}/pipelines",
- target_dir=self.base_pipelines_path,
+ target_dir=self.base_pipelines_dir_path,
overwrite=overwrite,
)
copy_file_from_package(
package_name=self.package_name,
file_path_in_package=f"{PIPELEX_LIBRARIES_PATH}/__init__.py",
- target_path=f"{self.config_folder_path}/__init__.py",
+ target_path=f"{self.config_dir_path}/__init__.py",
overwrite=overwrite,
)
copy_file_from_package(
package_name=self.package_name,
file_path_in_package=f"{PIPELEX_LIBRARIES_PATH}/pipelines/__init__.py",
- target_path=f"{self.pipelines_path}/__init__.py",
+ target_path=f"{self.pipelines_dir_path}/__init__.py",
overwrite=overwrite,
)
copy_file_from_package(
package_name=self.package_name,
file_path_in_package=f"{PIPELEX_LIBRARIES_PATH}/pipelines/__init__.py",
- target_path=f"{self.base_pipelines_path}/__init__.py",
+ target_path=f"{self.base_pipelines_dir_path}/__init__.py",
overwrite=overwrite,
)
# llm_integrations
copy_folder_from_package(
package_name=self.package_name,
folder_path_in_package=f"{PIPELEX_LIBRARIES_PATH}/llm_integrations",
- target_dir=self.llm_integrations_path,
+ target_dir=self.llm_integrations_dir_path,
overwrite=overwrite,
)
@@ -97,7 +97,7 @@ class LibraryConfig(ConfigModel):
copy_folder_from_package(
package_name=self.package_name,
folder_path_in_package=f"{PIPELEX_LIBRARIES_PATH}/llm_deck",
- target_dir=self.llm_deck_path,
+ target_dir=self.llm_deck_dir_path,
overwrite=overwrite,
non_overwrite_files=["overrides.toml"],
)
@@ -106,7 +106,7 @@ class LibraryConfig(ConfigModel):
copy_folder_from_package(
package_name=self.package_name,
folder_path_in_package=f"{PIPELEX_LIBRARIES_PATH}/templates",
- target_dir=self.templates_path,
+ target_dir=self.templates_dir_path,
overwrite=overwrite,
)
@@ -114,6 +114,6 @@ class LibraryConfig(ConfigModel):
copy_folder_from_package(
package_name=self.package_name,
folder_path_in_package=f"{PIPELEX_LIBRARIES_PATH}/plugins",
- target_dir=self.plugins_path,
+ target_dir=self.plugins_dir_path,
overwrite=overwrite,
)
diff --git a/pipelex/libraries/library_manager.py b/pipelex/libraries/library_manager.py
index e3fd2435a..a72e32048 100644
--- a/pipelex/libraries/library_manager.py
+++ b/pipelex/libraries/library_manager.py
@@ -1,44 +1,32 @@
-import importlib
import os
from pathlib import Path
from typing import Any, ClassVar, Dict, List, Optional, Type
-from kajson.exceptions import ClassRegistryInheritanceError, ClassRegistryNotFoundError
-from kajson.kajson_manager import KajsonManager
-from pydantic import ValidationError
from typing_extensions import override
from pipelex import log
from pipelex.cogt.llm.llm_models.llm_deck import LLMDeck
from pipelex.config import get_config
-from pipelex.core.concept_factory import ConceptFactory
-from pipelex.core.concept_library import ConceptLibrary
-from pipelex.core.domain import Domain
-from pipelex.core.domain_library import DomainLibrary
-from pipelex.core.pipe_abstract import PipeAbstract
-from pipelex.core.pipe_blueprint import PipeBlueprint, PipeSpecificFactoryProtocol
-from pipelex.core.pipe_library import PipeLibrary
+from pipelex.core.bundles.pipelex_bundle import PipelexBundle
+from pipelex.core.bundles.pipelex_bundle_factory import PipelexBundleFactory
+from pipelex.core.concepts.concept_factory import ConceptFactory
+from pipelex.core.concepts.concept_library import ConceptLibrary
+from pipelex.core.domains.domain_library import DomainLibrary
+from pipelex.core.interpreter import PipelexInterpreter
+from pipelex.core.pipes.pipe_abstract import PipeAbstract
+from pipelex.core.pipes.pipe_library import PipeLibrary
from pipelex.exceptions import (
ConceptLibraryError,
LibraryError,
- PipeFactoryError,
PipeLibraryError,
- StaticValidationError,
)
from pipelex.libraries.library_config import LibraryConfig
from pipelex.libraries.library_manager_abstract import LibraryManagerAbstract
-from pipelex.libraries.pipeline_blueprint import (
- ConceptBlueprintError,
- PipeBlueprintError,
- PipelineBlueprint,
- PipelineBlueprintValidationError,
-)
from pipelex.tools.class_registry_utils import ClassRegistryUtils
from pipelex.tools.misc.file_utils import find_files_in_dir
from pipelex.tools.misc.json_utils import deep_update
from pipelex.tools.misc.toml_utils import TOMLValidationError, load_toml_from_path, validate_toml_file
from pipelex.tools.runtime_manager import runtime_manager
-from pipelex.tools.typing.pydantic_utils import format_pydantic_validation_error
from pipelex.types import StrEnum
@@ -67,210 +55,24 @@ class LibraryManager(LibraryManagerAbstract):
"system_prompt_jto_structure",
"prompt_template_to_structure",
]
- domain_library: DomainLibrary
- concept_library: ConceptLibrary
- pipe_library: PipeLibrary
- llm_deck: Optional[LLMDeck] = None
- library_config: ClassVar[LibraryConfig]
-
- @classmethod
- def make_empty(cls, config_folder_path: str) -> "LibraryManager":
- cls.domain_library = DomainLibrary.make_empty()
- cls.concept_library = ConceptLibrary.make_empty()
- cls.pipe_library = PipeLibrary.make_empty()
- cls.library_config = LibraryConfig(config_folder_path=config_folder_path)
- return cls()
-
- @classmethod
- def make(
- cls, domain_library: DomainLibrary, concept_library: ConceptLibrary, pipe_library: PipeLibrary, config_folder_path: str
- ) -> "LibraryManager":
- cls.domain_library = domain_library
- cls.concept_library = concept_library
- cls.pipe_library = pipe_library
- cls.library_config = LibraryConfig(config_folder_path=config_folder_path)
- return cls()
-
- @override
- def get_plugin_config_path(self) -> str:
- return self.library_config.get_default_plugin_config_path()
-
- @override
- def setup(self) -> None:
- pass
-
- @override
- def teardown(self) -> None:
- self.llm_deck = None
- self.pipe_library.teardown()
- self.concept_library.teardown()
- self.domain_library.teardown()
-
- def libraries_paths(self) -> List[str]:
- library_paths = [self.library_config.pipelines_path]
- if runtime_manager.is_unit_testing:
- log.debug("Registering test pipeline structures for unit testing")
- library_paths += [self.library_config.test_pipelines_path]
- return library_paths
-
- def load_failure_modes(self):
- failing_pipelines_path = get_config().pipelex.library_config.failing_pipelines_path
- self.load_combo_libraries(library_paths=[Path(failing_pipelines_path)])
-
- def load_libraries(self):
- log.debug("LibraryManager loading separate libraries")
- library_paths = self.libraries_paths()
- # self._validate_toml_files()
- for library_path in library_paths:
- ClassRegistryUtils.register_classes_in_folder(
- folder_path=library_path,
- )
-
- native_concepts = ConceptFactory.list_native_concepts()
- self.concept_library.add_concepts(concepts=native_concepts)
-
- toml_file_paths = self.list_toml_files_from_path(library_paths=library_paths)
- # remove failing_pipelines_path from the list
- failing_pipelines_path = get_config().pipelex.library_config.failing_pipelines_path
- toml_file_paths = [path for path in toml_file_paths if path != Path(failing_pipelines_path)]
- self.load_combo_libraries(library_paths=toml_file_paths)
-
- def load_deck(self) -> LLMDeck:
- llm_deck_paths = self.library_config.get_llm_deck_paths()
- full_llm_deck_dict: Dict[str, Any] = {}
- if not llm_deck_paths:
- raise LLMDeckNotFoundError("No LLM deck paths found. Please run `pipelex init-libraries` to create it.")
-
- for llm_deck_path in llm_deck_paths:
- if not os.path.exists(llm_deck_path):
- raise LLMDeckNotFoundError(f"LLM deck path `{llm_deck_path}` not found. Please run `pipelex init-libraries` to create it.")
- try:
- llm_deck_dict = load_toml_from_path(path=llm_deck_path)
- log.debug(f"Loaded LLM deck from {llm_deck_path}")
- deep_update(full_llm_deck_dict, llm_deck_dict)
- except Exception as exc:
- log.error(f"Failed to load LLM deck file '{llm_deck_path}': {exc}")
- raise
-
- self.llm_deck = LLMDeck.model_validate(full_llm_deck_dict)
- return self.llm_deck
-
- def list_toml_files_from_path(self, library_paths: List[str]) -> List[Path]:
- toml_file_paths: List[Path] = []
- for libraries_path in library_paths:
- # Use the existing utility function specifically for TOML files
- found_file_paths = find_files_in_dir(
- dir_path=libraries_path,
- pattern="*.toml",
- is_recursive=True,
- )
- log.debug(f"Searching for TOML files in {libraries_path}, found '{found_file_paths}'")
- if not found_file_paths:
- log.warning(f"No TOML files found in library path: {libraries_path}")
- toml_file_paths.extend(found_file_paths)
- return toml_file_paths
- @override
- def load_combo_libraries(self, library_paths: List[Path]):
- log.debug("LibraryManager loading combo libraries")
- # Find all .toml files in the directories and their subdirectories
-
- # First pass: load all domains
- for toml_path in library_paths:
- blueprint = self._load_blueprint_from_file(toml_path)
- domain = Domain(
- code=blueprint.domain,
- definition=blueprint.definition,
- system_prompt=blueprint.system_prompt,
- system_prompt_to_structure=blueprint.system_prompt_to_structure,
- prompt_template_to_structure=blueprint.prompt_template_to_structure,
- )
- self.domain_library.add_domain_details(domain=domain)
-
- # Second pass: load all concepts
- for toml_path in library_paths:
- nb_concepts_before = len(self.concept_library.root)
- blueprint = self._load_blueprint_from_file(toml_path)
- self._load_concepts_from_blueprint(blueprint=blueprint, file_path=str(toml_path))
- nb_concepts_loaded = len(self.concept_library.root) - nb_concepts_before
- log.verbose(f"Loaded {nb_concepts_loaded} concepts from '{toml_path.name}'")
-
- # Third pass: load all pipes
- for toml_path in library_paths:
- nb_pipes_before = len(self.pipe_library.root)
- blueprint = self._load_blueprint_from_file(toml_path)
- self._load_pipes_from_blueprint(blueprint=blueprint, file_path=str(toml_path))
- nb_pipes_loaded = len(self.pipe_library.root) - nb_pipes_before
- log.verbose(f"Loaded {nb_pipes_loaded} pipes from '{toml_path.name}'")
-
- def _load_blueprint_from_file(self, toml_path: Path) -> PipelineBlueprint:
- """Load and validate a pipeline blueprint from a TOML file."""
- try:
- toml_data = load_toml_from_path(path=str(toml_path))
- blueprint = PipelineBlueprint.model_validate(toml_data)
- return blueprint
- except ValidationError as exc:
- error_msg = format_pydantic_validation_error(exc)
- raise PipelineBlueprintValidationError(
- file_path=str(toml_path),
- validation_error_msg=error_msg,
- ) from exc
- except Exception as exc:
- raise LibraryError(f"Failed to load TOML file '{toml_path}': {exc}") from exc
-
- def _load_concepts_from_blueprint(self, blueprint: PipelineBlueprint, file_path: str):
- """Load concepts from a validated blueprint."""
- for concept_name, concept_data in blueprint.concept.items():
- try:
- if isinstance(concept_data, str):
- # Simple string definition
- concept_from_def = ConceptFactory.make_concept_from_definition_str(
- domain_code=blueprint.domain,
- concept_str=concept_name,
- definition=concept_data,
- )
- self.concept_library.add_new_concept(concept=concept_from_def)
- else:
- # Complex ConceptBlueprint - guaranteed by blueprint schema
- concept_from_blueprint = ConceptFactory.make_concept_from_blueprint(
- domain=blueprint.domain,
- code=concept_name,
- concept_blueprint=concept_data,
- )
- self.concept_library.add_new_concept(concept=concept_from_blueprint)
- except ValidationError as exc:
- error_msg = format_pydantic_validation_error(exc)
- raise ConceptBlueprintError(
- file_path=file_path,
- concept_name=concept_name,
- error_msg=error_msg,
- ) from exc
-
- def _load_pipes_from_blueprint(self, blueprint: PipelineBlueprint, file_path: str):
- """Load pipes from a validated blueprint."""
- for pipe_name, pipe_data in blueprint.pipe.items():
- try:
- # pipe_data is guaranteed to be Dict[str, Any] by the blueprint schema
- pipe = LibraryManager.make_pipe_from_blueprint(
- domain_code=blueprint.domain,
- pipe_code=pipe_name,
- details_dict=pipe_data.copy(),
- )
- self.pipe_library.add_new_pipe(pipe=pipe)
- except ValidationError as validation_error:
- error_msg = format_pydantic_validation_error(validation_error)
- raise PipeBlueprintError(
- file_path=file_path,
- pipe_name=pipe_name,
- error_msg=error_msg,
- ) from validation_error
- except StaticValidationError as static_validation_error:
- static_validation_error.file_path = file_path
- raise static_validation_error
+ def __init__(
+ self,
+ domain_library: DomainLibrary,
+ concept_library: ConceptLibrary,
+ pipe_library: PipeLibrary,
+ library_config: LibraryConfig,
+ ):
+ self.domain_library = domain_library
+ self.concept_library = concept_library
+ self.pipe_library = pipe_library
+ self.library_config = library_config
+ self.llm_deck: Optional[LLMDeck] = None
@override
def validate_libraries(self):
log.debug("LibraryManager validating libraries")
+
if self.llm_deck is None:
raise LibraryError("LLM deck is not loaded")
@@ -284,6 +86,7 @@ class LibraryManager(LibraryManagerAbstract):
"""Validate all TOML files used by the library manager for formatting issues."""
log.debug("LibraryManager validating TOML file formatting")
+ # Validation of LLM deck paths
llm_deck_paths = self.library_config.get_llm_deck_paths()
for llm_deck_path in llm_deck_paths:
if os.path.exists(llm_deck_path):
@@ -293,25 +96,7 @@ class LibraryManager(LibraryManagerAbstract):
log.error(f"TOML formatting issues in LLM deck file '{llm_deck_path}': {exc}")
raise LibraryError(f"TOML validation failed for LLM deck file '{llm_deck_path}': {exc}") from exc
- # Validate pipeline library TOML files (same pattern as _load_combo_libraries)
- library_paths = self.libraries_paths()
- toml_file_paths: List[Path] = []
- for libraries_path in library_paths:
- if os.path.exists(libraries_path):
- found_file_paths = find_files_in_dir(
- dir_path=libraries_path,
- pattern="*.toml",
- is_recursive=True,
- )
- toml_file_paths.extend(found_file_paths)
-
- for toml_path in toml_file_paths:
- try:
- validate_toml_file(str(toml_path))
- except TOMLValidationError as exc:
- log.error(f"TOML formatting issues in library file '{toml_path}': {exc}")
- raise LibraryError(f"TOML validation failed for library file '{toml_path}': {exc}") from exc
-
+ # Validation of template paths
template_paths = self.library_config.get_templates_paths()
for template_path in template_paths:
if os.path.exists(template_path):
@@ -321,181 +106,111 @@ class LibraryManager(LibraryManagerAbstract):
log.error(f"TOML formatting issues in template file '{template_path}': {exc}")
raise LibraryError(f"TOML validation failed for template file '{template_path}': {exc}") from exc
- @classmethod
- def make_pipe_from_blueprint(
- cls,
- domain_code: str,
- pipe_code: str,
- details_dict: Dict[str, Any],
- ) -> PipeAbstract:
- pipe_definition: str
- pipe_class_name: str
- if "type" in details_dict and "definition" in details_dict:
- # New format: type = "PipeClassName" and definition = "description"
- pipe_class_name = details_dict.pop("type") # Remove type from details_dict
- pipe_definition = details_dict["definition"] # Keep definition for the factory
- else:
- # TODO(DEPRECATED_REMOVAL): Remove old pipe syntax support - scheduled for removal in v0.3.0
- # Fallback to old format for backward compatibility:
- # PipeClassName = "the pipe's definition in natural language"
- try:
- pipe_class_name, pipe_definition = next(iter(details_dict.items()))
- log.warning(f"""Pipe '{pipe_code}' uses deprecated syntax. Please migrate to new format:
-replace this syntax:
-```
-{pipe_class_name} = "{pipe_definition}"
-```
-by this:
-```
-type = "{pipe_class_name}"
-definition = "{pipe_definition}"
-```
-Old syntax will be removed in v0.3.0.
- """)
- details_dict.pop(pipe_class_name)
- except StopIteration as details_dict_empty_error:
- raise PipeFactoryError(f"Pipe '{pipe_code}' could not be created because its blueprint is empty.") from details_dict_empty_error
-
- # the factory class name for that specific type of Pipe is the pipe class name with "Factory" suffix
- factory_class_name = f"{pipe_class_name}Factory"
- try:
- pipe_factory: Type[PipeSpecificFactoryProtocol[Any, Any]] = KajsonManager.get_class_registry().get_required_subclass(
- name=factory_class_name,
- base_class=PipeSpecificFactoryProtocol,
- )
- except ClassRegistryNotFoundError as factory_not_found_error:
- raise PipeFactoryError(
- f"Pipe '{pipe_code}' couldn't be created: factory '{factory_class_name}' not found: {factory_not_found_error}"
- ) from factory_not_found_error
- except ClassRegistryInheritanceError as factory_inheritance_error:
- raise PipeFactoryError(
- f"Pipe '{pipe_code}' couldn't be created: factory '{factory_class_name}' is not a subclass of {type(PipeSpecificFactoryProtocol)}."
- ) from factory_inheritance_error
-
- details_dict["definition"] = pipe_definition
- details_dict["domain"] = domain_code
- pipe_from_blueprint: PipeAbstract = pipe_factory.make_pipe_from_details_dict(
- domain_code=domain_code,
- pipe_code=pipe_code,
- details_dict=details_dict,
- )
- return pipe_from_blueprint
-
- @classmethod
- def make_pipe_from_details_dict(
- cls,
- domain_code: str,
- pipe_code: str,
- details_dict: Dict[str, Any],
- ) -> PipeAbstract:
- # Delegate to the new make_pipe_from_blueprint method
- return cls.make_pipe_from_blueprint(
- domain_code=domain_code,
- pipe_code=pipe_code,
- details_dict=details_dict,
- )
-
- @classmethod
- def load_pipe_from_blueprint(
- cls,
- pipe_code: str,
- pipe_blueprint: PipeBlueprint,
- ) -> PipeAbstract:
- """Create a Pipe from a concrete PipeBlueprint instance.
-
- This resolves the corresponding factory by converting the blueprint class name
- (e.g. "PipeLLMBlueprint") to its factory name ("PipeLLMFactory").
- """
- # The blueprint must be a concrete subclass like PipeLLMBlueprint, PipeOcrBlueprint, etc.
- blueprint_class_name = type(pipe_blueprint).__name__
- if blueprint_class_name == "PipeBlueprint":
- raise PipeFactoryError("Cannot load pipe from base PipeBlueprint. Please provide a specific blueprint subclass (e.g. PipeLLMBlueprint).")
-
- # Derive factory name: PipeLLMBlueprint -> PipeLLMFactory
- factory_class_name = blueprint_class_name.replace("Blueprint", "Factory")
-
- try:
- pipe_factory: Type[PipeSpecificFactoryProtocol[Any, Any]] = KajsonManager.get_class_registry().get_required_subclass(
- name=factory_class_name,
- base_class=PipeSpecificFactoryProtocol,
+ @override
+ def setup(self) -> None:
+ native_concepts = ConceptFactory.list_native_concepts()
+ self.concept_library.add_concepts(concepts=native_concepts)
+
+ @override
+ def teardown(self) -> None:
+ self.llm_deck = None
+ self.pipe_library.teardown()
+ self.concept_library.teardown()
+ self.domain_library.teardown()
+
+ @override
+ def reset(self) -> None:
+ self.teardown()
+ self.setup()
+
+ def _get_pipeline_library_dirs(self) -> List[Path]:
+ library_dirs = [Path(self.library_config.pipelines_dir_path)]
+ if runtime_manager.is_unit_testing:
+ log.debug("Registering test pipeline structures for unit testing")
+ library_dirs += [Path(self.library_config.test_pipelines_dir_path)]
+ return library_dirs
+
+ def _get_pipelex_toml_files_from_dirs(self, dirs: List[Path]) -> List[Path]:
+ """Get all valid Pipelex TOML files from the given directories."""
+
+ all_toml_paths: List[Path] = []
+ for dir_path in dirs:
+ if not dir_path.exists():
+ raise LibraryError(f"Directory does not exist: {dir_path}")
+
+ # Find all TOML files in the directory
+ toml_files = find_files_in_dir(
+ dir_path=str(dir_path),
+ pattern="*.toml",
+ is_recursive=True,
)
- except ClassRegistryNotFoundError as factory_not_found_error:
- raise PipeFactoryError(
- f"Pipe '{pipe_code}' couldn't be created: factory '{factory_class_name}' not found: {factory_not_found_error}"
- ) from factory_not_found_error
- except ClassRegistryInheritanceError as factory_inheritance_error:
- raise PipeFactoryError(
- f"Pipe '{pipe_code}' couldn't be created: factory '{factory_class_name}' is not a subclass of {type(PipeSpecificFactoryProtocol)}."
- ) from factory_inheritance_error
-
- # Domain is part of the blueprint
- domain_code = pipe_blueprint.domain
-
- # Let the specific factory build the concrete Pipe instance
- pipe_from_blueprint: PipeAbstract = pipe_factory.make_pipe_from_blueprint(
- domain_code=domain_code,
- pipe_code=pipe_code,
- pipe_blueprint=pipe_blueprint, # type: ignore[arg-type]
- )
- return pipe_from_blueprint
-
- @classmethod
- def make_pipe_blueprint_from_details(
- cls,
- domain_code: str,
- details_dict: Dict[str, Any],
- ) -> PipeBlueprint:
- """Create a concrete PipeBlueprint subclass instance from details.
-
- This resolves the factory based on the provided details (supports both
- new and legacy formats), locates the corresponding <PipeClassName>Blueprint
- in the factory module, and validates the blueprint.
- """
- log.dev("Making Pipe blueprint")
- # Determine pipe class name from details
- if "type" in details_dict and "definition" in details_dict:
- pipe_class_name = details_dict.pop("type")
- normalized_details = details_dict.copy()
- log.dev(f"New format, pipe_class_name: {pipe_class_name}")
- log.dev(f"Normalized details: {normalized_details}")
- else:
+
+ # Filter to only include valid Pipelex files
+ for toml_file in toml_files:
+ if PipelexInterpreter.is_pipelex_file(toml_file):
+ all_toml_paths.append(toml_file)
+ else:
+ log.debug(f"Skipping non-Pipelex TOML file: {toml_file}")
+
+ return all_toml_paths
+
+ @override
+ def load_from_file(self, toml_path: Path) -> None:
+ if not PipelexInterpreter.is_pipelex_file(toml_path):
+ raise LibraryError(f"File is not a valid Pipelex TOML file: {toml_path}")
+
+ converter = PipelexInterpreter(file_path=toml_path)
+ blueprint = converter.make_pipelex_bundle_blueprint()
+ pipelex_bundle = PipelexBundleFactory.make_from_blueprint(blueprint=blueprint)
+ self.load_from_pipelex_bundle(pipelex_bundle=pipelex_bundle)
+
+ @override
+ def load_from_pipelex_bundle(self, pipelex_bundle: PipelexBundle) -> List[PipeAbstract]:
+ self.domain_library.add_domain(domain=pipelex_bundle.domain)
+ self.concept_library.add_concepts(concepts=list(pipelex_bundle.concepts.values()))
+ self.pipe_library.add_pipes(pipes=list(pipelex_bundle.pipes.values()))
+ return list(pipelex_bundle.pipes.values())
+
+ @override
+ def load_libraries(self, library_dirs: Optional[List[Path]] = None, library_file_paths: Optional[List[Path]] = None) -> None:
+ dirs_to_use: List[Path] = self._get_pipeline_library_dirs()
+ all_toml_paths: List[Path] = self._get_pipelex_toml_files_from_dirs(dirs_to_use)
+
+ # Remove failing pipelines from the list
+ failing_pipelines_file_paths = get_config().pipelex.library_config.failing_pipelines_file_paths
+ failing_paths_set = {Path(fp) for fp in failing_pipelines_file_paths}
+ all_toml_paths = [path for path in all_toml_paths if path not in failing_paths_set]
+
+ if library_dirs is not None:
+ dirs_to_use = library_dirs
+
+ # Register classes in the directories
+ for library_dir in dirs_to_use:
+ ClassRegistryUtils.register_classes_in_folder(folder_path=str(library_dir))
+
+ if library_file_paths is not None:
+ all_toml_paths = library_file_paths
+
+ for toml_file_path in all_toml_paths:
+ self.load_from_file(toml_path=toml_file_path)
+
+ # TODO: move to LLMDeckManager
+ def load_deck(self) -> LLMDeck:
+ llm_deck_paths = self.library_config.get_llm_deck_paths()
+ full_llm_deck_dict: Dict[str, Any] = {}
+ if not llm_deck_paths:
+ raise LLMDeckNotFoundError("No LLM deck paths found. Please run `pipelex init-libraries` to create it.")
+
+ for llm_deck_path in llm_deck_paths:
+ if not os.path.exists(llm_deck_path):
+ raise LLMDeckNotFoundError(f"LLM deck path `{llm_deck_path}` not found. Please run `pipelex init-libraries` to create it.")
try:
- pipe_class_name, legacy_definition = next(iter(details_dict.items()))
- normalized_details = {"definition": legacy_definition}
- log.dev(f"Legacy format, pipe_class_name: {pipe_class_name}")
- log.dev(f"Normalized details: {normalized_details}")
- except StopIteration as empty_err:
- raise PipeFactoryError("Pipe details are empty; cannot determine pipe type") from empty_err
-
- factory_class_name = f"{pipe_class_name}Factory"
- log.dev(f"Factory class name: {factory_class_name}")
-
- # Resolve factory to locate its module (where the Blueprint class is defined)
- try:
- pipe_factory: Type[PipeSpecificFactoryProtocol[Any, Any]] = KajsonManager.get_class_registry().get_required_subclass(
- name=factory_class_name,
- base_class=PipeSpecificFactoryProtocol,
- )
- except ClassRegistryNotFoundError as factory_not_found_error:
- raise PipeFactoryError(
- f"Factory '{factory_class_name}' not found for pipe type '{pipe_class_name}': {factory_not_found_error}"
- ) from factory_not_found_error
- except ClassRegistryInheritanceError as factory_inheritance_error:
- raise PipeFactoryError(
- f"Factory '{factory_class_name}' is not a subclass of {type(PipeSpecificFactoryProtocol)}."
- ) from factory_inheritance_error
-
- factory_module = importlib.import_module(pipe_factory.__module__)
- blueprint_class_name = f"{pipe_class_name}Blueprint"
- log.dev(f"Blueprint class name: {blueprint_class_name}")
-
- try:
- blueprint_cls: Type[PipeBlueprint] = getattr(factory_module, blueprint_class_name)
- except AttributeError as exc:
- raise PipeFactoryError(f"Cannot find blueprint class '{blueprint_class_name}' in module '{pipe_factory.__module__}'") from exc
-
- details_with_domain = normalized_details.copy()
- details_with_domain["domain"] = domain_code
-
- log.dev(f"Making Pipe blueprint for class {blueprint_class_name} from details: {details_with_domain}")
- return blueprint_cls.model_validate(details_with_domain)
+ llm_deck_dict = load_toml_from_path(path=llm_deck_path)
+ log.debug(f"Loaded LLM deck from {llm_deck_path}")
+ deep_update(full_llm_deck_dict, llm_deck_dict)
+ except Exception as exc:
+ log.error(f"Failed to load LLM deck file '{llm_deck_path}': {exc}")
+ raise
+
+ self.llm_deck = LLMDeck.model_validate(full_llm_deck_dict)
+ return self.llm_deck
diff --git a/pipelex/libraries/library_manager_abstract.py b/pipelex/libraries/library_manager_abstract.py
index 7f1d6553f..de1ec0afa 100644
--- a/pipelex/libraries/library_manager_abstract.py
+++ b/pipelex/libraries/library_manager_abstract.py
@@ -1,6 +1,9 @@
from abc import ABC, abstractmethod
from pathlib import Path
-from typing import List
+from typing import List, Optional
+
+from pipelex.core.bundles.pipelex_bundle import PipelexBundle
+from pipelex.core.pipes.pipe_abstract import PipeAbstract
class LibraryManagerAbstract(ABC):
@@ -13,13 +16,21 @@ class LibraryManagerAbstract(ABC):
pass
@abstractmethod
- def load_combo_libraries(self, library_paths: List[Path]):
+ def reset(self) -> None:
pass
@abstractmethod
- def get_plugin_config_path(self) -> str:
+ def validate_libraries(self) -> None:
pass
@abstractmethod
- def validate_libraries(self) -> None:
+ def load_libraries(self, library_dirs: Optional[List[Path]] = None, library_file_paths: Optional[List[Path]] = None) -> None:
+ pass
+
+ @abstractmethod
+ def load_from_file(self, toml_path: Path) -> None:
+ pass
+
+ @abstractmethod
+ def load_from_pipelex_bundle(self, pipelex_bundle: PipelexBundle) -> List[PipeAbstract]:
pass
diff --git a/pipelex/libraries/library_manager_factory.py b/pipelex/libraries/library_manager_factory.py
new file mode 100644
index 000000000..686b31f29
--- /dev/null
+++ b/pipelex/libraries/library_manager_factory.py
@@ -0,0 +1,38 @@
+from pipelex.core.concepts.concept_library import ConceptLibrary
+from pipelex.core.domains.domain_library import DomainLibrary
+from pipelex.core.pipes.pipe_library import PipeLibrary
+from pipelex.libraries.library_config import LibraryConfig
+from pipelex.libraries.library_manager import LibraryManager
+
+
+class LibraryManagerFactory:
+ """Factory for creating LibraryManager instances."""
+
+ @classmethod
+ def make_empty(cls, config_dir_path: str) -> "LibraryManager":
+ domain_library = DomainLibrary.make_empty()
+ concept_library = ConceptLibrary.make_empty()
+ pipe_library = PipeLibrary.make_empty()
+ library_config = LibraryConfig(config_dir_path=config_dir_path)
+
+ return LibraryManager(
+ domain_library=domain_library,
+ concept_library=concept_library,
+ pipe_library=pipe_library,
+ library_config=library_config,
+ )
+
+ @classmethod
+ def make(
+ cls, domain_library: DomainLibrary, concept_library: ConceptLibrary, pipe_library: PipeLibrary, config_dir_path: str
+ ) -> "LibraryManager":
+ """Create a LibraryManager with provided libraries."""
+
+ library_config = LibraryConfig(config_dir_path=config_dir_path)
+
+ return LibraryManager(
+ domain_library=domain_library,
+ concept_library=concept_library,
+ pipe_library=pipe_library,
+ library_config=library_config,
+ )
diff --git a/pipelex/libraries/llm_deck/base_llm_deck.toml b/pipelex/libraries/llm_deck/base_llm_deck.toml
index dc7329dba..726851f7e 100644
--- a/pipelex/libraries/llm_deck/base_llm_deck.toml
+++ b/pipelex/libraries/llm_deck/base_llm_deck.toml
@@ -20,11 +20,11 @@ best-gemini = "gemini-2.5-pro"
best-mistral = "mistral-large"
best-grok = "grok-3"
-cheap_llm_for_text = "gpt-5-nano"
-cheap_llm_for_object = "gpt-5-nano"
+cheap_llm_for_text = "gpt-4o-mini"
+cheap_llm_for_object = "gpt-4o-mini"
-base_llm_for_text = "gpt-5-nano"
-base_llm_for_object = "gpt-5-nano"
+base_llm_for_text = "gpt-4o-mini"
+base_llm_for_object = "gpt-4o-mini"
####################################################################################################
diff --git a/pipelex/libraries/pipeline_blueprint.py b/pipelex/libraries/pipeline_blueprint.py
deleted file mode 100644
index 9d485045c..000000000
--- a/pipelex/libraries/pipeline_blueprint.py
+++ /dev/null
@@ -1,58 +0,0 @@
-from typing import Any, Dict, Optional, Union
-
-from pydantic import Field
-
-from pipelex.core.concept_factory import ConceptBlueprint
-from pipelex.core.stuff_content import StructuredContent
-from pipelex.exceptions import LibraryError
-
-
-class PipelineBlueprintValidationError(LibraryError):
- """Error raised when a pipeline blueprint fails validation."""
-
- def __init__(self, file_path: str, validation_error_msg: str):
- self.file_path = file_path
- self.validation_error_msg = validation_error_msg
- super().__init__(f"Pipeline blueprint validation failed for '{file_path}': {validation_error_msg}")
-
-
-class ConceptBlueprintError(LibraryError):
- """Error raised when loading concept blueprints."""
-
- def __init__(self, file_path: str, concept_name: str, error_msg: str):
- self.file_path = file_path
- self.concept_name = concept_name
- super().__init__(f"Error loading concept '{concept_name}' from '{file_path}': {error_msg}")
-
-
-class PipeBlueprintError(LibraryError):
- """Error raised when loading pipe blueprints."""
-
- def __init__(self, file_path: str, pipe_name: str, error_msg: str):
- self.file_path = file_path
- self.pipe_name = pipe_name
- super().__init__(f"Error loading pipe '{pipe_name}' from '{file_path}': {error_msg}")
-
-
-class PipelineBlueprint(StructuredContent):
- """Complete blueprint of a pipeline TOML definition."""
-
- # Domain information (required)
- domain: str
- definition: Optional[str] = None
- system_prompt: Optional[str] = None
- system_prompt_to_structure: Optional[str] = None
- prompt_template_to_structure: Optional[str] = None
-
- # Concepts section - concept_name -> definition (string) or blueprint (dict)
- concept: Dict[str, Union[str, ConceptBlueprint]] = Field(default_factory=dict)
-
- # Pipes section - pipe_name -> blueprint dict
- pipe: Dict[str, Dict[str, Any]] = Field(default_factory=dict)
-
- @property
- def desc(self) -> str:
- if self.definition:
- return f"{self.domain} • {self.definition[:100]}"
- else:
- return self.domain
diff --git a/pipelex/libraries/pipelines/images.toml b/pipelex/libraries/pipelines/images.toml
index 9d4197279..b46d0869a 100644
--- a/pipelex/libraries/pipelines/images.toml
+++ b/pipelex/libraries/pipelines/images.toml
@@ -10,7 +10,6 @@ refines = "Text"
[concept.Photo]
definition = "Photo"
-structure = "ImageContent"
refines = "Image"
[pipe]
diff --git a/pipelex/libraries/pipelines/meta/meta_pipeline.toml b/pipelex/libraries/pipelines/meta/meta_pipeline.toml
index 21d6d99d0..cbeb7ad44 100644
--- a/pipelex/libraries/pipelines/meta/meta_pipeline.toml
+++ b/pipelex/libraries/pipelines/meta/meta_pipeline.toml
@@ -5,17 +5,19 @@ definition = "Meta domain is the domain of pipelines about pipelines"
PipelineDraft = "Rough solution to create a pipeline"
PipelineDraftText = "Rough solution to create a pipeline in text format"
PipeDraft = "Rough solution to create a pipe"
-PipelineBlueprint = "Details enabling to create a pipeline"
ConceptBlueprint = "Details enabling to create a concept"
PipeBlueprint = "Details enabling to create a pipe"
-[pipe]
+[concept.PipelexBundleBlueprint]
+definition = "Details enabling to create a pipeline"
+structure = "PipelexBundleBlueprintStuff"
+[pipe]
[pipe.build_blueprint]
type = "PipeSequence"
definition = "Build a pipeline blueprint from natural language requirements"
-inputs = { draft_pipeline_rules = "Text", build_pipeline_rules = "Text", requirements = "Text", pipeline_name = "Text", domain = "Text" }
-output = "PipelineBlueprint"
+inputs = { draft_pipeline_rules = "Text", build_pipeline_rules = "Text", create_structured_output_rules = "Text", requirements = "Text", pipeline_name = "Text", domain = "Text" }
+output = "PipelexBundleBlueprint"
steps = [
{ pipe = "draft_pipeline_text", result = "pipeline_draft" },
{ pipe = "structure_blueprint", result = "pipeline_blueprint" },
@@ -24,7 +26,7 @@ steps = [
[pipe.draft_pipeline_text]
type = "PipeLLM"
definition = "Generate a rough pipeline draft from natural language requirements"
-inputs = { draft_pipeline_rules = "Text", requirements = "Text", pipeline_name = "Text", domain = "Text" }
+inputs = { draft_pipeline_rules = "Text", create_structured_output_rules = "Text", requirements = "Text", pipeline_name = "Text", domain = "Text" }
output = "PipelineDraftText"
llm = "llm_to_pipelex"
prompt_template = """
@@ -32,6 +34,9 @@ You are a Pipelex pipeline architect. Build a pipeline draft text showing how we
@draft_pipeline_rules
+Do create structured output for better concept representation.
+
+@create_structured_output_rules
---
Name of the pipeline: $pipeline_name.
@@ -44,7 +49,7 @@ Requirements: $requirements
type = "PipeLLM"
definition = "Generate a pipeline blueprint from natural language requirements"
inputs = { build_pipeline_rules = "Text", pipeline_draft = "Text", pipeline_name = "Text", domain = "Text", requirements = "Text" }
-output = "PipelineBlueprint"
+output = "PipelexBundleBlueprint"
llm = "llm_to_pipelex"
# structuring_method = "preliminary_text"
prompt_template = """
diff --git a/pipelex/libraries/pipelines/meta/pipeline_draft.py b/pipelex/libraries/pipelines/meta/pipeline_draft.py
index 26e5ab970..c756a9d6b 100644
--- a/pipelex/libraries/pipelines/meta/pipeline_draft.py
+++ b/pipelex/libraries/pipelines/meta/pipeline_draft.py
@@ -2,7 +2,8 @@ from typing import Dict
from pydantic import Field
-from pipelex.core.stuff_content import StructuredContent
+from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint
+from pipelex.core.stuffs.stuff_content import StructuredContent
class PipeDraft(StructuredContent):
@@ -25,3 +26,9 @@ class PipelineDraft(StructuredContent):
# Pipes section - pipe_name -> blueprint dict
pipe: Dict[str, PipeDraft] = Field(default_factory=dict)
+
+
+class PipelexBundleBlueprintStuff(PipelexBundleBlueprint, StructuredContent):
+ """Complete blueprint of a pipelex bundle TOML file."""
+
+ pass
diff --git a/pipelex/pipe_controllers/pipe_batch.py b/pipelex/pipe_controllers/pipe_batch.py
index 95d075f98..f7116e03f 100644
--- a/pipelex/pipe_controllers/pipe_batch.py
+++ b/pipelex/pipe_controllers/pipe_batch.py
@@ -7,13 +7,13 @@ from typing_extensions import Self, override
from pipelex import log
from pipelex.config import get_config
-from pipelex.core.pipe_input_spec import PipeInputSpec
-from pipelex.core.pipe_output import PipeOutput
-from pipelex.core.pipe_run_params import BatchParams, PipeRunMode, PipeRunParams
-from pipelex.core.stuff import Stuff
-from pipelex.core.stuff_content import ListContent, StuffContent
-from pipelex.core.stuff_factory import StuffFactory
-from pipelex.core.working_memory import MAIN_STUFF_NAME, WorkingMemory
+from pipelex.core.memory.working_memory import MAIN_STUFF_NAME, WorkingMemory
+from pipelex.core.pipes.pipe_input_spec import PipeInputSpec
+from pipelex.core.pipes.pipe_output import PipeOutput
+from pipelex.core.pipes.pipe_run_params import BatchParams, PipeRunMode, PipeRunParams
+from pipelex.core.stuffs.stuff import Stuff
+from pipelex.core.stuffs.stuff_content import ListContent, StuffContent
+from pipelex.core.stuffs.stuff_factory import StuffFactory
from pipelex.exceptions import (
PipeInputError,
PipeInputNotFoundError,
diff --git a/pipelex/pipe_controllers/pipe_batch_factory.py b/pipelex/pipe_controllers/pipe_batch_factory.py
index 82fee5325..7a4797f67 100644
--- a/pipelex/pipe_controllers/pipe_batch_factory.py
+++ b/pipelex/pipe_controllers/pipe_batch_factory.py
@@ -1,21 +1,23 @@
-from typing import Any, Dict, Optional
+from typing import Literal, Optional
from typing_extensions import override
-from pipelex.core.pipe_blueprint import PipeBlueprint, PipeSpecificFactoryProtocol
-from pipelex.core.pipe_input_spec import PipeInputSpec
-from pipelex.core.pipe_run_params import BatchParams
+from pipelex.core.pipes.pipe_blueprint import PipeBlueprint
+from pipelex.core.pipes.pipe_factory import PipeFactoryProtocol
+from pipelex.core.pipes.pipe_input_spec import PipeInputSpec
+from pipelex.core.pipes.pipe_run_params import BatchParams
from pipelex.pipe_controllers.pipe_batch import PipeBatch
class PipeBatchBlueprint(PipeBlueprint):
+ type: Literal["PipeBatch"] = "PipeBatch"
branch_pipe_code: str
input_list_name: Optional[str] = None
input_item_name: Optional[str] = None
-class PipeBatchFactory(PipeSpecificFactoryProtocol[PipeBatchBlueprint, PipeBatch]):
+class PipeBatchFactory(PipeFactoryProtocol[PipeBatchBlueprint, PipeBatch]):
@classmethod
@override
def make_pipe_from_blueprint(
@@ -32,23 +34,8 @@ class PipeBatchFactory(PipeSpecificFactoryProtocol[PipeBatchBlueprint, PipeBatch
domain=domain_code,
code=pipe_code,
definition=pipe_blueprint.definition,
- inputs=PipeInputSpec.make_from_dict(concepts_dict=pipe_blueprint.inputs or {}),
+ inputs=PipeInputSpec.make_from_blueprint(domain=domain_code, blueprint=pipe_blueprint.inputs or {}),
output_concept_code=pipe_blueprint.output,
branch_pipe_code=pipe_blueprint.branch_pipe_code,
batch_params=batch_params,
)
-
- @classmethod
- @override
- def make_pipe_from_details_dict(
- cls,
- domain_code: str,
- pipe_code: str,
- details_dict: Dict[str, Any],
- ) -> PipeBatch:
- pipe_blueprint = PipeBatchBlueprint.model_validate(details_dict)
- return cls.make_pipe_from_blueprint(
- domain_code=domain_code,
- pipe_code=pipe_code,
- pipe_blueprint=pipe_blueprint,
- )
diff --git a/pipelex/pipe_controllers/pipe_condition.py b/pipelex/pipe_controllers/pipe_condition.py
index 986da64bb..f0252a0ca 100644
--- a/pipelex/pipe_controllers/pipe_condition.py
+++ b/pipelex/pipe_controllers/pipe_condition.py
@@ -6,10 +6,10 @@ from typing_extensions import Self, override
from pipelex import log
from pipelex.config import StaticValidationReaction, get_config
-from pipelex.core.pipe_input_spec import PipeInputSpec
-from pipelex.core.pipe_output import PipeOutput
-from pipelex.core.pipe_run_params import PipeRunParams
-from pipelex.core.working_memory import WorkingMemory
+from pipelex.core.memory.working_memory import WorkingMemory
+from pipelex.core.pipes.pipe_input_spec import PipeInputSpec
+from pipelex.core.pipes.pipe_output import PipeOutput
+from pipelex.core.pipes.pipe_run_params import PipeRunParams
from pipelex.exceptions import (
DryRunError,
PipeConditionError,
diff --git a/pipelex/pipe_controllers/pipe_condition_factory.py b/pipelex/pipe_controllers/pipe_condition_factory.py
index 3578fde9e..75fae4fbd 100644
--- a/pipelex/pipe_controllers/pipe_condition_factory.py
+++ b/pipelex/pipe_controllers/pipe_condition_factory.py
@@ -1,22 +1,25 @@
-from typing import Any, Dict, Optional
+from typing import Dict, Literal, Optional
from typing_extensions import override
-from pipelex.core.pipe_blueprint import PipeBlueprint, PipeSpecificFactoryProtocol
-from pipelex.core.pipe_input_spec import PipeInputSpec
+from pipelex.core.pipes.pipe_blueprint import PipeBlueprint
+from pipelex.core.pipes.pipe_factory import PipeFactoryProtocol
+from pipelex.core.pipes.pipe_input_spec import PipeInputSpec
from pipelex.pipe_controllers.pipe_condition import PipeCondition
class PipeConditionBlueprint(PipeBlueprint):
+ type: Literal["PipeCondition"] = "PipeCondition"
expression_template: Optional[str] = None
expression: Optional[str] = None
# TODO: make the values of pipe_map a Union[str, PipeAdapter] or something to set a specific alias
+ # TODO: Add a Blueprint for the pipe_map
pipe_map: Dict[str, str]
default_pipe_code: Optional[str] = None
add_alias_from_expression_to: Optional[str] = None
-class PipeConditionFactory(PipeSpecificFactoryProtocol[PipeConditionBlueprint, PipeCondition]):
+class PipeConditionFactory(PipeFactoryProtocol[PipeConditionBlueprint, PipeCondition]):
@classmethod
@override
def make_pipe_from_blueprint(
@@ -29,7 +32,7 @@ class PipeConditionFactory(PipeSpecificFactoryProtocol[PipeConditionBlueprint, P
domain=domain_code,
code=pipe_code,
definition=pipe_blueprint.definition,
- inputs=PipeInputSpec.make_from_dict(concepts_dict=pipe_blueprint.inputs or {}),
+ inputs=PipeInputSpec.make_from_blueprint(domain=domain_code, blueprint=pipe_blueprint.inputs or {}),
output_concept_code=pipe_blueprint.output,
expression_template=pipe_blueprint.expression_template,
expression=pipe_blueprint.expression,
@@ -37,18 +40,3 @@ class PipeConditionFactory(PipeSpecificFactoryProtocol[PipeConditionBlueprint, P
default_pipe_code=pipe_blueprint.default_pipe_code,
add_alias_from_expression_to=pipe_blueprint.add_alias_from_expression_to,
)
-
- @classmethod
- @override
- def make_pipe_from_details_dict(
- cls,
- domain_code: str,
- pipe_code: str,
- details_dict: Dict[str, Any],
- ) -> PipeCondition:
- pipe_blueprint = PipeConditionBlueprint.model_validate(details_dict)
- return cls.make_pipe_from_blueprint(
- domain_code=domain_code,
- pipe_code=pipe_code,
- pipe_blueprint=pipe_blueprint,
- )
diff --git a/pipelex/pipe_controllers/pipe_controller.py b/pipelex/pipe_controllers/pipe_controller.py
index de4c3aafd..344efc1b7 100644
--- a/pipelex/pipe_controllers/pipe_controller.py
+++ b/pipelex/pipe_controllers/pipe_controller.py
@@ -4,10 +4,10 @@ from typing import Optional
from typing_extensions import override
from pipelex import log
-from pipelex.core.pipe_abstract import PipeAbstract
-from pipelex.core.pipe_output import PipeOutput
-from pipelex.core.pipe_run_params import PipeRunMode, PipeRunParams
-from pipelex.core.working_memory import WorkingMemory
+from pipelex.core.memory.working_memory import WorkingMemory
+from pipelex.core.pipes.pipe_abstract import PipeAbstract
+from pipelex.core.pipes.pipe_output import PipeOutput
+from pipelex.core.pipes.pipe_run_params import PipeRunMode, PipeRunParams
from pipelex.pipeline.job_metadata import JobMetadata
diff --git a/pipelex/pipe_controllers/pipe_parallel.py b/pipelex/pipe_controllers/pipe_parallel.py
index fddac805d..9123ce048 100644
--- a/pipelex/pipe_controllers/pipe_parallel.py
+++ b/pipelex/pipe_controllers/pipe_parallel.py
@@ -6,13 +6,13 @@ from typing_extensions import Self, override
from pipelex import log
from pipelex.config import StaticValidationReaction, get_config
-from pipelex.core.pipe_input_spec import PipeInputSpec
-from pipelex.core.pipe_output import PipeOutput
-from pipelex.core.pipe_run_params import PipeRunMode, PipeRunParams
-from pipelex.core.stuff import Stuff
-from pipelex.core.stuff_content import StuffContent
-from pipelex.core.stuff_factory import StuffFactory
-from pipelex.core.working_memory import WorkingMemory
+from pipelex.core.memory.working_memory import WorkingMemory
+from pipelex.core.pipes.pipe_input_spec import PipeInputSpec
+from pipelex.core.pipes.pipe_output import PipeOutput
+from pipelex.core.pipes.pipe_run_params import PipeRunMode, PipeRunParams
+from pipelex.core.stuffs.stuff import Stuff
+from pipelex.core.stuffs.stuff_content import StuffContent
+from pipelex.core.stuffs.stuff_factory import StuffFactory
from pipelex.exceptions import DryRunError, PipeDefinitionError, PipeRunParamsError, StaticValidationError, StaticValidationErrorType
from pipelex.hub import get_pipeline_tracker, get_required_pipe
from pipelex.pipe_controllers.pipe_controller import PipeController
diff --git a/pipelex/pipe_controllers/pipe_parallel_factory.py b/pipelex/pipe_controllers/pipe_parallel_factory.py
index 871e2ca0c..d2d25ceb3 100644
--- a/pipelex/pipe_controllers/pipe_parallel_factory.py
+++ b/pipelex/pipe_controllers/pipe_parallel_factory.py
@@ -1,10 +1,11 @@
-from typing import Any, Dict, List, Optional
+from typing import List, Literal, Optional
from typing_extensions import override
-from pipelex.core.concept import Concept
-from pipelex.core.pipe_blueprint import PipeBlueprint, PipeSpecificFactoryProtocol
-from pipelex.core.pipe_input_spec import PipeInputSpec
+from pipelex.core.concepts.concept import Concept
+from pipelex.core.pipes.pipe_blueprint import PipeBlueprint
+from pipelex.core.pipes.pipe_factory import PipeFactoryProtocol
+from pipelex.core.pipes.pipe_input_spec import PipeInputSpec
from pipelex.exceptions import PipeDefinitionError
from pipelex.hub import get_concept_provider
from pipelex.pipe_controllers.pipe_parallel import PipeParallel
@@ -13,12 +14,13 @@ from pipelex.pipe_controllers.sub_pipe_factory import SubPipeBlueprint
class PipeParallelBlueprint(PipeBlueprint):
+ type: Literal["PipeParallel"] = "PipeParallel"
parallels: List[SubPipeBlueprint]
add_each_output: bool = True
combined_output: Optional[str] = None
-class PipeParallelFactory(PipeSpecificFactoryProtocol[PipeParallelBlueprint, PipeParallel]):
+class PipeParallelFactory(PipeFactoryProtocol[PipeParallelBlueprint, PipeParallel]):
@classmethod
@override
def make_pipe_from_blueprint(
@@ -43,24 +45,9 @@ class PipeParallelFactory(PipeSpecificFactoryProtocol[PipeParallelBlueprint, Pip
domain=domain_code,
code=pipe_code,
definition=pipe_blueprint.definition,
- inputs=PipeInputSpec.make_from_dict(concepts_dict=pipe_blueprint.inputs or {}),
+ inputs=PipeInputSpec.make_from_blueprint(domain=domain_code, blueprint=pipe_blueprint.inputs or {}),
output_concept_code=pipe_blueprint.output,
parallel_sub_pipes=parallel_sub_pipes,
add_each_output=pipe_blueprint.add_each_output,
combined_output=pipe_blueprint.combined_output,
)
-
- @classmethod
- @override
- def make_pipe_from_details_dict(
- cls,
- domain_code: str,
- pipe_code: str,
- details_dict: Dict[str, Any],
- ) -> PipeParallel:
- pipe_blueprint = PipeParallelBlueprint.model_validate(details_dict)
- return cls.make_pipe_from_blueprint(
- domain_code=domain_code,
- pipe_code=pipe_code,
- pipe_blueprint=pipe_blueprint,
- )
diff --git a/pipelex/pipe_controllers/pipe_sequence.py b/pipelex/pipe_controllers/pipe_sequence.py
index e6a8f85a4..8d561d186 100644
--- a/pipelex/pipe_controllers/pipe_sequence.py
+++ b/pipelex/pipe_controllers/pipe_sequence.py
@@ -5,10 +5,10 @@ from typing_extensions import Self, override
from pipelex import log
from pipelex.config import StaticValidationReaction, get_config
-from pipelex.core.pipe_input_spec import PipeInputSpec
-from pipelex.core.pipe_output import PipeOutput
-from pipelex.core.pipe_run_params import PipeRunMode, PipeRunParams
-from pipelex.core.working_memory import WorkingMemory
+from pipelex.core.memory.working_memory import WorkingMemory
+from pipelex.core.pipes.pipe_input_spec import PipeInputSpec
+from pipelex.core.pipes.pipe_output import PipeOutput
+from pipelex.core.pipes.pipe_run_params import PipeRunMode, PipeRunParams
from pipelex.exceptions import (
PipeRunParamsError,
StaticValidationError,
diff --git a/pipelex/pipe_controllers/pipe_sequence_factory.py b/pipelex/pipe_controllers/pipe_sequence_factory.py
index ce5500121..e49b76865 100644
--- a/pipelex/pipe_controllers/pipe_sequence_factory.py
+++ b/pipelex/pipe_controllers/pipe_sequence_factory.py
@@ -1,18 +1,20 @@
-from typing import Any, Dict, List
+from typing import List, Literal
from typing_extensions import override
-from pipelex.core.pipe_blueprint import PipeBlueprint, PipeSpecificFactoryProtocol
-from pipelex.core.pipe_input_spec import PipeInputSpec
+from pipelex.core.pipes.pipe_blueprint import PipeBlueprint
+from pipelex.core.pipes.pipe_factory import PipeFactoryProtocol
+from pipelex.core.pipes.pipe_input_spec import PipeInputSpec
from pipelex.pipe_controllers.pipe_sequence import PipeSequence
from pipelex.pipe_controllers.sub_pipe_factory import SubPipeBlueprint
class PipeSequenceBlueprint(PipeBlueprint):
+ type: Literal["PipeSequence"] = "PipeSequence"
steps: List[SubPipeBlueprint]
-class PipeSequenceFactory(PipeSpecificFactoryProtocol[PipeSequenceBlueprint, PipeSequence]):
+class PipeSequenceFactory(PipeFactoryProtocol[PipeSequenceBlueprint, PipeSequence]):
@classmethod
@override
def make_pipe_from_blueprint(
@@ -26,22 +28,7 @@ class PipeSequenceFactory(PipeSpecificFactoryProtocol[PipeSequenceBlueprint, Pip
domain=domain_code,
code=pipe_code,
definition=pipe_blueprint.definition,
- inputs=PipeInputSpec.make_from_dict(concepts_dict=pipe_blueprint.inputs or {}),
+ inputs=PipeInputSpec.make_from_blueprint(domain=domain_code, blueprint=pipe_blueprint.inputs or {}),
output_concept_code=pipe_blueprint.output,
sequential_sub_pipes=pipe_steps,
)
-
- @classmethod
- @override
- def make_pipe_from_details_dict(
- cls,
- domain_code: str,
- pipe_code: str,
- details_dict: Dict[str, Any],
- ) -> PipeSequence:
- pipe_blueprint = PipeSequenceBlueprint.model_validate(details_dict)
- return cls.make_pipe_from_blueprint(
- domain_code=domain_code,
- pipe_code=pipe_code,
- pipe_blueprint=pipe_blueprint,
- )
diff --git a/pipelex/pipe_controllers/sub_pipe.py b/pipelex/pipe_controllers/sub_pipe.py
index 0b32fb684..8e7372658 100644
--- a/pipelex/pipe_controllers/sub_pipe.py
+++ b/pipelex/pipe_controllers/sub_pipe.py
@@ -3,9 +3,9 @@ from typing import Optional
from pydantic import BaseModel
from pipelex import log
-from pipelex.core.pipe_output import PipeOutput
-from pipelex.core.pipe_run_params import BatchParams, PipeOutputMultiplicity, PipeRunMode, PipeRunParams
-from pipelex.core.working_memory import WorkingMemory
+from pipelex.core.memory.working_memory import WorkingMemory
+from pipelex.core.pipes.pipe_output import PipeOutput
+from pipelex.core.pipes.pipe_run_params import BatchParams, PipeOutputMultiplicity, PipeRunMode, PipeRunParams
from pipelex.exceptions import PipeInputError, WorkingMemoryStuffNotFoundError
from pipelex.hub import get_pipe_router, get_pipeline_tracker, get_required_pipe
from pipelex.pipe_controllers.pipe_batch import PipeBatch
diff --git a/pipelex/pipe_controllers/sub_pipe_factory.py b/pipelex/pipe_controllers/sub_pipe_factory.py
index 9e256e046..4991b49b6 100644
--- a/pipelex/pipe_controllers/sub_pipe_factory.py
+++ b/pipelex/pipe_controllers/sub_pipe_factory.py
@@ -3,7 +3,7 @@ from typing import Optional, Union
from pydantic import BaseModel, ConfigDict, model_validator
from typing_extensions import Self
-from pipelex.core.pipe_run_params import BatchParams, make_output_multiplicity
+from pipelex.core.pipes.pipe_run_params import BatchParams, make_output_multiplicity
from pipelex.exceptions import PipeDefinitionError
from pipelex.pipe_controllers.sub_pipe import SubPipe
from pipelex.tools.typing.validation_utils import has_more_than_one_among_attributes_from_list
diff --git a/pipelex/pipe_operators/pipe_func.py b/pipelex/pipe_operators/pipe_func.py
index 1656dfb58..5a803e6e3 100644
--- a/pipelex/pipe_operators/pipe_func.py
+++ b/pipelex/pipe_operators/pipe_func.py
@@ -3,13 +3,13 @@ from typing import List, Optional, Set, cast, get_type_hints
from typing_extensions import override
from pipelex import log
-from pipelex.core.pipe_input_spec import PipeInputSpec, TypedNamedInputRequirement
-from pipelex.core.pipe_output import PipeOutput
-from pipelex.core.pipe_run_params import PipeRunParams
-from pipelex.core.stuff_content import ListContent, StuffContent, TextContent
-from pipelex.core.stuff_factory import StuffFactory
-from pipelex.core.working_memory import WorkingMemory
-from pipelex.core.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.memory.working_memory import WorkingMemory
+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.pipes.pipe_input_spec import PipeInputSpec, TypedNamedInputRequirement
+from pipelex.core.pipes.pipe_output import PipeOutput
+from pipelex.core.pipes.pipe_run_params import PipeRunParams
+from pipelex.core.stuffs.stuff_content import ListContent, StuffContent, TextContent
+from pipelex.core.stuffs.stuff_factory import StuffFactory
from pipelex.exceptions import DryRunError
from pipelex.pipe_operators.pipe_operator import PipeOperator
from pipelex.pipeline.job_metadata import JobMetadata
diff --git a/pipelex/pipe_operators/pipe_func_factory.py b/pipelex/pipe_operators/pipe_func_factory.py
index c38ac6c47..c702acd44 100644
--- a/pipelex/pipe_operators/pipe_func_factory.py
+++ b/pipelex/pipe_operators/pipe_func_factory.py
@@ -1,17 +1,19 @@
-from typing import Any, Dict
+from typing import Literal
from typing_extensions import override
-from pipelex.core.pipe_blueprint import PipeBlueprint, PipeSpecificFactoryProtocol
-from pipelex.core.pipe_input_spec import PipeInputSpec
+from pipelex.core.pipes.pipe_blueprint import PipeBlueprint
+from pipelex.core.pipes.pipe_factory import PipeFactoryProtocol
+from pipelex.core.pipes.pipe_input_spec import PipeInputSpec
from pipelex.pipe_operators.pipe_func import PipeFunc
class PipeFuncBlueprint(PipeBlueprint):
+ type: Literal["PipeFunc"] = "PipeFunc"
function_name: str
-class PipeFuncFactory(PipeSpecificFactoryProtocol[PipeFuncBlueprint, PipeFunc]):
+class PipeFuncFactory(PipeFactoryProtocol[PipeFuncBlueprint, PipeFunc]):
@classmethod
@override
def make_pipe_from_blueprint(
@@ -24,22 +26,7 @@ class PipeFuncFactory(PipeSpecificFactoryProtocol[PipeFuncBlueprint, PipeFunc]):
domain=domain_code,
code=pipe_code,
definition=pipe_blueprint.definition,
- inputs=PipeInputSpec.make_from_dict(concepts_dict=pipe_blueprint.inputs or {}),
+ inputs=PipeInputSpec.make_from_blueprint(domain=domain_code, blueprint=pipe_blueprint.inputs or {}),
output_concept_code=pipe_blueprint.output,
function_name=pipe_blueprint.function_name,
)
-
- @classmethod
- @override
- def make_pipe_from_details_dict(
- cls,
- domain_code: str,
- pipe_code: str,
- details_dict: Dict[str, Any],
- ) -> PipeFunc:
- pipe_blueprint = PipeFuncBlueprint.model_validate(details_dict)
- return cls.make_pipe_from_blueprint(
- domain_code=domain_code,
- pipe_code=pipe_code,
- pipe_blueprint=pipe_blueprint,
- )
diff --git a/pipelex/pipe_operators/pipe_img_gen.py b/pipelex/pipe_operators/pipe_img_gen.py
index 742b9c18f..de624c0e0 100644
--- a/pipelex/pipe_operators/pipe_img_gen.py
+++ b/pipelex/pipe_operators/pipe_img_gen.py
@@ -10,14 +10,14 @@ from pipelex.cogt.imgg.imgg_handle import ImggHandle
from pipelex.cogt.imgg.imgg_job_components import AspectRatio, Background, ImggJobParams, Quality
from pipelex.cogt.imgg.imgg_prompt import ImggPrompt
from pipelex.config import StaticValidationReaction, get_config
-from pipelex.core.concept_native import NativeConcept
-from pipelex.core.pipe_input_spec import PipeInputSpec
-from pipelex.core.pipe_output import PipeOutput
-from pipelex.core.pipe_run_params import PipeOutputMultiplicity, PipeRunMode, PipeRunParams, output_multiplicity_to_apply
-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory
-from pipelex.core.stuff_content import ImageContent, ListContent, StuffContent
-from pipelex.core.stuff_factory import StuffFactory
-from pipelex.core.working_memory import WorkingMemory
+from pipelex.core.concepts.concept_native import NativeConcept
+from pipelex.core.memory.working_memory import WorkingMemory
+from pipelex.core.pipes.pipe_input_spec import PipeInputSpec
+from pipelex.core.pipes.pipe_output import PipeOutput
+from pipelex.core.pipes.pipe_run_params import PipeOutputMultiplicity, PipeRunMode, PipeRunParams, output_multiplicity_to_apply
+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory
+from pipelex.core.stuffs.stuff_content import ImageContent, ListContent, StuffContent
+from pipelex.core.stuffs.stuff_factory import StuffFactory
from pipelex.exceptions import (
PipeDefinitionError,
PipeInputError,
@@ -47,6 +47,9 @@ class PipeImgGenOutput(PipeOutput):
return the_urls
+DEFAULT_PROMPT_VAR_NAME = "prompt"
+
+
class PipeImgGen(PipeOperator):
output_concept_code: str = NativeConcept.IMAGE.code
imgg_prompt: Optional[str] = None
@@ -92,7 +95,7 @@ class PipeImgGen(PipeOperator):
if self.img_gen_prompt_var_name:
return {self.img_gen_prompt_var_name}
else:
- return {"imgg_prompt"}
+ return {DEFAULT_PROMPT_VAR_NAME}
def _validate_inputs(self):
concept_provider = get_concept_provider()
diff --git a/pipelex/pipe_operators/pipe_img_gen_factory.py b/pipelex/pipe_operators/pipe_img_gen_factory.py
index 2467681bc..528a0a1ba 100644
--- a/pipelex/pipe_operators/pipe_img_gen_factory.py
+++ b/pipelex/pipe_operators/pipe_img_gen_factory.py
@@ -1,18 +1,20 @@
-from typing import Any, Dict, Literal, Optional, Union
+from typing import Literal, Optional, Union
from pydantic import Field, model_validator
from typing_extensions import Self, override
from pipelex.cogt.imgg.imgg_handle import ImggHandle
from pipelex.cogt.imgg.imgg_job_components import AspectRatio, Quality
-from pipelex.core.pipe_blueprint import PipeBlueprint, PipeSpecificFactoryProtocol
-from pipelex.core.pipe_input_spec import PipeInputSpec
+from pipelex.core.pipes.pipe_blueprint import PipeBlueprint
+from pipelex.core.pipes.pipe_factory import PipeFactoryProtocol
+from pipelex.core.pipes.pipe_input_spec import PipeInputSpec
from pipelex.exceptions import PipeDefinitionError
from pipelex.pipe_operators.pipe_img_gen import PipeImgGen
from pipelex.tools.typing.validation_utils import has_more_than_one_among_attributes_from_lists
class PipeImgGenBlueprint(PipeBlueprint):
+ type: Literal["PipeImgGen"] = "PipeImgGen"
img_gen_prompt: Optional[str] = None
imgg_handle: Optional[ImggHandle] = None
aspect_ratio: Optional[AspectRatio] = Field(default=None, strict=False)
@@ -24,6 +26,7 @@ class PipeImgGenBlueprint(PipeBlueprint):
is_raw: Optional[bool] = None
seed: Optional[Union[int, Literal["auto"]]] = None
nb_output: Optional[int] = Field(default=None, ge=1)
+ img_gen_prompt_var_name: Optional[str] = None
@model_validator(mode="after")
def validate_imgg_prompt_and_imgg_prompt_stuff_name(self) -> Self:
@@ -37,7 +40,7 @@ class PipeImgGenBlueprint(PipeBlueprint):
return self
-class PipeImgGenFactory(PipeSpecificFactoryProtocol[PipeImgGenBlueprint, PipeImgGen]):
+class PipeImgGenFactory(PipeFactoryProtocol[PipeImgGenBlueprint, PipeImgGen]):
@classmethod
@override
def make_pipe_from_blueprint(
@@ -51,7 +54,7 @@ class PipeImgGenFactory(PipeSpecificFactoryProtocol[PipeImgGenBlueprint, PipeImg
domain=domain_code,
code=pipe_code,
definition=pipe_blueprint.definition,
- inputs=PipeInputSpec.make_from_dict(concepts_dict=pipe_blueprint.inputs or {}),
+ inputs=PipeInputSpec.make_from_blueprint(domain=domain_code, blueprint=pipe_blueprint.inputs or {}),
output_concept_code=pipe_blueprint.output,
output_multiplicity=output_multiplicity,
imgg_prompt=pipe_blueprint.img_gen_prompt,
@@ -63,19 +66,5 @@ class PipeImgGenFactory(PipeSpecificFactoryProtocol[PipeImgGenBlueprint, PipeImg
safety_tolerance=pipe_blueprint.safety_tolerance,
is_raw=pipe_blueprint.is_raw,
seed=pipe_blueprint.seed,
- )
-
- @classmethod
- @override
- def make_pipe_from_details_dict(
- cls,
- domain_code: str,
- pipe_code: str,
- details_dict: Dict[str, Any],
- ) -> PipeImgGen:
- pipe_blueprint = PipeImgGenBlueprint.model_validate(details_dict)
- return cls.make_pipe_from_blueprint(
- domain_code=domain_code,
- pipe_code=pipe_code,
- pipe_blueprint=pipe_blueprint,
+ img_gen_prompt_var_name=pipe_blueprint.img_gen_prompt_var_name,
)
diff --git a/pipelex/pipe_operators/pipe_jinja2.py b/pipelex/pipe_operators/pipe_jinja2.py
index 8cb3d3d69..b4376ef4c 100644
--- a/pipelex/pipe_operators/pipe_jinja2.py
+++ b/pipelex/pipe_operators/pipe_jinja2.py
@@ -9,14 +9,14 @@ from pipelex import log
from pipelex.cogt.content_generation.content_generator_dry import ContentGeneratorDry
from pipelex.cogt.content_generation.content_generator_protocol import ContentGeneratorProtocol
from pipelex.config import get_config
-from pipelex.core.concept_native import NativeConcept
-from pipelex.core.pipe_input_spec import PipeInputSpec
-from pipelex.core.pipe_output import PipeOutput
-from pipelex.core.pipe_run_params import PipeRunMode, PipeRunParams
-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory
-from pipelex.core.stuff import Stuff
-from pipelex.core.stuff_content import TextContent
-from pipelex.core.working_memory import WorkingMemory
+from pipelex.core.concepts.concept_native import NativeConcept
+from pipelex.core.memory.working_memory import WorkingMemory
+from pipelex.core.pipes.pipe_input_spec import PipeInputSpec
+from pipelex.core.pipes.pipe_output import PipeOutput
+from pipelex.core.pipes.pipe_run_params import PipeRunMode, PipeRunParams
+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory
+from pipelex.core.stuffs.stuff import Stuff
+from pipelex.core.stuffs.stuff_content import TextContent
from pipelex.exceptions import PipeDefinitionError, PipeRunParamsError
from pipelex.hub import get_content_generator, get_template, get_template_provider
from pipelex.pipe_operators.pipe_operator import PipeOperator
diff --git a/pipelex/pipe_operators/pipe_jinja2_factory.py b/pipelex/pipe_operators/pipe_jinja2_factory.py
index 13597ff3f..228ec13ad 100644
--- a/pipelex/pipe_operators/pipe_jinja2_factory.py
+++ b/pipelex/pipe_operators/pipe_jinja2_factory.py
@@ -1,10 +1,11 @@
-from typing import Any, Dict, Optional
+from typing import Literal, Optional
from typing_extensions import override
from pipelex.config import get_config
-from pipelex.core.pipe_blueprint import PipeBlueprint, PipeSpecificFactoryProtocol
-from pipelex.core.pipe_input_spec import PipeInputSpec
+from pipelex.core.pipes.pipe_blueprint import PipeBlueprint
+from pipelex.core.pipes.pipe_factory import PipeFactoryProtocol
+from pipelex.core.pipes.pipe_input_spec import PipeInputSpec
from pipelex.exceptions import PipeDefinitionError
from pipelex.pipe_operators.pipe_jinja2 import PipeJinja2
from pipelex.tools.templating.jinja2_parsing import check_jinja2_parsing
@@ -14,13 +15,14 @@ from pipelex.tools.templating.templating_models import PromptingStyle
class PipeJinja2Blueprint(PipeBlueprint):
+ type: Literal["PipeJinja2"] = "PipeJinja2"
jinja2_name: Optional[str] = None
jinja2: Optional[str] = None
prompting_style: Optional[PromptingStyle] = None
template_category: Jinja2TemplateCategory = Jinja2TemplateCategory.LLM_PROMPT
-class PipeJinja2Factory(PipeSpecificFactoryProtocol[PipeJinja2Blueprint, PipeJinja2]):
+class PipeJinja2Factory(PipeFactoryProtocol[PipeJinja2Blueprint, PipeJinja2]):
@classmethod
@override
def make_pipe_from_blueprint(
@@ -42,7 +44,7 @@ class PipeJinja2Factory(PipeSpecificFactoryProtocol[PipeJinja2Blueprint, PipeJin
domain=domain_code,
code=pipe_code,
definition=pipe_blueprint.definition,
- inputs=PipeInputSpec.make_from_dict(concepts_dict=pipe_blueprint.inputs or {}),
+ inputs=PipeInputSpec.make_from_blueprint(domain=domain_code, blueprint=pipe_blueprint.inputs or {}),
output_concept_code=pipe_blueprint.output,
jinja2_name=pipe_blueprint.jinja2_name,
jinja2=preprocessed_template,
@@ -50,21 +52,6 @@ class PipeJinja2Factory(PipeSpecificFactoryProtocol[PipeJinja2Blueprint, PipeJin
template_category=pipe_blueprint.template_category,
)
- @classmethod
- @override
- def make_pipe_from_details_dict(
- cls,
- domain_code: str,
- pipe_code: str,
- details_dict: Dict[str, Any],
- ) -> PipeJinja2:
- pipe_blueprint = PipeJinja2Blueprint.model_validate(details_dict)
- return cls.make_pipe_from_blueprint(
- domain_code=domain_code,
- pipe_code=pipe_code,
- pipe_blueprint=pipe_blueprint,
- )
-
@classmethod
def make_pipe_jinja2_from_template_str(
cls,
diff --git a/pipelex/pipe_operators/pipe_llm.py b/pipelex/pipe_operators/pipe_llm.py
index 62c2294f8..54db47d07 100644
--- a/pipelex/pipe_operators/pipe_llm.py
+++ b/pipelex/pipe_operators/pipe_llm.py
@@ -11,21 +11,21 @@ from pipelex.cogt.llm.llm_models.llm_setting import LLMSetting, LLMSettingChoice
from pipelex.cogt.llm.llm_prompt import LLMPrompt
from pipelex.cogt.llm.llm_prompt_factory_abstract import LLMPromptFactoryAbstract
from pipelex.config import StaticValidationReaction, get_config
-from pipelex.core.concept import Concept
-from pipelex.core.concept_code_factory import ConceptCodeFactory
-from pipelex.core.concept_native import NativeConcept, NativeConceptClass
-from pipelex.core.domain import Domain, SpecialDomain
-from pipelex.core.pipe_input_spec import PipeInputSpec
-from pipelex.core.pipe_output import PipeOutput
-from pipelex.core.pipe_run_params import (
+from pipelex.core.concepts.concept import Concept
+from pipelex.core.concepts.concept_code_factory import ConceptCodeFactory
+from pipelex.core.concepts.concept_native import NativeConcept, NativeConceptClass
+from pipelex.core.domains.domain import Domain, SpecialDomain
+from pipelex.core.memory.working_memory import WorkingMemory
+from pipelex.core.pipes.pipe_input_spec import PipeInputSpec
+from pipelex.core.pipes.pipe_output import PipeOutput
+from pipelex.core.pipes.pipe_run_params import (
PipeOutputMultiplicity,
PipeRunParamKey,
PipeRunParams,
output_multiplicity_to_apply,
)
-from pipelex.core.stuff_content import ListContent, StructuredContent, StuffContent, TextContent
-from pipelex.core.stuff_factory import StuffFactory
-from pipelex.core.working_memory import WorkingMemory
+from pipelex.core.stuffs.stuff_content import ListContent, StructuredContent, StuffContent, TextContent
+from pipelex.core.stuffs.stuff_factory import StuffFactory
from pipelex.exceptions import (
PipeDefinitionError,
PipeInputError,
diff --git a/pipelex/pipe_operators/pipe_llm_factory.py b/pipelex/pipe_operators/pipe_llm_factory.py
index 69d6d420c..13ebe68c0 100644
--- a/pipelex/pipe_operators/pipe_llm_factory.py
+++ b/pipelex/pipe_operators/pipe_llm_factory.py
@@ -1,12 +1,13 @@
-from typing import Any, Dict, List, Optional
+from typing import List, Literal, Optional
from pydantic import model_validator
from typing_extensions import Self, override
from pipelex.cogt.llm.llm_models.llm_setting import LLMSettingChoices, LLMSettingOrPresetId
-from pipelex.core.pipe_blueprint import PipeBlueprint, PipeSpecificFactoryProtocol
-from pipelex.core.pipe_input_spec import PipeInputSpec
-from pipelex.core.pipe_run_params import make_output_multiplicity
+from pipelex.core.pipes.pipe_blueprint import PipeBlueprint
+from pipelex.core.pipes.pipe_factory import PipeFactoryProtocol
+from pipelex.core.pipes.pipe_input_spec import PipeInputSpec
+from pipelex.core.pipes.pipe_run_params import make_output_multiplicity
from pipelex.exceptions import PipeDefinitionError
from pipelex.hub import get_concept_provider, get_optional_domain
from pipelex.pipe_operators.pipe_jinja2 import PipeJinja2
@@ -19,6 +20,7 @@ from pipelex.tools.typing.validation_utils import has_more_than_one_among_attrib
class PipeLLMBlueprint(PipeBlueprint):
+ type: Literal["PipeLLM"] = "PipeLLM"
system_prompt_template: Optional[str] = None
system_prompt_template_name: Optional[str] = None
system_prompt_name: Optional[str] = None
@@ -53,7 +55,7 @@ class PipeLLMBlueprint(PipeBlueprint):
return self
-class PipeLLMFactory(PipeSpecificFactoryProtocol[PipeLLMBlueprint, PipeLLM]):
+class PipeLLMFactory(PipeFactoryProtocol[PipeLLMBlueprint, PipeLLM]):
@classmethod
@override
def make_pipe_from_blueprint(
@@ -91,7 +93,7 @@ class PipeLLMFactory(PipeSpecificFactoryProtocol[PipeLLMBlueprint, PipeLLM]):
domain_code=domain_code,
template_str=pipe_blueprint.prompt_template,
template_name=pipe_blueprint.template_name,
- inputs=PipeInputSpec.make_from_dict(pipe_blueprint.inputs) if pipe_blueprint.inputs else PipeInputSpec.make_empty(),
+ inputs=PipeInputSpec.make_from_blueprint(domain=domain_code, blueprint=pipe_blueprint.inputs or {}),
)
except Jinja2TemplateError as exc:
error_msg = f"Jinja2 syntax error in user prompt for pipe '{pipe_code}' in domain '{domain_code}': {exc}."
@@ -114,8 +116,8 @@ class PipeLLMFactory(PipeSpecificFactoryProtocol[PipeLLMBlueprint, PipeLLM]):
user_images: List[str] = []
if pipe_blueprint.inputs:
- for stuff_name, concept_code in (pipe_blueprint.inputs).items():
- concept = get_concept_provider().get_required_concept(concept_code=concept_code)
+ for stuff_name, requirement in (pipe_blueprint.inputs).items():
+ concept = get_concept_provider().get_required_concept(concept_code=requirement.concept_code)
if get_concept_provider().is_image_concept(concept_code=concept.code):
user_images.append(stuff_name)
else:
@@ -124,7 +126,7 @@ class PipeLLMFactory(PipeSpecificFactoryProtocol[PipeLLMBlueprint, PipeLLM]):
pipe_llm_prompt = PipeLLMPrompt(
code="adhoc_for_pipe_llm_prompt",
domain=domain_code,
- inputs=PipeInputSpec.make_from_dict(concepts_dict=pipe_blueprint.inputs or {}),
+ inputs=PipeInputSpec.make_from_blueprint(domain=domain_code, blueprint=pipe_blueprint.inputs or {}),
system_prompt_pipe_jinja2=system_prompt_pipe_jinja2,
system_prompt_verbatim_name=pipe_blueprint.system_prompt_name,
system_prompt=pipe_blueprint.system_prompt or system_prompt,
@@ -149,7 +151,7 @@ class PipeLLMFactory(PipeSpecificFactoryProtocol[PipeLLMBlueprint, PipeLLM]):
domain=domain_code,
code=pipe_code,
definition=pipe_blueprint.definition,
- inputs=PipeInputSpec.make_from_dict(concepts_dict=pipe_blueprint.inputs or {}),
+ inputs=PipeInputSpec.make_from_blueprint(domain=domain_code, blueprint=pipe_blueprint.inputs or {}),
output_concept_code=pipe_blueprint.output,
pipe_llm_prompt=pipe_llm_prompt,
llm_choices=llm_choices,
@@ -158,18 +160,3 @@ class PipeLLMFactory(PipeSpecificFactoryProtocol[PipeLLMBlueprint, PipeLLM]):
system_prompt_to_structure=pipe_blueprint.system_prompt_to_structure,
output_multiplicity=output_multiplicity,
)
-
- @classmethod
- @override
- def make_pipe_from_details_dict(
- cls,
- domain_code: str,
- pipe_code: str,
- details_dict: Dict[str, Any],
- ) -> PipeLLM:
- pipe_blueprint = PipeLLMBlueprint.model_validate(details_dict)
- return cls.make_pipe_from_blueprint(
- domain_code=domain_code,
- pipe_code=pipe_code,
- pipe_blueprint=pipe_blueprint,
- )
diff --git a/pipelex/pipe_operators/pipe_llm_prompt.py b/pipelex/pipe_operators/pipe_llm_prompt.py
index 7f65b58e4..612b87536 100644
--- a/pipelex/pipe_operators/pipe_llm_prompt.py
+++ b/pipelex/pipe_operators/pipe_llm_prompt.py
@@ -7,15 +7,15 @@ from pipelex import log
from pipelex.cogt.image.prompt_image import PromptImage
from pipelex.cogt.image.prompt_image_factory import PromptImageFactory
from pipelex.cogt.llm.llm_prompt import LLMPrompt
-from pipelex.core.concept import Concept
-from pipelex.core.concept_native import NativeConcept
-from pipelex.core.pipe_input_spec import PipeInputSpec
-from pipelex.core.pipe_output import PipeOutput
-from pipelex.core.pipe_run_params import PipeRunMode, PipeRunParams
-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory
-from pipelex.core.stuff_content import ImageContent, LLMPromptContent, StuffContent
-from pipelex.core.stuff_factory import StuffFactory
-from pipelex.core.working_memory import WorkingMemory
+from pipelex.core.concepts.concept import Concept
+from pipelex.core.concepts.concept_native import NativeConcept
+from pipelex.core.memory.working_memory import WorkingMemory
+from pipelex.core.pipes.pipe_input_spec import PipeInputSpec
+from pipelex.core.pipes.pipe_output import PipeOutput
+from pipelex.core.pipes.pipe_run_params import PipeRunMode, PipeRunParams
+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory
+from pipelex.core.stuffs.stuff_content import ImageContent, LLMPromptContent, StuffContent
+from pipelex.core.stuffs.stuff_factory import StuffFactory
from pipelex.exceptions import (
PipeDefinitionError,
PipeInputError,
diff --git a/pipelex/pipe_operators/pipe_ocr.py b/pipelex/pipe_operators/pipe_ocr.py
index b2eb568d3..7746c436e 100644
--- a/pipelex/pipe_operators/pipe_ocr.py
+++ b/pipelex/pipe_operators/pipe_ocr.py
@@ -11,13 +11,13 @@ from pipelex.cogt.ocr.ocr_handle import OcrHandle
from pipelex.cogt.ocr.ocr_input import OcrInput
from pipelex.cogt.ocr.ocr_job_components import OcrJobConfig, OcrJobParams
from pipelex.config import StaticValidationReaction, get_config
-from pipelex.core.concept_native import NativeConcept
-from pipelex.core.pipe_input_spec import PipeInputSpec
-from pipelex.core.pipe_output import PipeOutput
-from pipelex.core.pipe_run_params import PipeRunMode, PipeRunParams
-from pipelex.core.stuff_content import ImageContent, ListContent, PageContent, TextAndImagesContent, TextContent
-from pipelex.core.stuff_factory import StuffFactory
-from pipelex.core.working_memory import WorkingMemory
+from pipelex.core.concepts.concept_native import NativeConcept
+from pipelex.core.memory.working_memory import WorkingMemory
+from pipelex.core.pipes.pipe_input_spec import InputRequirementBlueprint, PipeInputSpec
+from pipelex.core.pipes.pipe_output import PipeOutput
+from pipelex.core.pipes.pipe_run_params import PipeRunMode, PipeRunParams
+from pipelex.core.stuffs.stuff_content import ImageContent, ListContent, PageContent, TextAndImagesContent, TextContent
+from pipelex.core.stuffs.stuff_factory import StuffFactory
from pipelex.exceptions import (
PipeDefinitionError,
StaticValidationError,
@@ -98,6 +98,7 @@ class PipeOcr(PipeOperator):
log.error(inadequate_input_concept_error.desc())
case StaticValidationReaction.RAISE:
raise inadequate_input_concept_error
+
if len(candidate_prompt_var_names) > 1:
too_many_candidate_inputs_error = StaticValidationError(
error_type=StaticValidationErrorType.TOO_MANY_CANDIDATE_INPUTS,
@@ -130,7 +131,10 @@ class PipeOcr(PipeOperator):
@override
def needed_inputs(self) -> PipeInputSpec:
- return PipeInputSpec.make_from_dict({PIPE_OCR_INPUT_NAME: self.inputs.root[PIPE_OCR_INPUT_NAME].concept_code})
+ return PipeInputSpec.make_from_blueprint(
+ domain=self.domain,
+ blueprint={PIPE_OCR_INPUT_NAME: InputRequirementBlueprint(concept_code=self.inputs.root[PIPE_OCR_INPUT_NAME].concept_code)},
+ )
@override
async def _run_operator_pipe(
diff --git a/pipelex/pipe_operators/pipe_ocr_factory.py b/pipelex/pipe_operators/pipe_ocr_factory.py
index ad9d015a9..621a7b7d8 100644
--- a/pipelex/pipe_operators/pipe_ocr_factory.py
+++ b/pipelex/pipe_operators/pipe_ocr_factory.py
@@ -1,4 +1,4 @@
-from typing import Any, Dict, Optional
+from typing import Literal, Optional
from typing_extensions import override
@@ -6,21 +6,22 @@ from pipelex.cogt.ocr.ocr_engine_factory import OcrEngineFactory
from pipelex.cogt.ocr.ocr_handle import OcrHandle
from pipelex.cogt.ocr.ocr_platform import OcrPlatform
from pipelex.config import get_config
-from pipelex.core.pipe_blueprint import PipeBlueprint, PipeSpecificFactoryProtocol
-from pipelex.core.pipe_input_spec import PipeInputSpec
+from pipelex.core.pipes.pipe_blueprint import PipeBlueprint
+from pipelex.core.pipes.pipe_factory import PipeFactoryProtocol
+from pipelex.core.pipes.pipe_input_spec import PipeInputSpec
from pipelex.pipe_operators.pipe_ocr import PipeOcr
class PipeOcrBlueprint(PipeBlueprint):
- definition: Optional[str] = None
+ type: Literal["PipeOcr"] = "PipeOcr"
ocr_platform: Optional[OcrPlatform] = None
- page_images: bool = False
- page_image_captions: bool = False
- page_views: bool = False
+ page_images: Optional[bool] = None
+ page_image_captions: Optional[bool] = None
+ page_views: Optional[bool] = None
page_views_dpi: Optional[int] = None
-class PipeOcrFactory(PipeSpecificFactoryProtocol[PipeOcrBlueprint, PipeOcr]):
+class PipeOcrFactory(PipeFactoryProtocol[PipeOcrBlueprint, PipeOcr]):
@classmethod
@override
def make_pipe_from_blueprint(
@@ -40,24 +41,9 @@ class PipeOcrFactory(PipeSpecificFactoryProtocol[PipeOcrBlueprint, PipeOcr]):
definition=pipe_blueprint.definition,
ocr_engine=ocr_engine,
output_concept_code=pipe_blueprint.output,
- inputs=PipeInputSpec.make_from_dict(concepts_dict=pipe_blueprint.inputs or {}),
- should_include_images=pipe_blueprint.page_images,
- should_caption_images=pipe_blueprint.page_image_captions,
- should_include_page_views=pipe_blueprint.page_views,
+ inputs=PipeInputSpec.make_from_blueprint(domain=domain_code, blueprint=pipe_blueprint.inputs or {}),
+ should_include_images=pipe_blueprint.page_images or False,
+ should_caption_images=pipe_blueprint.page_image_captions or False,
+ should_include_page_views=pipe_blueprint.page_views or False,
page_views_dpi=pipe_blueprint.page_views_dpi or get_config().cogt.ocr_config.default_page_views_dpi,
)
-
- @classmethod
- @override
- def make_pipe_from_details_dict(
- cls,
- domain_code: str,
- pipe_code: str,
- details_dict: Dict[str, Any],
- ) -> PipeOcr:
- pipe_blueprint = PipeOcrBlueprint.model_validate(details_dict)
- return cls.make_pipe_from_blueprint(
- domain_code=domain_code,
- pipe_code=pipe_code,
- pipe_blueprint=pipe_blueprint,
- )
diff --git a/pipelex/pipe_operators/pipe_operator.py b/pipelex/pipe_operators/pipe_operator.py
index 1b4c64218..6ff71c733 100644
--- a/pipelex/pipe_operators/pipe_operator.py
+++ b/pipelex/pipe_operators/pipe_operator.py
@@ -4,10 +4,10 @@ from typing import Optional
from typing_extensions import override
from pipelex import log
-from pipelex.core.pipe_abstract import PipeAbstract
-from pipelex.core.pipe_output import PipeOutput
-from pipelex.core.pipe_run_params import PipeRunMode, PipeRunParams
-from pipelex.core.working_memory import WorkingMemory
+from pipelex.core.memory.working_memory import WorkingMemory
+from pipelex.core.pipes.pipe_abstract import PipeAbstract
+from pipelex.core.pipes.pipe_output import PipeOutput
+from pipelex.core.pipes.pipe_run_params import PipeRunMode, PipeRunParams
from pipelex.hub import get_activity_manager
from pipelex.pipeline.activity.activity_models import ActivityReport
from pipelex.pipeline.job_metadata import JobMetadata
diff --git a/pipelex/pipe_operators/piped_llm_prompt_factory.py b/pipelex/pipe_operators/piped_llm_prompt_factory.py
index 542b9d435..911c49b93 100644
--- a/pipelex/pipe_operators/piped_llm_prompt_factory.py
+++ b/pipelex/pipe_operators/piped_llm_prompt_factory.py
@@ -5,9 +5,9 @@ from typing_extensions import override
from pipelex.cogt.llm.llm_prompt import LLMPrompt
from pipelex.cogt.llm.llm_prompt_factory_abstract import LLMPromptFactoryAbstract, make_empty_prompt
from pipelex.cogt.llm.llm_prompt_template_inputs import LLMPromptTemplateInputs
-from pipelex.core.pipe_output import PipeOutput
-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory
-from pipelex.core.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.pipes.pipe_output import PipeOutput
+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory
from pipelex.pipe_operators.pipe_llm_prompt import PipeLLMPrompt, PipeLLMPromptOutput
from pipelex.pipeline.job_metadata import JobMetadata
diff --git a/pipelex/pipe_works/pipe_dry.py b/pipelex/pipe_works/pipe_dry.py
index 01be2580a..278313f41 100644
--- a/pipelex/pipe_works/pipe_dry.py
+++ b/pipelex/pipe_works/pipe_dry.py
@@ -6,12 +6,12 @@ from typing import Dict, List, Tuple
from pipelex import log
from pipelex.config import get_config
-from pipelex.core.pipe_abstract import PipeAbstract
-from pipelex.core.pipe_input_spec import PipeInputSpec, TypedNamedInputRequirement
-from pipelex.core.pipe_run_params import PipeRunMode
-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory
-from pipelex.core.stuff_content import StuffContent, TextContent
-from pipelex.core.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.pipes.pipe_abstract import PipeAbstract
+from pipelex.core.pipes.pipe_input_spec import PipeInputSpec, TypedNamedInputRequirement
+from pipelex.core.pipes.pipe_run_params import PipeRunMode
+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory
+from pipelex.core.stuffs.stuff_content import StuffContent, TextContent
from pipelex.hub import get_class_registry, get_concept_provider, get_pipe_provider
from pipelex.pipeline.job_metadata import JobMetadata
@@ -53,6 +53,13 @@ async def dry_run_pipe_codes(pipe_codes: List[str]) -> Dict[str, str]:
return await dry_run_pipes(pipes=pipes)
+async def dry_run_pipe(pipe: PipeAbstract):
+ """
+ Dry run a pipe.
+ """
+ return await dry_run_pipes(pipes=[pipe])
+
+
async def dry_run_pipes(pipes: List[PipeAbstract]) -> Dict[str, str]:
"""
Dry run all pipes in the library using ThreadPoolExecutor for true parallelism.
diff --git a/pipelex/pipe_works/pipe_job.py b/pipelex/pipe_works/pipe_job.py
index 6454c9577..d25b0f61a 100644
--- a/pipelex/pipe_works/pipe_job.py
+++ b/pipelex/pipe_works/pipe_job.py
@@ -2,9 +2,9 @@ from typing import Optional
from pydantic import BaseModel, Field
-from pipelex.core.pipe_abstract import PipeAbstract
-from pipelex.core.pipe_run_params import PipeRunParams
-from pipelex.core.working_memory import WorkingMemory
+from pipelex.core.memory.working_memory import WorkingMemory
+from pipelex.core.pipes.pipe_abstract import PipeAbstract
+from pipelex.core.pipes.pipe_run_params import PipeRunParams
from pipelex.pipeline.job_metadata import JobMetadata
diff --git a/pipelex/pipe_works/pipe_job_factory.py b/pipelex/pipe_works/pipe_job_factory.py
index e1f351d85..73340757e 100644
--- a/pipelex/pipe_works/pipe_job_factory.py
+++ b/pipelex/pipe_works/pipe_job_factory.py
@@ -1,10 +1,10 @@
from typing import Optional
-from pipelex.core.pipe_abstract import PipeAbstract
-from pipelex.core.pipe_run_params import PipeRunParams
-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory
-from pipelex.core.working_memory import WorkingMemory
-from pipelex.core.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.memory.working_memory import WorkingMemory
+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.pipes.pipe_abstract import PipeAbstract
+from pipelex.core.pipes.pipe_run_params import PipeRunParams
+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory
from pipelex.pipe_works.pipe_job import PipeJob
from pipelex.pipeline.job_metadata import JobMetadata
diff --git a/pipelex/pipe_works/pipe_router.py b/pipelex/pipe_works/pipe_router.py
index d4339ed77..a581c66de 100644
--- a/pipelex/pipe_works/pipe_router.py
+++ b/pipelex/pipe_works/pipe_router.py
@@ -3,9 +3,9 @@ from typing import Optional, cast
from typing_extensions import override
from pipelex import log
-from pipelex.core.pipe_output import PipeOutputType
-from pipelex.core.pipe_run_params import PipeRunParams
-from pipelex.core.working_memory import WorkingMemory
+from pipelex.core.memory.working_memory import WorkingMemory
+from pipelex.core.pipes.pipe_output import PipeOutputType
+from pipelex.core.pipes.pipe_run_params import PipeRunParams
from pipelex.hub import get_required_pipe
from pipelex.pipe_works.pipe_job import PipeJob
from pipelex.pipe_works.pipe_job_factory import PipeJobFactory
diff --git a/pipelex/pipe_works/pipe_router_protocol.py b/pipelex/pipe_works/pipe_router_protocol.py
index ee014f4ee..b58d8c5d8 100644
--- a/pipelex/pipe_works/pipe_router_protocol.py
+++ b/pipelex/pipe_works/pipe_router_protocol.py
@@ -1,8 +1,8 @@
from typing import Optional, Protocol
-from pipelex.core.pipe_output import PipeOutputType
-from pipelex.core.pipe_run_params import PipeRunParams
-from pipelex.core.working_memory import WorkingMemory
+from pipelex.core.memory.working_memory import WorkingMemory
+from pipelex.core.pipes.pipe_output import PipeOutputType
+from pipelex.core.pipes.pipe_run_params import PipeRunParams
from pipelex.pipe_works.pipe_job import PipeJob
from pipelex.pipeline.job_metadata import JobMetadata
diff --git a/pipelex/pipelex.py b/pipelex/pipelex.py
index 068171963..ee74a1c9d 100644
--- a/pipelex/pipelex.py
+++ b/pipelex/pipelex.py
@@ -21,13 +21,13 @@ from pipelex.cogt.inference.inference_manager import InferenceManager
from pipelex.cogt.llm.llm_models.llm_model import LATEST_VERSION_NAME
from pipelex.cogt.llm.llm_models.llm_model_library import LLMModelLibrary
from pipelex.config import PipelexConfig, get_config
-from pipelex.core.concept_library import ConceptLibrary
-from pipelex.core.domain_library import DomainLibrary
-from pipelex.core.pipe_library import PipeLibrary
+from pipelex.core.concepts.concept_library import ConceptLibrary
+from pipelex.core.domains.domain_library import DomainLibrary
+from pipelex.core.pipes.pipe_library import PipeLibrary
from pipelex.core.registry_models import PipelexRegistryModels
from pipelex.exceptions import PipelexConfigError, PipelexSetupError
from pipelex.hub import PipelexHub, set_pipelex_hub
-from pipelex.libraries.library_manager import LibraryManager
+from pipelex.libraries.library_manager_factory import LibraryManagerFactory
from pipelex.pipe_works.pipe_router import PipeRouter
from pipelex.pipe_works.pipe_router_protocol import PipeRouterProtocol
from pipelex.pipeline.activity.activity_manager import ActivityManager
@@ -61,7 +61,7 @@ PACKAGE_VERSION = metadata(PACKAGE_NAME)["Version"]
class Pipelex(metaclass=MetaSingleton):
def __init__(
self,
- config_folder_path: str,
+ config_dir_path: str,
# Dependency injection
pipelex_hub: Optional[PipelexHub] = None,
config_cls: Optional[Type[ConfigRoot]] = None,
@@ -74,7 +74,7 @@ class Pipelex(metaclass=MetaSingleton):
activity_manager: Optional[ActivityManagerProtocol] = None,
reporting_delegate: Optional[ReportingProtocol] = None,
) -> None:
- self.config_folder_path = config_folder_path
+ self.config_dir_path = config_dir_path
self.pipelex_hub = pipelex_hub or PipelexHub()
set_pipelex_hub(self.pipelex_hub)
@@ -95,7 +95,7 @@ class Pipelex(metaclass=MetaSingleton):
log.debug("Logs are configured")
# tools
- self.template_provider = template_provider or TemplateLibrary.make_empty(config_folder_path=config_folder_path)
+ self.template_provider = template_provider or TemplateLibrary.make_empty(config_dir_path=config_dir_path)
self.pipelex_hub.set_template_provider(self.template_provider)
self.class_registry = class_registry or ClassRegistry()
@@ -105,7 +105,7 @@ class Pipelex(metaclass=MetaSingleton):
# cogt
self.plugin_manager = PluginManager()
self.pipelex_hub.set_plugin_manager(self.plugin_manager)
- self.llm_model_provider = llm_model_provider or LLMModelLibrary.make_empty(config_folder_path=config_folder_path)
+ self.llm_model_provider = llm_model_provider or LLMModelLibrary.make_empty(config_dir_path=config_dir_path)
self.pipelex_hub.set_llm_models_provider(self.llm_model_provider)
self.inference_manager = inference_manager or InferenceManager()
self.pipelex_hub.set_inference_manager(self.inference_manager)
@@ -124,13 +124,13 @@ class Pipelex(metaclass=MetaSingleton):
self.pipelex_hub.set_domain_provider(domain_provider=domain_library)
self.pipelex_hub.set_concept_provider(concept_provider=concept_library)
self.pipelex_hub.set_pipe_provider(pipe_provider=pipe_library)
- self.library_manager = LibraryManager.make(
+
+ self.library_manager = LibraryManagerFactory.make(
domain_library=domain_library,
concept_library=concept_library,
pipe_library=pipe_library,
- config_folder_path=config_folder_path,
+ config_dir_path=config_dir_path,
)
- self.library_manager.setup()
self.pipelex_hub.set_library_manager(library_manager=self.library_manager)
# pipelex pipeline
@@ -188,6 +188,7 @@ class Pipelex(metaclass=MetaSingleton):
try:
self.template_provider.setup()
self.llm_model_provider.setup()
+ self.library_manager.setup()
llm_deck = self.library_manager.load_deck()
for llm_model in self.llm_model_provider.get_all_llm_models():
if llm_model.version == LATEST_VERSION_NAME:
@@ -277,15 +278,15 @@ class Pipelex(metaclass=MetaSingleton):
raise PipelexSetupError("Could not find relative config folder path because of: Failed to get caller frame")
caller_file = current_frame.f_back.f_code.co_filename
caller_dir = os.path.dirname(os.path.abspath(caller_file))
- config_folder_path = os.path.abspath(os.path.join(caller_dir, relative_config_folder_path))
+ config_dir_path = os.path.abspath(os.path.join(caller_dir, relative_config_folder_path))
else:
- config_folder_path = os.path.abspath(os.path.join(os.getcwd(), relative_config_folder_path))
+ config_dir_path = os.path.abspath(os.path.join(os.getcwd(), relative_config_folder_path))
elif absolute_config_folder_path is not None:
- config_folder_path = absolute_config_folder_path
+ config_dir_path = absolute_config_folder_path
else:
- config_folder_path = "./pipelex_libraries"
+ config_dir_path = "./pipelex_libraries"
- pipelex_instance = cls(config_folder_path=config_folder_path)
+ pipelex_instance = cls(config_dir_path=config_dir_path)
pipelex_instance.setup()
pipelex_instance.setup_libraries()
log.info(f"Pipelex {PACKAGE_VERSION} initialized.")
diff --git a/pipelex/pipelex.toml b/pipelex/pipelex.toml
index 8aeaf2513..b9ca86d27 100644
--- a/pipelex/pipelex.toml
+++ b/pipelex/pipelex.toml
@@ -205,7 +205,7 @@ apply_to_jinja2_rendering = false
text_gen_truncate_length = 256
nb_list_items = 3
nb_ocr_pages = 4
-allowed_to_fail_pipes = ["infinite_loop_1"]
+allowed_to_fail_pipes = ["infinite_loop_1", "build_blueprint", "structure_blueprint"]
image_urls = [
"https://storage.googleapis.com/public_test_files_7fa6_4277_9ab/fashion/fashion_photo_1.jpg",
"https://storage.googleapis.com/public_test_files_7fa6_4277_9ab/fashion/fashion_photo_2.png",
diff --git a/pipelex/pipeline/activity/activity_handler.py b/pipelex/pipeline/activity/activity_handler.py
index 67e3929bb..2d9a38396 100644
--- a/pipelex/pipeline/activity/activity_handler.py
+++ b/pipelex/pipeline/activity/activity_handler.py
@@ -3,8 +3,8 @@ from typing import Set, cast
from pipelex import log
from pipelex.config import get_config
-from pipelex.core.stuff import Stuff
-from pipelex.core.stuff_content import (
+from pipelex.core.stuffs.stuff import Stuff
+from pipelex.core.stuffs.stuff_content import (
HtmlContent,
ImageContent,
ListContent,
diff --git a/pipelex/pipeline/execute.py b/pipelex/pipeline/execute.py
index 79981e5f4..d34b19253 100644
--- a/pipelex/pipeline/execute.py
+++ b/pipelex/pipeline/execute.py
@@ -1,15 +1,15 @@
from typing import List, Optional
from pipelex.client.protocol import ImplicitMemory
-from pipelex.core.pipe_output import PipeOutput
-from pipelex.core.pipe_run_params import (
+from pipelex.core.memory.working_memory import WorkingMemory
+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.pipes.pipe_output import PipeOutput
+from pipelex.core.pipes.pipe_run_params import (
FORCE_DRY_RUN_MODE_ENV_KEY,
PipeOutputMultiplicity,
PipeRunMode,
)
-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory
-from pipelex.core.working_memory import WorkingMemory
-from pipelex.core.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory
from pipelex.exceptions import PipelineInputError
from pipelex.hub import (
get_pipe_router,
diff --git a/pipelex/pipeline/start.py b/pipelex/pipeline/start.py
index 85027c1e2..995bd72fd 100644
--- a/pipelex/pipeline/start.py
+++ b/pipelex/pipeline/start.py
@@ -2,11 +2,11 @@ import asyncio
from typing import Optional
from pipelex.client.protocol import CompactMemory
-from pipelex.core.pipe_output import PipeOutput
-from pipelex.core.pipe_run_params import PipeOutputMultiplicity, PipeRunMode
-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory
-from pipelex.core.working_memory import WorkingMemory
-from pipelex.core.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.memory.working_memory import WorkingMemory
+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.pipes.pipe_output import PipeOutput
+from pipelex.core.pipes.pipe_run_params import PipeOutputMultiplicity, PipeRunMode
+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory
from pipelex.exceptions import StartPipelineException
from pipelex.hub import get_pipe_router, get_pipeline_manager, get_report_delegate, get_required_pipe
from pipelex.pipe_works.pipe_job_factory import PipeJobFactory
diff --git a/pipelex/pipeline/track/pipeline_tracker.py b/pipelex/pipeline/track/pipeline_tracker.py
index 0486e4719..3cb99fb9f 100644
--- a/pipelex/pipeline/track/pipeline_tracker.py
+++ b/pipelex/pipeline/track/pipeline_tracker.py
@@ -8,8 +8,8 @@ import networkx as nx
from typing_extensions import override
from pipelex import log
-from pipelex.core.concept import Concept
-from pipelex.core.stuff import Stuff
+from pipelex.core.concepts.concept import Concept
+from pipelex.core.stuffs.stuff import Stuff
from pipelex.exceptions import JobHistoryError
from pipelex.pipe_controllers.pipe_condition_details import PipeConditionDetails
from pipelex.pipeline.track.flow_chart import PipelineFlowChart
@@ -45,6 +45,11 @@ class PipelineTracker(PipelineTrackerProtocol):
self.nx_graph = nx.DiGraph()
self.start_node = None
+ @override
+ def reset(self):
+ self.teardown()
+ self.setup()
+
def _get_node_name(self, node: str) -> Optional[str]:
node_attributes = self.nx_graph.nodes[node]
node_name = node_attributes[NodeAttributeKey.NAME]
@@ -332,10 +337,10 @@ class PipelineTracker(PipelineTrackerProtocol):
title_to_print += f" for {title}"
print_mermaid_url(url=url, title=title_to_print)
- def _print_mermaid_flowchart_url(self, title: Optional[str] = None, subtitle: Optional[str] = None):
+ def _print_mermaid_flowchart_url(self, title: Optional[str] = None, subtitle: Optional[str] = None) -> Optional[str]:
if not self.nx_graph.nodes:
log.info("No nodes in the pipeline tracker")
- return
+ return None
if self.start_node is None:
raise JobHistoryError("Start node is not set")
flowchart = PipelineFlowChart(nx_graph=self.nx_graph, start_node=self.start_node, tracker_config=self._tracker_config)
@@ -344,6 +349,7 @@ class PipelineTracker(PipelineTrackerProtocol):
if title:
title_to_print += f" for {title}"
print_mermaid_url(url=url, title=title_to_print)
+ return url
@override
def output_flowchart(
@@ -351,8 +357,9 @@ class PipelineTracker(PipelineTrackerProtocol):
title: Optional[str] = None,
subtitle: Optional[str] = None,
is_detailed: bool = False,
- ):
+ ) -> Optional[str]:
if is_detailed:
self._print_mermaid_flowchart_code_and_url(title=title, subtitle=subtitle)
else:
- self._print_mermaid_flowchart_url(title=title, subtitle=subtitle)
+ return self._print_mermaid_flowchart_url(title=title, subtitle=subtitle)
+ return None
diff --git a/pipelex/pipeline/track/pipeline_tracker_protocol.py b/pipelex/pipeline/track/pipeline_tracker_protocol.py
index 95cb08528..1228130d9 100644
--- a/pipelex/pipeline/track/pipeline_tracker_protocol.py
+++ b/pipelex/pipeline/track/pipeline_tracker_protocol.py
@@ -6,7 +6,7 @@ from typing import List, Optional, Protocol
from typing_extensions import override
-from pipelex.core.stuff import Stuff
+from pipelex.core.stuffs.stuff import Stuff
from pipelex.pipe_controllers.pipe_condition_details import PipeConditionDetails
@@ -15,7 +15,8 @@ class PipelineTrackerProtocol(Protocol):
def teardown(self): ...
- # def reset(self): ...
+ def reset(self): ...
+
def add_pipe_step(
self,
from_stuff: Optional[Stuff],
@@ -66,7 +67,7 @@ class PipelineTrackerProtocol(Protocol):
title: Optional[str] = None,
subtitle: Optional[str] = None,
is_detailed: bool = False,
- ): ...
+ ) -> Optional[str]: ...
class PipelineTrackerNoOp(PipelineTrackerProtocol):
@@ -82,6 +83,10 @@ class PipelineTrackerNoOp(PipelineTrackerProtocol):
def teardown(self) -> None:
pass
+ @override
+ def reset(self) -> None:
+ pass
+
@override
def add_pipe_step(
self,
diff --git a/pipelex/test_extras/registry_test_models.py b/pipelex/test_extras/registry_test_models.py
index 24c198c70..d453f729c 100644
--- a/pipelex/test_extras/registry_test_models.py
+++ b/pipelex/test_extras/registry_test_models.py
@@ -1,6 +1,6 @@
from typing import ClassVar, List
-from pipelex.core.stuff_content import StructuredContent
+from pipelex.core.stuffs.stuff_content import StructuredContent
from pipelex.tools.registry_models import ModelType, RegistryModels
diff --git a/pipelex/test_extras/shared_pytest_plugins.py b/pipelex/test_extras/shared_pytest_plugins.py
index b05811580..c42082988 100644
--- a/pipelex/test_extras/shared_pytest_plugins.py
+++ b/pipelex/test_extras/shared_pytest_plugins.py
@@ -1,7 +1,7 @@
import pytest
from pytest import FixtureRequest, Parser
-from pipelex.core.pipe_run_params import PipeRunMode
+from pipelex.core.pipes.pipe_run_params import PipeRunMode
from pipelex.tools.runtime_manager import RunMode, runtime_manager
diff --git a/pipelex/tools/misc/mermaid_utils.py b/pipelex/tools/misc/mermaid_utils.py
index 0afe479be..b88f9e43a 100644
--- a/pipelex/tools/misc/mermaid_utils.py
+++ b/pipelex/tools/misc/mermaid_utils.py
@@ -24,7 +24,7 @@ def make_mermaid_url(mermaid_code: str) -> str:
},
}
encoded = encode_pako_from_string(json.dumps(as_dict))
- url = f"https://mermaid.live/edit#{encoded}"
+ url = f"https://mermaid.ink/img/{encoded}"
return url
diff --git a/pipelex/tools/misc/toml_utils.py b/pipelex/tools/misc/toml_utils.py
index 4dd7d9a79..ad2cf078d 100644
--- a/pipelex/tools/misc/toml_utils.py
+++ b/pipelex/tools/misc/toml_utils.py
@@ -1,10 +1,15 @@
from __future__ import annotations
-from typing import Any, Dict, List, Optional
+import re
+from typing import Any, Dict, List, Mapping, Optional, cast
import toml
+import tomlkit
+from tomlkit import array, document, inline_table, table
+from tomlkit import string as tk_string
from pipelex.tools.misc.file_utils import path_exists
+from pipelex.tools.misc.json_utils import remove_none_values_from_dict
class TOMLValidationError(Exception):
@@ -13,7 +18,7 @@ class TOMLValidationError(Exception):
pass
-def _validate_toml_content(content: str, file_path: str) -> None:
+def validate_toml_content(content: str, file_path: Optional[str] = None) -> None:
"""Validate TOML content for common formatting issues."""
lines = content.splitlines()
issues: List[str] = []
@@ -52,7 +57,7 @@ def validate_toml_file(path: str) -> None:
"""
with open(path, "r", encoding="utf-8") as file:
content = file.read()
- _validate_toml_content(content, path)
+ validate_toml_content(content, path)
def clean_trailing_whitespace(content: str) -> str:
@@ -122,3 +127,188 @@ def failable_load_toml_from_path(path: str) -> Optional[Dict[str, Any]]:
except toml.TomlDecodeError as exc:
print(f"Failed to parse TOML file '{path}': {exc}")
return None
+
+
+def make_toml_string(
+ text: str,
+ prefer_literal: bool = False,
+ force_multiline: bool = False,
+ ensure_trailing_newline: bool = True,
+ ensure_leading_blank_line_in_value: bool = False,
+):
+ """
+ Build a tomlkit string node.
+ - If `force_multiline` or the text contains '\\n', we emit a triple-quoted multiline string.
+ - When multiline, `ensure_trailing_newline` puts the closing quotes on their own line.
+ - When multiline, `ensure_leading_blank_line_in_value` inserts a real blank line at the start of the value.
+ """
+ needs_multiline = force_multiline or ("\n" in text)
+ normalized = text
+
+ if needs_multiline:
+ if ensure_leading_blank_line_in_value and not normalized.startswith("\n"):
+ normalized = "\n" + normalized
+ if ensure_trailing_newline and not normalized.endswith("\n"):
+ normalized = normalized + "\n"
+
+ use_literal = prefer_literal and ("'''" not in normalized)
+ return tk_string(normalized, multiline=needs_multiline, literal=use_literal)
+
+
+def _convert_to_inline(value: Any):
+ """Recursively convert Python values; dicts -> inline tables; lists kept as arrays."""
+ if isinstance(value, Mapping):
+ value = cast(Mapping[str, Any], value)
+ inline_table_obj = inline_table()
+ for key, value_item in value.items():
+ inline_table_obj[key] = _convert_to_inline(value_item)
+
+ return inline_table_obj
+
+ if isinstance(value, list):
+ value = cast(List[Any], value)
+ array_obj = array()
+ array_obj.multiline(True) # set to False for single-line arrays
+ for element in value:
+ if isinstance(element, Mapping):
+ element = cast(Mapping[str, Any], element)
+ inline_element = inline_table()
+ for inner_key, inner_value in element.items():
+ inline_element[inner_key] = _convert_to_inline(inner_value)
+ array_obj.append(inline_element) # pyright: ignore[reportUnknownMemberType]
+ else:
+ array_obj.append(_convert_to_inline(element)) # pyright: ignore[reportUnknownMemberType]
+ return array_obj
+
+ if isinstance(value, str):
+ # Triple quotes if needed (or forced); closing quotes on their own line.
+ return make_toml_string(
+ value,
+ prefer_literal=False, # flip to True for '''...'''
+ force_multiline=False, # flip to True to force """...""" even without \n
+ ensure_trailing_newline=True, # keep closing """ on its own line
+ ensure_leading_blank_line_in_value=False, # flip to True to keep a blank first line
+ )
+ return value
+
+
+def _filter_empty_values(value: Any) -> Any:
+ """Filter out empty lists and None values from data structures."""
+ if isinstance(value, dict):
+ filtered: Dict[str, Any] = {}
+ for k, v in cast(Dict[str, Any], value).items():
+ filtered_v = _filter_empty_values(v)
+ # Keep empty dicts but skip empty lists and None values
+ if filtered_v is not None and (not isinstance(filtered_v, list) or filtered_v):
+ filtered[k] = filtered_v
+ return filtered
+ elif isinstance(value, list):
+ return [_filter_empty_values(item) for item in cast(List[Any], value) if _filter_empty_values(item) is not None]
+ else:
+ return value
+
+
+def _create_ordered_inline_table(data: Mapping[str, Any]) -> Any:
+ """Create an inline table with fields in the expected order."""
+ inline_table_obj = inline_table()
+
+ # Define the preferred order for concept structure fields
+ field_order = ["type", "definition", "required", "choices", "item_type", "key_type", "value_type"]
+
+ # Add fields in preferred order first
+ for field in field_order:
+ if field in data:
+ value = data[field]
+ # Skip empty lists
+ if isinstance(value, list) and not value:
+ continue
+ inline_table_obj[field] = _convert_to_inline(value)
+
+ # Add any remaining fields not in the preferred order
+ for key, value in data.items():
+ if key not in field_order:
+ # Skip empty lists
+ if isinstance(value, list) and not value:
+ continue
+ inline_table_obj[key] = _convert_to_inline(value)
+
+ return inline_table_obj
+
+
+def dict_to_toml(data: Mapping[str, Any]) -> str:
+ """Convert dictionary to TOML format matching Pipelex expectations."""
+ data = remove_none_values_from_dict(data=data)
+ data = _filter_empty_values(data)
+ document_root = document()
+
+ # Handle top-level fields first (domain, definition, system_prompt, etc.)
+ for key, value in data.items():
+ if not isinstance(value, Mapping):
+ document_root.add(key, _convert_to_inline(value))
+
+ # Handle sections (concepts, pipes)
+ for section_key, section_value in data.items():
+ if isinstance(section_value, Mapping):
+ section_value = cast(Mapping[str, Any], section_value)
+
+ # Skip empty sections
+ if not section_value:
+ continue
+
+ # Create the section table
+ section_table = table()
+
+ # Process each item in the section
+ for item_key, item_value in section_value.items():
+ if isinstance(item_value, str):
+ # Simple string value (e.g., "SimpleData = 'Simple data concept'")
+ section_table.add(item_key, _convert_to_inline(item_value))
+ elif isinstance(item_value, Mapping):
+ # Complex object that needs its own table
+ item_value = cast(Mapping[str, Any], item_value)
+ item_table = table()
+
+ # Handle the structure field specially
+ for field_key, field_value in item_value.items():
+ if field_key == "structure" and isinstance(field_value, Mapping):
+ # Structure should be its own table [section.item.structure]
+ structure_table = table()
+ for struct_key, struct_value in cast(Mapping[str, Any], field_value).items():
+ if isinstance(struct_value, Mapping):
+ structure_table.add(struct_key, _create_ordered_inline_table(cast(Mapping[str, Any], struct_value)))
+ else:
+ structure_table.add(struct_key, _convert_to_inline(struct_value))
+ item_table.add(field_key, structure_table)
+ else:
+ # Skip empty lists like refines = []
+ if isinstance(field_value, list) and not field_value:
+ continue
+ item_table.add(field_key, _convert_to_inline(field_value))
+
+ section_table.add(item_key, item_table)
+
+ document_root.add(section_key, section_table)
+
+ dumped_content = tomlkit.dumps(document_root) # pyright: ignore[reportUnknownMemberType]
+
+ # Post-process to fix inline table spacing: {key = "value"} -> { key = "value" }
+
+ # Add space after opening brace and before closing brace for TOML inline tables only
+ # This regex matches TOML inline tables (containing = signs) but not Jinja2 templates
+ dumped_content = re.sub(r"\{([^}]*=[^}]*)\}", r"{ \1 }", dumped_content)
+
+ return dumped_content
+
+
+def save_toml_to_path(data: Dict[str, Any], path: str) -> None:
+ """Save dictionary as TOML to file path.
+
+ Args:
+ data: Dictionary to save as TOML
+ path: Path where to save the TOML file
+ """
+ data_cleaned = data
+ with open(path, "w", encoding="utf-8") as file:
+ toml_content: str = dict_to_toml(data=data_cleaned)
+ cleaned_content = clean_trailing_whitespace(toml_content)
+ file.write(cleaned_content)
diff --git a/pipelex/tools/templating/template_library.py b/pipelex/tools/templating/template_library.py
index e94006862..70e7a8624 100644
--- a/pipelex/tools/templating/template_library.py
+++ b/pipelex/tools/templating/template_library.py
@@ -26,8 +26,8 @@ class TemplateLibrary(TemplateProviderAbstract, RootModel[TemplateLibraryRoot]):
library_config: ClassVar[LibraryConfig]
@classmethod
- def make_empty(cls, config_folder_path: str) -> "TemplateLibrary":
- cls.library_config = LibraryConfig(config_folder_path=config_folder_path)
+ def make_empty(cls, config_dir_path: str) -> "TemplateLibrary":
+ cls.library_config = LibraryConfig(config_dir_path=config_dir_path)
return cls()
@override
diff --git a/pyproject.toml b/pyproject.toml
index 5a681376a..194524aff 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
[project]
name = "pipelex"
-version = "0.6.10"
+version = "0.7.0"
description = "Pipelex is an open-source dev tool based on a simple declarative language that lets you define replicable, structured, composable LLM pipelines."
authors = [{ name = "Evotis S.A.S.", email = "[email protected]" }]
maintainers = [{ name = "Pipelex staff", email = "[email protected]" }]
diff --git a/tests/cases/source_code.py b/tests/cases/source_code.py
index 463b191db..4253aa1b9 100644
--- a/tests/cases/source_code.py
+++ b/tests/cases/source_code.py
@@ -2,8 +2,8 @@
from typing import List
-from pipelex.core.stuff_content import TextContent
-from pipelex.core.working_memory import WorkingMemory
+from pipelex.core.memory.working_memory import WorkingMemory
+from pipelex.core.stuffs.stuff_content import TextContent
def wrap_lines(working_memory: WorkingMemory) -> TextContent:
diff --git a/tests/conftest.py b/tests/conftest.py
index 685dc6078..1e93d46e4 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -7,7 +7,7 @@ import pipelex.config
import pipelex.pipelex
from pipelex import log
from pipelex.config import get_config
-from pipelex.core.concept_provider_abstract import ConceptProviderAbstract
+from pipelex.core.concepts.concept_provider_abstract import ConceptProviderAbstract
from pipelex.hub import get_concept_provider
from tests.cases.registry import Fruit
diff --git a/tests/data/tools_data/mock_folder_with_classes/subfolder/test_model_c.py b/tests/data/tools_data/mock_folder_with_classes/subfolder/test_model_c.py
index b8e966651..3340ac437 100644
--- a/tests/data/tools_data/mock_folder_with_classes/subfolder/test_model_c.py
+++ b/tests/data/tools_data/mock_folder_with_classes/subfolder/test_model_c.py
@@ -1,4 +1,4 @@
-from pipelex.core.stuff_content import StuffContent
+from pipelex.core.stuffs.stuff_content import StuffContent
class Class3(StuffContent):
diff --git a/tests/data/tools_data/mock_folder_with_classes/test_model_a.py b/tests/data/tools_data/mock_folder_with_classes/test_model_a.py
index d1252e29d..f8c231d70 100644
--- a/tests/data/tools_data/mock_folder_with_classes/test_model_a.py
+++ b/tests/data/tools_data/mock_folder_with_classes/test_model_a.py
@@ -1,6 +1,6 @@
from pydantic import BaseModel
-from pipelex.core.stuff_content import StuffContent
+from pipelex.core.stuffs.stuff_content import StuffContent
class ClassA(BaseModel):
diff --git a/tests/data/tools_data/mock_folder_with_classes/test_model_b.py b/tests/data/tools_data/mock_folder_with_classes/test_model_b.py
index e263e3f23..c7a3c1fdd 100644
--- a/tests/data/tools_data/mock_folder_with_classes/test_model_b.py
+++ b/tests/data/tools_data/mock_folder_with_classes/test_model_b.py
@@ -1,4 +1,4 @@
-from pipelex.core.stuff_content import StructuredContent, StuffContent
+from pipelex.core.stuffs.stuff_content import StructuredContent, StuffContent
class Class2(StuffContent):
diff --git a/tests/data/tools_data/problematic_test_cases.toml b/tests/data/tools_data/problematic_test_cases.toml
index d88e90a07..9b2977c95 100644
--- a/tests/data/tools_data/problematic_test_cases.toml
+++ b/tests/data/tools_data/problematic_test_cases.toml
@@ -6,7 +6,8 @@ TestInput = "A test input concept"
[pipe]
[pipe.test_pipe_with_issues]
-PipeLLM = "Test pipe with trailing whitespace issues"
+type = "PipeLLM"
+definition = "Test pipe with trailing whitespace issues"
inputs = { input_data = "TestInput" }
output = "native.Text"
prompt_template = """
@@ -14,7 +15,8 @@ This template has trailing whitespace after closing quotes
"""
[pipe.another_problematic_pipe]
-PipeLLM = "Another pipe with issues"
+type = "PipeLLM"
+definition = "Another pipe with issues"
output = "native.Text"
prompt_template = """
Multiple issues here
diff --git a/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_1.py b/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_1.py
index 16cd21ea5..9862c2c57 100644
--- a/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_1.py
+++ b/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_1.py
@@ -2,11 +2,11 @@ import pytest
from pytest import FixtureRequest
from pipelex import pretty_print
-from pipelex.core.pipe_output import PipeOutput
-from pipelex.core.pipe_run_params import PipeRunMode
-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory
-from pipelex.core.stuff_factory import StuffFactory
-from pipelex.core.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.pipes.pipe_output import PipeOutput
+from pipelex.core.pipes.pipe_run_params import PipeRunMode
+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory
+from pipelex.core.stuffs.stuff_factory import StuffFactory
from pipelex.hub import get_pipe_router, get_report_delegate
from pipelex.pipeline.job_metadata import JobMetadata
from tests.test_pipelines.pipe_controllers.pipe_condition.pipe_condition import CategoryInput
diff --git a/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_2.py b/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_2.py
index 18dd9e1a9..a466918d0 100644
--- a/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_2.py
+++ b/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_2.py
@@ -2,11 +2,11 @@ import pytest
from pytest import FixtureRequest
from pipelex import pretty_print
-from pipelex.core.pipe_output import PipeOutput
-from pipelex.core.pipe_run_params import PipeRunMode
-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory
-from pipelex.core.stuff_factory import StuffFactory
-from pipelex.core.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.pipes.pipe_output import PipeOutput
+from pipelex.core.pipes.pipe_run_params import PipeRunMode
+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory
+from pipelex.core.stuffs.stuff_factory import StuffFactory
from pipelex.hub import get_pipe_router, get_report_delegate
from pipelex.pipeline.job_metadata import JobMetadata
from tests.test_pipelines.pipe_controllers.pipe_condition.pipe_condition import CategoryInput
diff --git a/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.py b/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.py
index 3abd11548..133375a42 100644
--- a/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.py
+++ b/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.py
@@ -3,10 +3,10 @@
import pytest
from pipelex import pretty_print
-from pipelex.core.pipe_input_spec import PipeInputSpec, TypedNamedInputRequirement
-from pipelex.core.pipe_run_params import PipeRunMode
-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory
-from pipelex.core.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.pipes.pipe_input_spec import InputRequirementBlueprint, PipeInputSpec, TypedNamedInputRequirement
+from pipelex.core.pipes.pipe_run_params import PipeRunMode
+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory
from pipelex.exceptions import DryRunError
from pipelex.pipe_controllers.pipe_condition import PipeCondition
from pipelex.pipeline.job_metadata import JobMetadata
@@ -23,7 +23,9 @@ class TestPipeConditionSimple:
pipe_condition = PipeCondition(
code="test_condition_fail",
domain="test_domain",
- inputs=PipeInputSpec.make_from_dict(concepts_dict={"user_category": "test_pipe_condition.CategoryInput"}),
+ inputs=PipeInputSpec.make_from_blueprint(
+ domain="test_domain", blueprint={"user_category": InputRequirementBlueprint(concept_code="test_pipe_condition.CategoryInput")}
+ ),
output_concept_code="native.Text",
expression_template="{{ user_category.category }}",
pipe_map={"small": "process_small", "medium": "process_medium", "large": "process_large"},
@@ -52,7 +54,9 @@ class TestPipeConditionSimple:
pipe_condition = PipeCondition(
code="test_condition_succeed",
domain="test_domain",
- inputs=PipeInputSpec.make_from_dict(concepts_dict={"user_status": "test_pipe_condition.CategoryInput"}),
+ inputs=PipeInputSpec.make_from_blueprint(
+ domain="test_domain", blueprint={"user_status": InputRequirementBlueprint(concept_code="test_pipe_condition.CategoryInput")}
+ ),
output_concept_code="native.Text",
expression_template="{{ user_status.category }}",
pipe_map={
diff --git a/tests/e2e/pipelex/pipes/pipe_controller/pipe_parallel/test_pipe_parallel_1.py b/tests/e2e/pipelex/pipes/pipe_controller/pipe_parallel/test_pipe_parallel_1.py
index 1652398c7..f22730e93 100644
--- a/tests/e2e/pipelex/pipes/pipe_controller/pipe_parallel/test_pipe_parallel_1.py
+++ b/tests/e2e/pipelex/pipes/pipe_controller/pipe_parallel/test_pipe_parallel_1.py
@@ -4,11 +4,11 @@ import pytest
from pytest import FixtureRequest
from pipelex import pretty_print
-from pipelex.core.pipe_output import PipeOutput
-from pipelex.core.pipe_run_params import PipeRunMode
-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory
-from pipelex.core.stuff_factory import StuffFactory
-from pipelex.core.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.pipes.pipe_output import PipeOutput
+from pipelex.core.pipes.pipe_run_params import PipeRunMode
+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory
+from pipelex.core.stuffs.stuff_factory import StuffFactory
from pipelex.hub import get_pipe_router, get_report_delegate
from pipelex.pipeline.job_metadata import JobMetadata
from tests.test_pipelines.pipe_controllers.pipe_parallel.pipe_parallel import ContentAnalysis, DocumentInput, LengthAnalysis
diff --git a/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_batching.py b/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_batching.py
index 00d947ccf..72f3517f1 100644
--- a/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_batching.py
+++ b/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_batching.py
@@ -5,11 +5,11 @@ from typing import cast
import pytest
from pipelex import log
-from pipelex.core.pipe_input_spec import TypedNamedInputRequirement
-from pipelex.core.pipe_run_params import PipeRunMode
-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory
-from pipelex.core.stuff_factory import StuffFactory
-from pipelex.core.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.pipes.pipe_input_spec import TypedNamedInputRequirement
+from pipelex.core.pipes.pipe_run_params import PipeRunMode
+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory
+from pipelex.core.stuffs.stuff_factory import StuffFactory
from pipelex.hub import get_required_pipe
from pipelex.pipeline.execute import execute_pipeline
from pipelex.pipeline.job_metadata import JobMetadata
diff --git a/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_multiplicity.py b/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_multiplicity.py
index ed03fecf5..489e5e2b8 100644
--- a/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_multiplicity.py
+++ b/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_multiplicity.py
@@ -3,10 +3,10 @@
import pytest
from pytest import FixtureRequest
-from pipelex.core.pipe_run_params import PipeRunMode
-from pipelex.core.stuff_content import TextContent
-from pipelex.core.stuff_factory import StuffFactory
-from pipelex.core.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.pipes.pipe_run_params import PipeRunMode
+from pipelex.core.stuffs.stuff_content import TextContent
+from pipelex.core.stuffs.stuff_factory import StuffFactory
from pipelex.pipeline.execute import execute_pipeline
diff --git a/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_simple.py b/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_simple.py
index 901ee4667..0e561f39f 100644
--- a/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_simple.py
+++ b/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_simple.py
@@ -3,11 +3,11 @@
import pytest
from pipelex import pretty_print
-from pipelex.core.pipe_run_params import PipeRunMode
-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory
-from pipelex.core.stuff_content import TextContent
-from pipelex.core.stuff_factory import StuffFactory
-from pipelex.core.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.pipes.pipe_run_params import PipeRunMode
+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory
+from pipelex.core.stuffs.stuff_content import TextContent
+from pipelex.core.stuffs.stuff_factory import StuffFactory
from pipelex.hub import get_required_pipe
from pipelex.pipeline.job_metadata import JobMetadata
diff --git a/tests/integration/pipelex/client/test_client.py b/tests/integration/pipelex/client/test_client.py
index 24bf1f8aa..7fa887796 100644
--- a/tests/integration/pipelex/client/test_client.py
+++ b/tests/integration/pipelex/client/test_client.py
@@ -6,10 +6,10 @@ from pydantic import BaseModel
from pipelex import pretty_print
from pipelex.client.client import PipelexClient
from pipelex.client.protocol import COMPACT_MEMORY_KEY, PipelineState
-from pipelex.core.stuff import Stuff
-from pipelex.core.stuff_content import TextContent
-from pipelex.core.stuff_factory import StuffFactory
-from pipelex.core.working_memory import WorkingMemory
+from pipelex.core.memory.working_memory import WorkingMemory
+from pipelex.core.stuffs.stuff import Stuff
+from pipelex.core.stuffs.stuff_content import TextContent
+from pipelex.core.stuffs.stuff_factory import StuffFactory
class Example(BaseModel):
diff --git a/tests/integration/pipelex/cogt/test_content_generator.py b/tests/integration/pipelex/cogt/test_content_generator.py
index 3a2e75623..b0614b2b3 100644
--- a/tests/integration/pipelex/cogt/test_content_generator.py
+++ b/tests/integration/pipelex/cogt/test_content_generator.py
@@ -125,7 +125,6 @@ class TestContentGenerator:
context=context,
jinja2="The answer is: {{ the_answer }}",
)
- pretty_print(jinja2_text, title="jinja2_text")
assert isinstance(jinja2_text, str)
assert jinja2_text == "The answer is: elementary, my dear Watson"
diff --git a/tests/integration/pipelex/conftest.py b/tests/integration/pipelex/conftest.py
index ab299e51c..c576962c0 100644
--- a/tests/integration/pipelex/conftest.py
+++ b/tests/integration/pipelex/conftest.py
@@ -23,12 +23,12 @@ def llm_preset_id(request: pytest.FixtureRequest) -> str:
params=[
# "o1",
# "gpt-4o",
- # "gpt-4o-mini",
+ "gpt-4o-mini",
# "gpt-4-5-preview",
# "o1-mini",
# "o3-mini",
- # "gpt-5-mini",
- "gpt-5-nano",
+ # "gpt-5-mini", # TODO: fix this
+ # "gpt-5-nano", # TODO: fix this
# "best-gpt",
# "claude-3-haiku",
# "claude-3-5-sonnet",
@@ -62,9 +62,9 @@ def llm_handle(request: pytest.FixtureRequest) -> str:
# "o1",
# "o3-mini",
# "gpt-4o",
- # "gpt-4o-mini",
+ "gpt-4o-mini",
# "gpt-5-mini",
- "gpt-5-nano",
+ # "gpt-5-nano",
# "gpt-5-chat",
# "gpt-4-5-preview",
# "claude-3-haiku",
diff --git a/tests/integration/pipelex/pipes/conftest.py b/tests/integration/pipelex/pipes/conftest.py
index dc04dd01f..33ff3509f 100644
--- a/tests/integration/pipelex/pipes/conftest.py
+++ b/tests/integration/pipelex/pipes/conftest.py
@@ -4,7 +4,7 @@ from typing import AsyncIterator, Awaitable, Callable
import pytest_asyncio
from pytest import FixtureRequest
-from pipelex.core.pipe_output import PipeOutput
+from pipelex.core.pipes.pipe_output import PipeOutput
from pipelex.hub import get_activity_manager
from pipelex.pipeline.activity.activity_handler import ActivityHandlerForResultFiles
from pipelex.tools.misc.file_utils import get_incremental_directory_path, remove_folder
diff --git a/tests/integration/pipelex/pipes/pipe_controller/pipe_batch/test_pipe_batch_simple.py b/tests/integration/pipelex/pipes/pipe_controller/pipe_batch/test_pipe_batch_simple.py
index 7a7a0e7f9..69c0003d6 100644
--- a/tests/integration/pipelex/pipes/pipe_controller/pipe_batch/test_pipe_batch_simple.py
+++ b/tests/integration/pipelex/pipes/pipe_controller/pipe_batch/test_pipe_batch_simple.py
@@ -6,12 +6,12 @@ import pytest
from pytest import FixtureRequest
from pipelex import pretty_print
-from pipelex.core.pipe_input_spec import PipeInputSpec
-from pipelex.core.pipe_run_params import BatchParams, PipeRunMode
-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory
-from pipelex.core.stuff_content import ListContent, StuffContent, TextContent
-from pipelex.core.stuff_factory import StuffFactory
-from pipelex.core.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.pipes.pipe_input_spec import InputRequirementBlueprint, PipeInputSpec
+from pipelex.core.pipes.pipe_run_params import BatchParams, PipeRunMode
+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory
+from pipelex.core.stuffs.stuff_content import ListContent, StuffContent, TextContent
+from pipelex.core.stuffs.stuff_factory import StuffFactory
from pipelex.pipe_controllers.pipe_batch import PipeBatch
from pipelex.pipeline.job_metadata import JobMetadata
@@ -29,7 +29,13 @@ class TestPipeBatchSimple:
domain="test_integration",
code="simple_batch",
branch_pipe_code="uppercase_transformer", # This exists in the TOML file
- inputs=PipeInputSpec.make_from_dict(concepts_dict={"text_list": "Text", "text_item": "Text"}),
+ inputs=PipeInputSpec.make_from_blueprint(
+ domain="test_integration",
+ blueprint={
+ "text_list": InputRequirementBlueprint(concept_code="Text"),
+ "text_item": InputRequirementBlueprint(concept_code="Text"),
+ },
+ ),
output_concept_code="test_integration.UppercaseText",
batch_params=BatchParams(input_list_stuff_name="text_list", input_item_stuff_name="text_item"),
)
diff --git a/tests/integration/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_complex.py b/tests/integration/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_complex.py
index 3c9ea97ff..3a29a8fb5 100644
--- a/tests/integration/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_complex.py
+++ b/tests/integration/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_complex.py
@@ -6,12 +6,12 @@ import pytest
from pytest import FixtureRequest
from pipelex import pretty_print
-from pipelex.core.pipe_output import PipeOutput
-from pipelex.core.pipe_run_params import PipeRunMode
-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory
-from pipelex.core.stuff_content import TextContent
-from pipelex.core.stuff_factory import StuffFactory
-from pipelex.core.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.pipes.pipe_output import PipeOutput
+from pipelex.core.pipes.pipe_run_params import PipeRunMode
+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory
+from pipelex.core.stuffs.stuff_content import TextContent
+from pipelex.core.stuffs.stuff_factory import StuffFactory
from pipelex.exceptions import DryRunError
from pipelex.hub import get_pipe_router
from pipelex.pipeline.job_metadata import JobMetadata
diff --git a/tests/integration/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.py b/tests/integration/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.py
index 9179ded28..269b15903 100644
--- a/tests/integration/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.py
+++ b/tests/integration/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.py
@@ -6,13 +6,13 @@ import pytest
from pytest import FixtureRequest
from pipelex import pretty_print
-from pipelex.core.pipe_input_spec import PipeInputSpec
-from pipelex.core.pipe_output import PipeOutput
-from pipelex.core.pipe_run_params import PipeRunMode
-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory
-from pipelex.core.stuff_content import TextContent
-from pipelex.core.stuff_factory import StuffFactory
-from pipelex.core.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.pipes.pipe_input_spec import InputRequirementBlueprint, PipeInputSpec
+from pipelex.core.pipes.pipe_output import PipeOutput
+from pipelex.core.pipes.pipe_run_params import PipeRunMode
+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory
+from pipelex.core.stuffs.stuff_content import TextContent
+from pipelex.core.stuffs.stuff_factory import StuffFactory
from pipelex.exceptions import DryRunError
from pipelex.hub import get_pipe_router
from pipelex.pipe_controllers.pipe_condition import PipeCondition
@@ -31,7 +31,9 @@ class TestPipeConditionSimple:
pipe_condition = PipeCondition(
domain="test_integration",
code="text_length_condition",
- inputs=PipeInputSpec.make_from_dict(concepts_dict={"input_text": "Text"}),
+ inputs=PipeInputSpec.make_from_blueprint(
+ domain="test_integration", blueprint={"input_text": InputRequirementBlueprint(concept_code="Text")}
+ ),
output_concept_code="Text",
expression_template="{% if input_text.text|length > 5 %}long{% else %}short{% endif %}",
pipe_map={"long": "capitalize_long_text", "short": "add_prefix_short_text"},
@@ -100,7 +102,9 @@ class TestPipeConditionSimple:
pipe_condition = PipeCondition(
domain="test_integration",
code="text_length_condition",
- inputs=PipeInputSpec.make_from_dict(concepts_dict={"input_text": "Text"}),
+ inputs=PipeInputSpec.make_from_blueprint(
+ domain="test_integration", blueprint={"input_text": InputRequirementBlueprint(concept_code="Text")}
+ ),
output_concept_code="Text",
expression_template="{% if input_text.text|length > 5 %}long{% else %}short{% endif %}",
pipe_map={"long": "capitalize_long_text", "short": "add_prefix_short_text"},
diff --git a/tests/integration/pipelex/pipes/pipe_controller/pipe_parallel/test_pipe_parallel_simple.py b/tests/integration/pipelex/pipes/pipe_controller/pipe_parallel/test_pipe_parallel_simple.py
index 4ee80eea5..f5191c50d 100644
--- a/tests/integration/pipelex/pipes/pipe_controller/pipe_parallel/test_pipe_parallel_simple.py
+++ b/tests/integration/pipelex/pipes/pipe_controller/pipe_parallel/test_pipe_parallel_simple.py
@@ -5,12 +5,12 @@ from typing import cast
import pytest
from pytest import FixtureRequest
-from pipelex.core.pipe_input_spec import PipeInputSpec
-from pipelex.core.pipe_run_params import PipeRunMode
-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory
-from pipelex.core.stuff_content import TextContent
-from pipelex.core.stuff_factory import StuffFactory
-from pipelex.core.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.pipes.pipe_input_spec import InputRequirementBlueprint, PipeInputSpec
+from pipelex.core.pipes.pipe_run_params import PipeRunMode
+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory
+from pipelex.core.stuffs.stuff_content import TextContent
+from pipelex.core.stuffs.stuff_factory import StuffFactory
from pipelex.pipe_controllers.pipe_parallel import PipeParallel
from pipelex.pipe_controllers.sub_pipe import SubPipe
from pipelex.pipeline.job_metadata import JobMetadata
@@ -28,7 +28,9 @@ class TestPipeParallelSimple:
pipe_parallel = PipeParallel(
domain="test_integration",
code="parallel_text_analyzer",
- inputs=PipeInputSpec.make_from_dict(concepts_dict={"input_text": "Text"}),
+ inputs=PipeInputSpec.make_from_blueprint(
+ domain="test_integration", blueprint={"input_text": InputRequirementBlueprint(concept_code="Text")}
+ ),
output_concept_code="Text",
parallel_sub_pipes=[
SubPipe(pipe_code="analyze_sentiment", output_name="sentiment_result"),
@@ -140,7 +142,9 @@ class TestPipeParallelSimple:
pipe_parallel = PipeParallel(
domain="test_integration",
code="parallel_text_analyzer",
- inputs=PipeInputSpec.make_from_dict(concepts_dict={"input_text": "Text"}),
+ inputs=PipeInputSpec.make_from_blueprint(
+ domain="test_integration", blueprint={"input_text": InputRequirementBlueprint(concept_code="Text")}
+ ),
output_concept_code="Text",
parallel_sub_pipes=[
SubPipe(pipe_code="analyze_sentiment", output_name="sentiment_result"),
diff --git a/tests/integration/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_dry_run.py b/tests/integration/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_dry_run.py
index cfc1daea9..b49f15b07 100644
--- a/tests/integration/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_dry_run.py
+++ b/tests/integration/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_dry_run.py
@@ -4,14 +4,14 @@ import pytest
from pytest import FixtureRequest
from pipelex import pretty_print
-from pipelex.core.pipe_output import PipeOutput
-from pipelex.core.pipe_run_params import PipeRunMode
-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory
-from pipelex.core.stuff import Stuff
-from pipelex.core.stuff_content import ListContent
-from pipelex.core.stuff_factory import StuffFactory
-from pipelex.core.working_memory import WorkingMemory
-from pipelex.core.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.memory.working_memory import WorkingMemory
+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.pipes.pipe_output import PipeOutput
+from pipelex.core.pipes.pipe_run_params import PipeRunMode
+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory
+from pipelex.core.stuffs.stuff import Stuff
+from pipelex.core.stuffs.stuff_content import ListContent
+from pipelex.core.stuffs.stuff_factory import StuffFactory
from pipelex.hub import get_pipe_router, get_report_delegate
from pipelex.pipeline.job_metadata import JobMetadata
from pipelex.tools.misc.json_utils import load_json_list_from_path
diff --git a/tests/integration/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_simple.py b/tests/integration/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_simple.py
index 965565348..704ea1ab7 100644
--- a/tests/integration/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_simple.py
+++ b/tests/integration/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_simple.py
@@ -6,12 +6,12 @@ import pytest
from pytest import FixtureRequest
from pipelex import pretty_print
-from pipelex.core.pipe_input_spec import PipeInputSpec
-from pipelex.core.pipe_run_params import PipeRunMode
-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory
-from pipelex.core.stuff_content import TextContent
-from pipelex.core.stuff_factory import StuffFactory
-from pipelex.core.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.pipes.pipe_input_spec import InputRequirementBlueprint, PipeInputSpec
+from pipelex.core.pipes.pipe_run_params import PipeRunMode
+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory
+from pipelex.core.stuffs.stuff_content import TextContent
+from pipelex.core.stuffs.stuff_factory import StuffFactory
from pipelex.pipe_controllers.pipe_sequence import PipeSequence
from pipelex.pipe_controllers.sub_pipe import SubPipe
from pipelex.pipeline.job_metadata import JobMetadata
@@ -29,7 +29,9 @@ class TestPipeSequenceSimple:
pipe_sequence = PipeSequence(
domain="test_integration",
code="simple_sequence",
- inputs=PipeInputSpec.make_from_dict(concepts_dict={"input_text": "Text"}),
+ inputs=PipeInputSpec.make_from_blueprint(
+ domain="test_integration", blueprint={"input_text": InputRequirementBlueprint(concept_code="Text")}
+ ),
output_concept_code="Text",
sequential_sub_pipes=[
SubPipe(pipe_code="capitalize_text", output_name="capitalized_text"),
diff --git a/tests/integration/pipelex/pipes/pipe_operator/pipe_func/test_pipe_func.py b/tests/integration/pipelex/pipes/pipe_operator/pipe_func/test_pipe_func.py
index d36d6219f..5a8ec7a0e 100644
--- a/tests/integration/pipelex/pipes/pipe_operator/pipe_func/test_pipe_func.py
+++ b/tests/integration/pipelex/pipes/pipe_operator/pipe_func/test_pipe_func.py
@@ -1,12 +1,12 @@
import pytest
from pipelex import log, pretty_print
-from pipelex.core.concept_native import NativeConcept
-from pipelex.core.pipe_run_params import PipeRunMode
-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory
-from pipelex.core.stuff_content import TextContent
-from pipelex.core.stuff_factory import StuffFactory
-from pipelex.core.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.concepts.concept_native import NativeConcept
+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.pipes.pipe_run_params import PipeRunMode
+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory
+from pipelex.core.stuffs.stuff_content import TextContent
+from pipelex.core.stuffs.stuff_factory import StuffFactory
from pipelex.hub import get_pipe_router, get_report_delegate
from pipelex.pipe_operators.pipe_func import PipeFunc, PipeFuncOutput
from pipelex.pipe_works.pipe_job_factory import PipeJobFactory
diff --git a/tests/integration/pipelex/pipes/pipe_operator/pipe_imgg/test_pipe_imgg.py b/tests/integration/pipelex/pipes/pipe_operator/pipe_imgg/test_pipe_imgg.py
index 82ed87fee..35c71093c 100644
--- a/tests/integration/pipelex/pipes/pipe_operator/pipe_imgg/test_pipe_imgg.py
+++ b/tests/integration/pipelex/pipes/pipe_operator/pipe_imgg/test_pipe_imgg.py
@@ -2,9 +2,9 @@ import pytest
from pipelex import pretty_print
from pipelex.cogt.imgg.imgg_handle import ImggHandle
-from pipelex.core.concept_native import NativeConcept
-from pipelex.core.pipe_run_params import PipeRunMode
-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory
+from pipelex.core.concepts.concept_native import NativeConcept
+from pipelex.core.pipes.pipe_run_params import PipeRunMode
+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory
from pipelex.hub import get_pipe_router
from pipelex.pipe_operators.pipe_img_gen import PipeImgGen, PipeImgGenOutput
from pipelex.pipe_works.pipe_job_factory import PipeJobFactory
diff --git a/tests/integration/pipelex/pipes/pipe_operator/pipe_llm/test_generate_pipeline_blueprint.py b/tests/integration/pipelex/pipes/pipe_operator/pipe_llm/test_generate_pipeline_blueprint.py
deleted file mode 100644
index 1e1805bba..000000000
--- a/tests/integration/pipelex/pipes/pipe_operator/pipe_llm/test_generate_pipeline_blueprint.py
+++ /dev/null
@@ -1,46 +0,0 @@
-import pytest
-
-from pipelex import pretty_print
-from pipelex.core.pipe_run_params import PipeRunMode
-from pipelex.create.helpers import get_support_file
-from pipelex.libraries.pipeline_blueprint import PipelineBlueprint
-from pipelex.pipeline.execute import execute_pipeline
-
-
[email protected](loop_scope="class")
- "requirements",
- [
- (
- "Create a domain 'sample' to describe simple demo pipelines. "
- "Define concept ThingDescription = 'A short natural language description of a thing'. "
- "Add a pipe write_thing that takes input { text = Text } and outputs ThingDescription using PipeLLM."
- ),
- (
- "Domain: gadgets. Concept: GadgetSpec = 'Technical description of a gadget'. "
- "Add pipe summarize_gadget: inputs { text = Text } -> output GadgetSpec via PipeLLM."
- ),
- ],
-)
-async def test_generate_pipeline_blueprint(pipe_run_mode: PipeRunMode, requirements: str):
- pipe_output = await execute_pipeline(
- pipe_code="build_blueprint",
- input_memory={
- "domain": "test_domain",
- "pipeline_name": "test_pipeline",
- "requirements": requirements,
- "draft_pipeline_rules": get_support_file(subpath="create/draft_pipelines.md"),
- "build_pipeline_rules": get_support_file(subpath="create/build_pipelines.md"),
- },
- pipe_run_mode=pipe_run_mode,
- )
-
- blueprint = pipe_output.main_stuff_as(content_type=PipelineBlueprint)
- assert isinstance(blueprint, PipelineBlueprint)
- # Basic sanity checks
- assert isinstance(blueprint.domain, str) and blueprint.domain != ""
-
- pretty_print(blueprint, title="PipelineBlueprint")
diff --git a/tests/integration/pipelex/pipes/pipe_operator/pipe_llm/test_pipe_llm.py b/tests/integration/pipelex/pipes/pipe_operator/pipe_llm/test_pipe_llm.py
index 862a0f0d1..0f93d28a1 100644
--- a/tests/integration/pipelex/pipes/pipe_operator/pipe_llm/test_pipe_llm.py
+++ b/tests/integration/pipelex/pipes/pipe_operator/pipe_llm/test_pipe_llm.py
@@ -3,12 +3,12 @@ from typing import List
import pytest
from pipelex import log, pretty_print
-from pipelex.core.concept_native import NativeConcept
-from pipelex.core.pipe_input_spec import PipeInputSpec
-from pipelex.core.pipe_run_params import PipeRunMode
-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory
-from pipelex.core.stuff import Stuff
-from pipelex.core.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.concepts.concept_native import NativeConcept
+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.pipes.pipe_input_spec import InputRequirementBlueprint, PipeInputSpec
+from pipelex.core.pipes.pipe_run_params import PipeRunMode
+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory
+from pipelex.core.stuffs.stuff import Stuff
from pipelex.hub import get_pipe_router, get_report_delegate
from pipelex.pipe_operators.pipe_llm import PipeLLM, PipeLLMOutput
from pipelex.pipe_operators.pipe_llm_prompt import PipeLLMPrompt
@@ -67,7 +67,9 @@ class TestPipeLLM:
pipe=PipeLLM(
code="adhoc_for_test_pipe_llm_image",
domain="generic",
- inputs=PipeInputSpec.make_from_dict(concepts_dict={stuff_name: stuff.concept_code}),
+ inputs=PipeInputSpec.make_from_blueprint(
+ domain="generic", blueprint={stuff_name: InputRequirementBlueprint(concept_code=stuff.concept_code)}
+ ),
output_concept_code=NativeConcept.TEXT.code,
pipe_llm_prompt=PipeLLMPrompt(
code="adhoc_for_test_pipe_llm_image",
diff --git a/tests/integration/pipelex/pipes/pipe_operator/pipe_ocr/test_pipe_ocr.py b/tests/integration/pipelex/pipes/pipe_operator/pipe_ocr/test_pipe_ocr.py
index 7b11c43b0..87a6b6757 100644
--- a/tests/integration/pipelex/pipes/pipe_operator/pipe_ocr/test_pipe_ocr.py
+++ b/tests/integration/pipelex/pipes/pipe_operator/pipe_ocr/test_pipe_ocr.py
@@ -1,12 +1,12 @@
import pytest
from pipelex import pretty_print
-from pipelex.core.concept_native import NativeConcept
-from pipelex.core.pipe_input_spec import PipeInputSpec
-from pipelex.core.pipe_run_params import PipeRunMode
-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory
-from pipelex.core.stuff_content import PageContent
-from pipelex.core.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.concepts.concept_native import NativeConcept
+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.pipes.pipe_input_spec import InputRequirementBlueprint, PipeInputSpec
+from pipelex.core.pipes.pipe_run_params import PipeRunMode
+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory
+from pipelex.core.stuffs.stuff_content import PageContent
from pipelex.hub import get_pipe_router
from pipelex.pipe_operators.pipe_ocr import PIPE_OCR_INPUT_NAME, PipeOcr, PipeOcrOutput
from pipelex.pipe_works.pipe_job_factory import PipeJobFactory
@@ -28,7 +28,7 @@ class TestPipeOCR:
pipe=PipeOcr(
code="adhoc_for_test_pipe_ocr_image",
domain="generic",
- inputs=PipeInputSpec.make_from_dict(concepts_dict={"page_scan": "Image"}),
+ inputs=PipeInputSpec.make_from_blueprint(domain="generic", blueprint={"page_scan": InputRequirementBlueprint(concept_code="Image")}),
should_include_images=True,
should_caption_images=False,
should_include_page_views=True,
@@ -58,7 +58,9 @@ class TestPipeOCR:
pipe=PipeOcr(
code="adhoc_for_test_pipe_ocr_pdf",
domain="generic",
- inputs=PipeInputSpec.make_from_dict(concepts_dict={PIPE_OCR_INPUT_NAME: "PDF"}),
+ inputs=PipeInputSpec.make_from_blueprint(
+ domain="generic", blueprint={PIPE_OCR_INPUT_NAME: InputRequirementBlueprint(concept_code=NativeConcept.PDF.code)}
+ ),
should_include_images=True,
should_caption_images=False,
should_include_page_views=True,
diff --git a/tests/integration/pipelex/pipes/test_image_inputs.py b/tests/integration/pipelex/pipes/test_image_inputs.py
index bc5d8f6fb..d5eef6d24 100644
--- a/tests/integration/pipelex/pipes/test_image_inputs.py
+++ b/tests/integration/pipelex/pipes/test_image_inputs.py
@@ -2,12 +2,12 @@ import pytest
from pytest import FixtureRequest
from pipelex import pretty_print
-from pipelex.core.pipe_output import PipeOutput
-from pipelex.core.pipe_run_params import PipeRunMode
-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory
-from pipelex.core.stuff_content import ImageContent, PageContent, TextAndImagesContent, TextContent
-from pipelex.core.stuff_factory import StuffFactory
-from pipelex.core.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.pipes.pipe_output import PipeOutput
+from pipelex.core.pipes.pipe_run_params import PipeRunMode
+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory
+from pipelex.core.stuffs.stuff_content import ImageContent, PageContent, TextAndImagesContent, TextContent
+from pipelex.core.stuffs.stuff_factory import StuffFactory
from pipelex.hub import get_pipe_router, get_report_delegate
from pipelex.pipeline.job_metadata import JobMetadata
from tests.cases import ImageTestCases
diff --git a/tests/integration/pipelex/pipes/test_pipe_jinja2.py b/tests/integration/pipelex/pipes/test_pipe_jinja2.py
index 438b01219..988869d06 100644
--- a/tests/integration/pipelex/pipes/test_pipe_jinja2.py
+++ b/tests/integration/pipelex/pipes/test_pipe_jinja2.py
@@ -1,9 +1,9 @@
import pytest
from pipelex import pretty_print
-from pipelex.core.pipe_run_params import PipeRunMode
-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory
-from pipelex.core.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.pipes.pipe_run_params import PipeRunMode
+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory
from pipelex.hub import get_pipe_router
from pipelex.pipe_operators.pipe_jinja2 import PipeJinja2, PipeJinja2Output
from pipelex.pipe_works.pipe_job_factory import PipeJobFactory
diff --git a/tests/integration/pipelex/pipes/test_pipe_llm_prompt.py b/tests/integration/pipelex/pipes/test_pipe_llm_prompt.py
index 7aa68df24..00354d843 100644
--- a/tests/integration/pipelex/pipes/test_pipe_llm_prompt.py
+++ b/tests/integration/pipelex/pipes/test_pipe_llm_prompt.py
@@ -2,11 +2,11 @@ from typing import List, Tuple, Type
import pytest
-from pipelex.core.concept_native import NativeConcept
-from pipelex.core.pipe_run_params import PipeRunMode
-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory
-from pipelex.core.stuff_content import StructuredContent
-from pipelex.core.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.concepts.concept_native import NativeConcept
+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.pipes.pipe_run_params import PipeRunMode
+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory
+from pipelex.core.stuffs.stuff_content import StructuredContent
from pipelex.hub import get_pipe_router
from pipelex.pipe_operators.pipe_llm_prompt import PipeLLMPrompt, PipeLLMPromptOutput
from pipelex.pipe_works.pipe_job_factory import PipeJobFactory
diff --git a/tests/integration/pipelex/pipes/test_pipe_running_variants.py b/tests/integration/pipelex/pipes/test_pipe_running_variants.py
index 4be3baa88..09d0a9e06 100644
--- a/tests/integration/pipelex/pipes/test_pipe_running_variants.py
+++ b/tests/integration/pipelex/pipes/test_pipe_running_variants.py
@@ -6,13 +6,13 @@ from pytest import FixtureRequest
from pipelex import log, pretty_print
from pipelex.config import get_config
-from pipelex.core.pipe_output import PipeOutput
-from pipelex.core.pipe_run_params import PipeOutputMultiplicity, PipeRunMode
-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory
-from pipelex.core.stuff import Stuff
-from pipelex.core.stuff_factory import StuffBlueprint
-from pipelex.core.working_memory import WorkingMemory
-from pipelex.core.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.memory.working_memory import WorkingMemory
+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.pipes.pipe_output import PipeOutput
+from pipelex.core.pipes.pipe_run_params import PipeOutputMultiplicity, PipeRunMode
+from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory
+from pipelex.core.stuffs.stuff import Stuff
+from pipelex.core.stuffs.stuff_factory import StuffBlueprint
from pipelex.hub import get_library_manager, get_pipe_router, get_report_delegate
from pipelex.pipeline.activity.activity_handler import ActivityHandlerForResultFiles
from pipelex.pipeline.job_metadata import JobMetadata
@@ -147,9 +147,11 @@ class TestPipeRunningVariants:
exception: Type[Exception],
expected_error_message: str,
):
- failing_pipelines_path = get_config().pipelex.library_config.failing_pipelines_path
+ failing_pipelines_file_paths = get_config().pipelex.library_config.failing_pipelines_file_paths
library_manager = get_library_manager()
- library_manager.load_combo_libraries(library_paths=[Path(failing_pipelines_path)])
+ library_manager.load_libraries(
+ library_file_paths=[Path(failing_pipeline_file_path) for failing_pipeline_file_path in failing_pipelines_file_paths]
+ )
log.verbose(f"This pipe '{pipe_code}' is supposed to cause an error of type: {exception.__name__}")
with pytest.raises(exception) as exc:
diff --git a/tests/integration/pipelex/plugins/test_external_plugin.py b/tests/integration/pipelex/plugins/test_external_plugin.py
index def57eaec..0e10cbcec 100644
--- a/tests/integration/pipelex/plugins/test_external_plugin.py
+++ b/tests/integration/pipelex/plugins/test_external_plugin.py
@@ -11,7 +11,7 @@ from pipelex.cogt.llm.llm_job_factory import LLMJobFactory
from pipelex.cogt.llm.llm_models.llm_setting import LLMSetting, LLMSettingChoices
from pipelex.cogt.llm.llm_worker_abstract import LLMWorkerAbstract
from pipelex.cogt.llm.token_category import NbTokensByCategoryDict, TokenCategory
-from pipelex.core.concept_native import NativeConcept
+from pipelex.core.concepts.concept_native import NativeConcept
from pipelex.hub import get_inference_manager, get_pipe_router, get_report_delegate
from pipelex.pipe_operators.pipe_llm import PipeLLM, PipeLLMOutput
from pipelex.pipe_operators.pipe_llm_prompt import PipeLLMPrompt
diff --git a/tests/integration/pipelex/test_data.py b/tests/integration/pipelex/test_data.py
index eba365325..c1f817596 100644
--- a/tests/integration/pipelex/test_data.py
+++ b/tests/integration/pipelex/test_data.py
@@ -1,15 +1,15 @@
from typing import ClassVar, List, Optional, Tuple, Type
-from pipelex.core.pipe_run_params import PipeOutputMultiplicity
-from pipelex.core.stuff import Stuff
-from pipelex.core.stuff_content import (
+from pipelex.core.pipes.pipe_run_params import PipeOutputMultiplicity
+from pipelex.core.stuffs.stuff import Stuff
+from pipelex.core.stuffs.stuff_content import (
ImageContent,
ListContent,
PDFContent,
StructuredContent,
TextContent,
)
-from pipelex.core.stuff_factory import StuffBlueprint, StuffFactory
+from pipelex.core.stuffs.stuff_factory import StuffBlueprint, StuffFactory
from pipelex.exceptions import PipeStackOverflowError
from pipelex.pipe_operators.pipe_ocr import PIPE_OCR_INPUT_NAME
from tests.cases import ImageTestCases, PDFTestCases
diff --git a/tests/integration/pipelex/test_fundamentals.py b/tests/integration/pipelex/test_fundamentals.py
index 6e751b674..551cf042a 100644
--- a/tests/integration/pipelex/test_fundamentals.py
+++ b/tests/integration/pipelex/test_fundamentals.py
@@ -13,8 +13,7 @@ class TestFundamentals:
pass
def test_validate_libraries(self):
- pipelex_instance = Pipelex.get_instance()
- pipelex_instance.validate_libraries()
+ Pipelex.get_instance().validate_libraries()
@pytest.mark.asyncio(loop_scope="class")
async def test_dry_run_all_pipes(self):
diff --git a/tests/integration/pipelex/test_inline_structure_concepts.py b/tests/integration/pipelex/test_inline_structure_concepts.py
index ee6c326fd..dd1ba11ab 100644
--- a/tests/integration/pipelex/test_inline_structure_concepts.py
+++ b/tests/integration/pipelex/test_inline_structure_concepts.py
@@ -1,10 +1,18 @@
"""Integration tests for inline structure definitions in concepts."""
+from typing import Dict
+
import pytest
-from pipelex.core.concept_factory import ConceptBlueprint, ConceptFactory
-from pipelex.core.stuff_content import StructuredContent
-from pipelex.exceptions import ConceptFactoryError, StructureClassError
+from pipelex.core.concepts.concept_blueprint import (
+ ConceptBlueprint,
+ ConceptStructureBlueprint,
+ ConceptStructureBlueprintFieldType,
+ ConceptStructureBlueprintType,
+)
+from pipelex.core.concepts.concept_factory import ConceptFactory
+from pipelex.core.stuffs.stuff_content import StructuredContent
+from pipelex.exceptions import StructureClassError
from pipelex.hub import get_class_registry
@@ -14,11 +22,19 @@ class TestInlineStructureConcepts:
def test_inline_structure_definition_creation(self):
"""Test that concepts with inline structure definitions are created correctly."""
# Define inline structure with mixed syntax
- inline_structure = {
- "dominant_feature": {"type": "text", "definition": "The most important feature", "required": True},
- "visual_elements": {"type": "text", "definition": "Key visual elements", "required": True},
- "composition": "Analysis of the image composition",
- "color_palette": "Description of the main colors",
+ inline_structure: Dict[str, ConceptStructureBlueprintType] = {
+ "dominant_feature": ConceptStructureBlueprint(
+ type=ConceptStructureBlueprintFieldType.TEXT, definition="The most important feature", required=False
+ ),
+ "visual_elements": ConceptStructureBlueprint(
+ type=ConceptStructureBlueprintFieldType.TEXT, definition="Key visual elements", required=False
+ ),
+ "composition": ConceptStructureBlueprint(
+ type=ConceptStructureBlueprintFieldType.TEXT, definition="Analysis of the image composition", required=False
+ ),
+ "color_palette": ConceptStructureBlueprint(
+ type=ConceptStructureBlueprintFieldType.TEXT, definition="Description of the main colors", required=False
+ ),
"mood_atmosphere": "The overall mood or atmosphere",
}
@@ -82,13 +98,26 @@ class TestInlineStructureConcepts:
def test_inline_structure_with_complex_types(self):
"""Test inline structure with complex field types."""
- inline_structure = {
- "title": {"type": "text", "definition": "Document title", "required": True},
- "tags": {"type": "list", "item_type": "text", "definition": "List of tags"},
- "metadata": {"type": "dict", "key_type": "text", "value_type": "text", "definition": "Metadata dictionary"},
- "priority": {"choices": ["low", "medium", "high"], "definition": "Priority level"},
- "page_count": {"type": "integer", "definition": "Number of pages"},
- "is_active": {"type": "boolean", "definition": "Whether document is active"},
+ inline_structure: Dict[str, ConceptStructureBlueprintType] = {
+ "title": ConceptStructureBlueprint(type=ConceptStructureBlueprintFieldType.TEXT, definition="Document title"),
+ "tags": ConceptStructureBlueprint(
+ type=ConceptStructureBlueprintFieldType.LIST,
+ item_type=ConceptStructureBlueprintFieldType.TEXT,
+ definition="List of tags",
+ required=False,
+ ),
+ "metadata": ConceptStructureBlueprint(
+ type=ConceptStructureBlueprintFieldType.DICT,
+ key_type=ConceptStructureBlueprintFieldType.TEXT,
+ value_type=ConceptStructureBlueprintFieldType.TEXT,
+ definition="Metadata dictionary",
+ required=False,
+ ),
+ "priority": ConceptStructureBlueprint(choices=["low", "medium", "high"], definition="Priority level", required=False),
+ "page_count": ConceptStructureBlueprint(type=ConceptStructureBlueprintFieldType.INTEGER, definition="Number of pages", required=False),
+ "is_active": ConceptStructureBlueprint(
+ type=ConceptStructureBlueprintFieldType.BOOLEAN, definition="Whether document is active", required=False
+ ),
}
blueprint = ConceptBlueprint(definition="Complex document structure", structure=inline_structure)
@@ -121,38 +150,32 @@ class TestInlineStructureConcepts:
def test_invalid_string_reference_raises_error(self):
"""Test that invalid string references raise appropriate errors."""
- blueprint = ConceptBlueprint(definition="Test invalid reference", structure="NonExistentClass")
-
- with pytest.raises(StructureClassError, match="Structure class 'NonExistentClass'"):
- ConceptFactory.make_concept_from_blueprint(domain="test_domain", code="TestInvalid", concept_blueprint=blueprint)
-
- def test_inline_structure_generation_error_handling(self):
- """Test error handling when inline structure generation fails."""
- # Create invalid structure definition that should cause generation to fail
- invalid_structure = {
- "invalid_field": {"type": "nonexistent_type", "definition": "This should fail"},
- }
-
- blueprint = ConceptBlueprint(definition="Test error handling", structure=invalid_structure)
-
- with pytest.raises(ConceptFactoryError, match="Error generating structure class"):
- ConceptFactory.make_concept_from_blueprint(domain="test_domain", code="TestError", concept_blueprint=blueprint)
+ with pytest.raises(
+ StructureClassError,
+ match="Structure class 'NonExistentClass' set for concept 'TestInvalidRef' in domain 'test_domain' is \
+not a registered subclass of StuffContent",
+ ):
+ _ = ConceptFactory.make_concept_from_blueprint(
+ domain="test_domain",
+ code="TestInvalidRef",
+ concept_blueprint=ConceptBlueprint(definition="Test invalid reference", structure="NonExistentClass"),
+ )
def test_multiple_inline_structures_do_not_conflict(self):
"""Test that multiple inline structures with same field names don't conflict."""
# First structure
- structure1 = {
- "name": {"type": "text", "definition": "Person name", "required": True},
- "age": {"type": "integer", "definition": "Person age"},
+ structure1: Dict[str, ConceptStructureBlueprintType] = {
+ "name": ConceptStructureBlueprint(type=ConceptStructureBlueprintFieldType.TEXT, definition="Person name"),
+ "age": ConceptStructureBlueprint(type=ConceptStructureBlueprintFieldType.INTEGER, definition="Person age", required=False),
}
blueprint1 = ConceptBlueprint(definition="Person information", structure=structure1)
concept1 = ConceptFactory.make_concept_from_blueprint(domain="test_domain", code="Person", concept_blueprint=blueprint1)
# Second structure with same field names but different context
- structure2 = {
- "name": {"type": "text", "definition": "Product name", "required": True},
- "age": {"type": "integer", "definition": "Product age in days"},
+ structure2: Dict[str, ConceptStructureBlueprintType] = {
+ "name": "Product name",
+ "age": ConceptStructureBlueprint(type=ConceptStructureBlueprintFieldType.INTEGER, definition="Product age in days", required=False),
}
blueprint2 = ConceptBlueprint(definition="Product information", structure=structure2)
diff --git a/tests/integration/pipelex/test_libraries.py b/tests/integration/pipelex/test_libraries.py
index 428383e0c..89f42cb75 100644
--- a/tests/integration/pipelex/test_libraries.py
+++ b/tests/integration/pipelex/test_libraries.py
@@ -2,6 +2,7 @@
make t TEST=TestLibraries
"""
+from pathlib import Path
from typing import Optional
import pytest
@@ -11,9 +12,9 @@ from rich.table import Table
from pipelex import pretty_print
from pipelex.config import get_config
-from pipelex.core.concept_library import ConceptLibrary
-from pipelex.core.pipe_library import PipeLibrary
-from pipelex.libraries.library_manager import LibraryManager
+from pipelex.core.concepts.concept_library import ConceptLibrary
+from pipelex.core.pipes.pipe_library import PipeLibrary
+from pipelex.libraries.library_manager_factory import LibraryManagerFactory
from tests.integration.pipelex.test_data import LibraryTestCases
@@ -90,10 +91,9 @@ class TestLibraries:
known_concept: str,
known_pipe: str,
):
- library_manager = LibraryManager.make_empty(config_folder_path="pipelex_libraries")
- test_pipelines_path = get_config().pipelex.library_config.test_pipelines_path
- test_library_paths = library_manager.list_toml_files_from_path(library_paths=[test_pipelines_path])
- library_manager.load_combo_libraries(library_paths=test_library_paths)
+ library_manager = LibraryManagerFactory.make_empty(config_dir_path="pipelex/libraries")
+ test_pipelines_dir = [Path(get_config().pipelex.library_config.test_pipelines_dir_path)]
+ library_manager.load_libraries(library_dirs=test_pipelines_dir)
# Verify that libraries were loaded
assert len(library_manager.concept_library.root) > 0, "No concepts were loaded"
assert len(library_manager.pipe_library.root) > 0, "No pipes were loaded"
diff --git a/tests/integration/pipelex/tools/class_registry/test_class_registry_utils.py b/tests/integration/pipelex/tools/class_registry/test_class_registry_utils.py
index 6e42f5fa9..2e4e40af4 100644
--- a/tests/integration/pipelex/tools/class_registry/test_class_registry_utils.py
+++ b/tests/integration/pipelex/tools/class_registry/test_class_registry_utils.py
@@ -2,7 +2,7 @@ from pathlib import Path
import pytest
-from pipelex.core.stuff_content import StuffContent
+from pipelex.core.stuffs.stuff_content import StuffContent
from pipelex.hub import get_class_registry
from pipelex.tools.class_registry_utils import ClassRegistryUtils
from tests.cases import ClassRegistryTestCases
diff --git a/tests/test_pipelines/concept_library/is_compatible_concept_code.py b/tests/test_pipelines/concept_library/is_compatible_concept_code.py
index 14ebdaaa9..e3ddcd65d 100644
--- a/tests/test_pipelines/concept_library/is_compatible_concept_code.py
+++ b/tests/test_pipelines/concept_library/is_compatible_concept_code.py
@@ -2,7 +2,7 @@ from typing import Dict, List, Optional
from pydantic import Field
-from pipelex.core.stuff_content import StructuredContent
+from pipelex.core.stuffs.stuff_content import StructuredContent
class FundamentalsDoc(StructuredContent):
diff --git a/tests/test_pipelines/datetime.py b/tests/test_pipelines/datetime.py
index e4315e917..1cbb94df8 100644
--- a/tests/test_pipelines/datetime.py
+++ b/tests/test_pipelines/datetime.py
@@ -1,6 +1,6 @@
from datetime import datetime
-from pipelex.core.stuff_content import StructuredContent
+from pipelex.core.stuffs.stuff_content import StructuredContent
class DateTimeEvent(StructuredContent):
diff --git a/tests/test_pipelines/discord_newsletter.py b/tests/test_pipelines/discord_newsletter.py
index 62a12a059..2680b2110 100644
--- a/tests/test_pipelines/discord_newsletter.py
+++ b/tests/test_pipelines/discord_newsletter.py
@@ -2,7 +2,7 @@ from typing import List
from pydantic import Field
-from pipelex.core.stuff_content import StructuredContent
+from pipelex.core.stuffs.stuff_content import StructuredContent
class Attachment(StructuredContent):
diff --git a/tests/test_pipelines/misc_tests/test_structures.py b/tests/test_pipelines/misc_tests/test_structures.py
index 5444d6a55..ea913265b 100644
--- a/tests/test_pipelines/misc_tests/test_structures.py
+++ b/tests/test_pipelines/misc_tests/test_structures.py
@@ -1,4 +1,4 @@
-from pipelex.core.stuff_content import StructuredContent
+from pipelex.core.stuffs.stuff_content import StructuredContent
class Article(StructuredContent):
diff --git a/tests/test_pipelines/pipe_controllers/pipe_batch/text_list.py b/tests/test_pipelines/pipe_controllers/pipe_batch/text_list.py
index 1b8052713..242ee47a1 100644
--- a/tests/test_pipelines/pipe_controllers/pipe_batch/text_list.py
+++ b/tests/test_pipelines/pipe_controllers/pipe_batch/text_list.py
@@ -1,4 +1,4 @@
-from pipelex.core.stuff_content import ListContent, TextContent
+from pipelex.core.stuffs.stuff_content import ListContent, TextContent
class TextList(ListContent[TextContent]):
diff --git a/tests/test_pipelines/pipe_controllers/pipe_condition/pipe_condition.py b/tests/test_pipelines/pipe_controllers/pipe_condition/pipe_condition.py
index acf5c8d6b..ffc36a013 100644
--- a/tests/test_pipelines/pipe_controllers/pipe_condition/pipe_condition.py
+++ b/tests/test_pipelines/pipe_controllers/pipe_condition/pipe_condition.py
@@ -1,6 +1,6 @@
from pydantic import Field
-from pipelex.core.stuff_content import StructuredContent
+from pipelex.core.stuffs.stuff_content import StructuredContent
class CategoryInput(StructuredContent):
diff --git a/tests/test_pipelines/pipe_controllers/pipe_condition/pipe_condition_complex.py b/tests/test_pipelines/pipe_controllers/pipe_condition/pipe_condition_complex.py
index 41527c4a0..791c1ffe1 100644
--- a/tests/test_pipelines/pipe_controllers/pipe_condition/pipe_condition_complex.py
+++ b/tests/test_pipelines/pipe_controllers/pipe_condition/pipe_condition_complex.py
@@ -2,7 +2,7 @@ from typing import Literal
from pydantic import Field
-from pipelex.core.stuff_content import StructuredContent
+from pipelex.core.stuffs.stuff_content import StructuredContent
class DocumentRequest(StructuredContent):
diff --git a/tests/test_pipelines/pipe_controllers/pipe_parallel/pipe_parallel.py b/tests/test_pipelines/pipe_controllers/pipe_parallel/pipe_parallel.py
index 7f8b63358..fea75b4e8 100644
--- a/tests/test_pipelines/pipe_controllers/pipe_parallel/pipe_parallel.py
+++ b/tests/test_pipelines/pipe_controllers/pipe_parallel/pipe_parallel.py
@@ -1,6 +1,6 @@
from pydantic import Field
-from pipelex.core.stuff_content import StructuredContent
+from pipelex.core.stuffs.stuff_content import StructuredContent
class DocumentInput(StructuredContent):
diff --git a/tests/test_pipelines/pipe_controllers/pipe_sequence/pipe_sequence.py b/tests/test_pipelines/pipe_controllers/pipe_sequence/pipe_sequence.py
index b7c7fb4b3..d1f6dcdaf 100644
--- a/tests/test_pipelines/pipe_controllers/pipe_sequence/pipe_sequence.py
+++ b/tests/test_pipelines/pipe_controllers/pipe_sequence/pipe_sequence.py
@@ -1,6 +1,6 @@
from pydantic import Field
-from pipelex.core.stuff_content import StructuredContent
+from pipelex.core.stuffs.stuff_content import StructuredContent
class Document(StructuredContent):
diff --git a/tests/test_pipelines/pipe_llm_prompt.py b/tests/test_pipelines/pipe_llm_prompt.py
index d920a22a6..79920a4f1 100644
--- a/tests/test_pipelines/pipe_llm_prompt.py
+++ b/tests/test_pipelines/pipe_llm_prompt.py
@@ -2,7 +2,7 @@ from typing import List, Literal
from pydantic import Field
-from pipelex.core.stuff_content import StructuredContent
+from pipelex.core.stuffs.stuff_content import StructuredContent
from pipelex.types import StrEnum
diff --git a/tests/test_pipelines/test_content_factory.py b/tests/test_pipelines/test_content_factory.py
index e6a11cd34..d40a2cb5e 100644
--- a/tests/test_pipelines/test_content_factory.py
+++ b/tests/test_pipelines/test_content_factory.py
@@ -1,4 +1,4 @@
-from pipelex.core.stuff_content import StructuredContent
+from pipelex.core.stuffs.stuff_content import StructuredContent
class MockRegisteredContent(StructuredContent):
diff --git a/tests/test_pipelines/tricky_questions.py b/tests/test_pipelines/tricky_questions.py
index c72be0fdd..b591b8c21 100644
--- a/tests/test_pipelines/tricky_questions.py
+++ b/tests/test_pipelines/tricky_questions.py
@@ -1,4 +1,4 @@
-from pipelex.core.stuff_content import StructuredContent
+from pipelex.core.stuffs.stuff_content import StructuredContent
class ThoughtfulAnswer(StructuredContent):
diff --git a/tests/unit/pipelex/client/test_api_serialization.py b/tests/unit/pipelex/client/test_api_serialization.py
index 504f6e3d0..e3d69a0a4 100644
--- a/tests/unit/pipelex/client/test_api_serialization.py
+++ b/tests/unit/pipelex/client/test_api_serialization.py
@@ -8,11 +8,11 @@ import pytest
from pydantic import BaseModel
from pipelex.client.api_serializer import ApiSerializer
-from pipelex.core.concept_native import NativeConcept
-from pipelex.core.stuff_content import NumberContent
-from pipelex.core.stuff_factory import StuffFactory
-from pipelex.core.working_memory import WorkingMemory
-from pipelex.core.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.concepts.concept_native import NativeConcept
+from pipelex.core.memory.working_memory import WorkingMemory
+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.stuffs.stuff_content import NumberContent
+from pipelex.core.stuffs.stuff_factory import StuffFactory
from tests.test_pipelines.datetime import DateTimeEvent
diff --git a/tests/unit/pipelex/core/bundles/test_pipelex_bundle.py b/tests/unit/pipelex/core/bundles/test_pipelex_bundle.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/unit/pipelex/core/test_concept_library.py b/tests/unit/pipelex/core/concepts/test_concept_library.py
similarity index 98%
rename from tests/unit/pipelex/core/test_concept_library.py
rename to tests/unit/pipelex/core/concepts/test_concept_library.py
index 58603661b..a719a4b89 100644
--- a/tests/unit/pipelex/core/test_concept_library.py
+++ b/tests/unit/pipelex/core/concepts/test_concept_library.py
@@ -1,6 +1,6 @@
-from pipelex.core.concept import Concept
-from pipelex.core.concept_native import NativeConcept
-from pipelex.core.concept_provider_abstract import ConceptProviderAbstract
+from pipelex.core.concepts.concept import Concept
+from pipelex.core.concepts.concept_native import NativeConcept
+from pipelex.core.concepts.concept_provider_abstract import ConceptProviderAbstract
class Testget_concept_providerIsNativeConcept:
diff --git a/tests/unit/pipelex/core/test_concept_pydantic_validation.py b/tests/unit/pipelex/core/concepts/test_concept_pydantic_validation.py
similarity index 98%
rename from tests/unit/pipelex/core/test_concept_pydantic_validation.py
rename to tests/unit/pipelex/core/concepts/test_concept_pydantic_validation.py
index e170387a3..023df56f6 100644
--- a/tests/unit/pipelex/core/test_concept_pydantic_validation.py
+++ b/tests/unit/pipelex/core/concepts/test_concept_pydantic_validation.py
@@ -1,7 +1,7 @@
import pytest
-from pipelex.core.concept import Concept
-from pipelex.core.concept_native import NativeConcept
+from pipelex.core.concepts.concept import Concept
+from pipelex.core.concepts.concept_native import NativeConcept
from pipelex.exceptions import ConceptCodeError, ConceptDomainError
diff --git a/tests/unit/pipelex/core/test_concept_refines.py b/tests/unit/pipelex/core/concepts/test_concept_refines.py
similarity index 97%
rename from tests/unit/pipelex/core/test_concept_refines.py
rename to tests/unit/pipelex/core/concepts/test_concept_refines.py
index c5b3cc696..c5acb9377 100644
--- a/tests/unit/pipelex/core/test_concept_refines.py
+++ b/tests/unit/pipelex/core/concepts/test_concept_refines.py
@@ -1,7 +1,7 @@
import pytest
-from pipelex.core.concept import Concept
-from pipelex.core.concept_native import NativeConcept
+from pipelex.core.concepts.concept import Concept
+from pipelex.core.concepts.concept_native import NativeConcept
from pipelex.exceptions import ConceptCodeError, ConceptDomainError, ConceptError
diff --git a/tests/unit/pipelex/core/test_dry_working_memory.py b/tests/unit/pipelex/core/memory/test_dry_working_memory.py
similarity index 97%
rename from tests/unit/pipelex/core/test_dry_working_memory.py
rename to tests/unit/pipelex/core/memory/test_dry_working_memory.py
index 17b76a39d..96e100e61 100644
--- a/tests/unit/pipelex/core/test_dry_working_memory.py
+++ b/tests/unit/pipelex/core/memory/test_dry_working_memory.py
@@ -6,10 +6,10 @@ import pytest
from pytest import FixtureRequest
from pipelex import log
-from pipelex.core.concept_native import NativeConcept
-from pipelex.core.pipe_input_spec import TypedNamedInputRequirement
-from pipelex.core.stuff_content import PageContent, TextContent
-from pipelex.core.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.concepts.concept_native import NativeConcept
+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.pipes.pipe_input_spec import TypedNamedInputRequirement
+from pipelex.core.stuffs.stuff_content import PageContent, TextContent
from tests.test_pipelines.tricky_questions import ThoughtfulAnswer
diff --git a/tests/unit/pipelex/core/test_working_memory.py b/tests/unit/pipelex/core/memory/test_working_memory.py
similarity index 95%
rename from tests/unit/pipelex/core/test_working_memory.py
rename to tests/unit/pipelex/core/memory/test_working_memory.py
index 0e023b08b..10fca9b4d 100644
--- a/tests/unit/pipelex/core/test_working_memory.py
+++ b/tests/unit/pipelex/core/memory/test_working_memory.py
@@ -2,11 +2,11 @@ from typing import ClassVar, List, Tuple
import pytest
-from pipelex.core.concept_native import NativeConcept
-from pipelex.core.stuff_content import HtmlContent, ImageContent, ListContent, NumberContent, TextAndImagesContent, TextContent
-from pipelex.core.stuff_factory import StuffFactory
-from pipelex.core.working_memory import WorkingMemory
-from pipelex.core.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.concepts.concept_native import NativeConcept
+from pipelex.core.memory.working_memory import WorkingMemory
+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.stuffs.stuff_content import HtmlContent, ImageContent, ListContent, NumberContent, TextAndImagesContent, TextContent
+from pipelex.core.stuffs.stuff_factory import StuffFactory
class TestWorkingMemoryData:
diff --git a/tests/unit/pipelex/core/test_working_memory_factory.py b/tests/unit/pipelex/core/memory/test_working_memory_factory.py
similarity index 96%
rename from tests/unit/pipelex/core/test_working_memory_factory.py
rename to tests/unit/pipelex/core/memory/test_working_memory_factory.py
index cde6aa5e6..2082ae354 100644
--- a/tests/unit/pipelex/core/test_working_memory_factory.py
+++ b/tests/unit/pipelex/core/memory/test_working_memory_factory.py
@@ -1,7 +1,7 @@
from pipelex.client.protocol import CompactMemory
-from pipelex.core.concept_native import NativeConcept
-from pipelex.core.stuff_content import ImageContent, PageContent, TextAndImagesContent, TextContent
-from pipelex.core.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.concepts.concept_native import NativeConcept
+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.stuffs.stuff_content import ImageContent, PageContent, TextAndImagesContent, TextContent
class TestWorkingMemoryFactory:
diff --git a/tests/unit/pipelex/core/test_pipe_condition_validation.py b/tests/unit/pipelex/core/pipes/test_pipe_condition_validation.py
similarity index 74%
rename from tests/unit/pipelex/core/test_pipe_condition_validation.py
rename to tests/unit/pipelex/core/pipes/test_pipe_condition_validation.py
index 18116b8e8..6f3efde60 100644
--- a/tests/unit/pipelex/core/test_pipe_condition_validation.py
+++ b/tests/unit/pipelex/core/pipes/test_pipe_condition_validation.py
@@ -1,4 +1,4 @@
-from pipelex.core.pipe_input_spec import PipeInputSpec
+from pipelex.core.pipes.pipe_input_spec import InputRequirementBlueprint, PipeInputSpec
from pipelex.pipe_controllers.pipe_condition import PipeCondition
@@ -10,7 +10,9 @@ class TestPipeConditionValidation:
pipe_condition = PipeCondition(
domain="test_domain",
code="test_condition",
- inputs=PipeInputSpec.make_from_dict(concepts_dict={"input_var": "test_domain.Text"}),
+ inputs=PipeInputSpec.make_from_blueprint(
+ domain="test_domain", blueprint={"input_var": InputRequirementBlueprint(concept_code="test_domain.Text")}
+ ),
output_concept_code="test_domain.ProcessedText",
expression="input_var",
pipe_map={"value1": "pipe_a", "value2": "pipe_b"},
@@ -29,7 +31,9 @@ class TestPipeConditionValidation:
pipe_condition_template = PipeCondition(
domain="test_domain",
code="test_condition_template",
- inputs=PipeInputSpec.make_from_dict(concepts_dict={"var": "test_domain.Text"}),
+ inputs=PipeInputSpec.make_from_blueprint(
+ domain="test_domain", blueprint={"var": InputRequirementBlueprint(concept_code="test_domain.Text")}
+ ),
output_concept_code="test_domain.Result",
expression_template="{{ var }}",
pipe_map={"value": "target_pipe"},
@@ -39,7 +43,9 @@ class TestPipeConditionValidation:
pipe_condition_expr = PipeCondition(
domain="test_domain",
code="test_condition_expr",
- inputs=PipeInputSpec.make_from_dict(concepts_dict={"var": "test_domain.Text"}),
+ inputs=PipeInputSpec.make_from_blueprint(
+ domain="test_domain", blueprint={"var": InputRequirementBlueprint(concept_code="test_domain.Text")}
+ ),
output_concept_code="test_domain.Result",
expression="var",
pipe_map={"value": "target_pipe"},
diff --git a/tests/unit/pipelex/core/test_pipe_parallel_validation.py b/tests/unit/pipelex/core/pipes/test_pipe_parallel_validation.py
similarity index 81%
rename from tests/unit/pipelex/core/test_pipe_parallel_validation.py
rename to tests/unit/pipelex/core/pipes/test_pipe_parallel_validation.py
index 32186496a..3d8e2826c 100644
--- a/tests/unit/pipelex/core/test_pipe_parallel_validation.py
+++ b/tests/unit/pipelex/core/pipes/test_pipe_parallel_validation.py
@@ -1,4 +1,4 @@
-from pipelex.core.pipe_input_spec import PipeInputSpec
+from pipelex.core.pipes.pipe_input_spec import InputRequirementBlueprint, PipeInputSpec
from pipelex.pipe_controllers.pipe_parallel import PipeParallel
from pipelex.pipe_controllers.sub_pipe import SubPipe
from pipelex.pipe_operators.pipe_llm import PipeLLM
@@ -32,7 +32,13 @@ class TestPipeParallelValidation:
pipe_parallel = PipeParallel(
domain="test_domain",
code="parallel_document_processor",
- inputs=PipeInputSpec.make_from_dict(concepts_dict={"document": "test_domain.document", "context": "test_domain.context"}),
+ inputs=PipeInputSpec.make_from_blueprint(
+ domain="test_domain",
+ blueprint={
+ "document": InputRequirementBlueprint(concept_code="test_domain.document"),
+ "context": InputRequirementBlueprint(concept_code="test_domain.context"),
+ },
+ ),
output_concept_code="test_domain.ProcessedAnalysis",
parallel_sub_pipes=[SubPipe(pipe_code="analyze_document", output_name="analysis_result")],
add_each_output=True,
@@ -55,7 +61,9 @@ class TestPipeParallelValidation:
pipe_parallel = PipeParallel(
domain="test_domain",
code="test_parallel",
- inputs=PipeInputSpec.make_from_dict(concepts_dict={"input_var": "test_domain.Text"}),
+ inputs=PipeInputSpec.make_from_blueprint(
+ domain="test_domain", blueprint={"input_var": InputRequirementBlueprint(concept_code="test_domain.Text")}
+ ),
output_concept_code="test_domain.ProcessedText",
parallel_sub_pipes=[SubPipe(pipe_code="test_pipe_1", output_name="result_1")],
add_each_output=True,
@@ -78,7 +86,13 @@ class TestPipeParallelValidation:
pipe_parallel = PipeParallel(
domain="test_domain",
code="parallel_document_processor",
- inputs=PipeInputSpec.make_from_dict(concepts_dict={"document": "test_domain.Document", "context": "test_domain.Context"}),
+ inputs=PipeInputSpec.make_from_blueprint(
+ domain="test_domain",
+ blueprint={
+ "document": InputRequirementBlueprint(concept_code="test_domain.Document"),
+ "context": InputRequirementBlueprint(concept_code="test_domain.Context"),
+ },
+ ),
output_concept_code="test_domain.ProcessedAnalysis",
parallel_sub_pipes=[], # No sub-pipes to avoid dependency issues
add_each_output=True,
diff --git a/tests/unit/pipelex/core/test_pipe_sequence_validation.py b/tests/unit/pipelex/core/pipes/test_pipe_sequence_validation.py
similarity index 77%
rename from tests/unit/pipelex/core/test_pipe_sequence_validation.py
rename to tests/unit/pipelex/core/pipes/test_pipe_sequence_validation.py
index c92816f2a..eac0444bf 100644
--- a/tests/unit/pipelex/core/test_pipe_sequence_validation.py
+++ b/tests/unit/pipelex/core/pipes/test_pipe_sequence_validation.py
@@ -1,4 +1,4 @@
-from pipelex.core.pipe_input_spec import PipeInputSpec
+from pipelex.core.pipes.pipe_input_spec import InputRequirementBlueprint, PipeInputSpec
from pipelex.pipe_controllers.pipe_sequence import PipeSequence
from pipelex.pipe_controllers.sub_pipe import SubPipe
@@ -11,7 +11,9 @@ class TestPipeSequenceValidation:
pipe_sequence = PipeSequence(
domain="test_domain",
code="test_sequence",
- inputs=PipeInputSpec.make_from_dict(concepts_dict={"text": "test_domain.Text"}),
+ inputs=PipeInputSpec.make_from_blueprint(
+ domain="test_domain", blueprint={"text": InputRequirementBlueprint(concept_code="test_domain.Text")}
+ ),
output_concept_code="test_domain.ProcessedText",
sequential_sub_pipes=[SubPipe(pipe_code="test_pipe_1", output_name="intermediate_result")],
)
@@ -27,7 +29,9 @@ class TestPipeSequenceValidation:
pipe_sequence = PipeSequence(
domain="test_domain",
code="test_sequence",
- inputs=PipeInputSpec.make_from_dict(concepts_dict={"initial_input": "test_domain.Text"}),
+ inputs=PipeInputSpec.make_from_blueprint(
+ domain="test_domain", blueprint={"initial_input": InputRequirementBlueprint(concept_code="test_domain.Text")}
+ ),
output_concept_code="test_domain.FinalOutput",
sequential_sub_pipes=[SubPipe(pipe_code="step_1", output_name="intermediate"), SubPipe(pipe_code="step_2", output_name="final_output")],
)
diff --git a/tests/unit/pipelex/core/test_stuff_content_factory.py b/tests/unit/pipelex/core/stuffs/test_stuff_content_factory.py
similarity index 98%
rename from tests/unit/pipelex/core/test_stuff_content_factory.py
rename to tests/unit/pipelex/core/stuffs/test_stuff_content_factory.py
index 38a4a4970..06e907e25 100644
--- a/tests/unit/pipelex/core/test_stuff_content_factory.py
+++ b/tests/unit/pipelex/core/stuffs/test_stuff_content_factory.py
@@ -2,8 +2,8 @@ from typing import Any, ClassVar, Dict, List, Tuple
import pytest
-from pipelex.core.stuff_content import StructuredContent, TextContent
-from pipelex.core.stuff_factory import StuffContentFactory
+from pipelex.core.stuffs.stuff_content import StructuredContent, TextContent
+from pipelex.core.stuffs.stuff_factory import StuffContentFactory
class TestCases:
diff --git a/tests/unit/pipelex/core/test_data/__init__.py b/tests/unit/pipelex/core/test_data/__init__.py
new file mode 100644
index 000000000..d63ec8f8f
--- /dev/null
+++ b/tests/unit/pipelex/core/test_data/__init__.py
@@ -0,0 +1,52 @@
+"""Organized test data for PipelexInterpreter tests."""
+
+from typing import ClassVar, List, Tuple
+
+from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint
+
+# Import all test data from organized modules
+from .complex.multi_feature import COMPLEX_TEST_CASES
+from .concepts.refining_concepts import REFINING_CONCEPT_TEST_CASES
+from .concepts.simple_concepts import SIMPLE_CONCEPT_TEST_CASES
+from .concepts.structured_concepts import STRUCTURED_CONCEPT_TEST_CASES
+from .domain.simple_domains import DOMAIN_TEST_CASES
+from .errors.invalid_toml import ERROR_TEST_CASES
+from .pipes.controllers.batch.pipe_batch import PIPE_BATCH_TEST_CASES
+from .pipes.controllers.condition.pipe_condition import PIPE_CONDITION_TEST_CASES
+from .pipes.controllers.parallel.pipe_parallel import PIPE_PARALLEL_TEST_CASES
+from .pipes.controllers.sequence.pipe_sequence import PIPE_SEQUENCE_TEST_CASES
+from .pipes.operators.func.pipe_func import PIPE_FUNC_TEST_CASES
+from .pipes.operators.img_gen.pipe_img_gen import PIPE_IMG_GEN_TEST_CASES
+from .pipes.operators.jinja2.pipe_jinja2 import PIPE_JINJA2_TEST_CASES
+from .pipes.operators.llm.pipe_llm import PIPE_LLM_TEST_CASES
+from .pipes.operators.ocr.pipe_ocr import PIPE_OCR_TEST_CASES
+
+
+class InterpreterTestCases:
+ """Test cases for PipelexInterpreter with TOML content and expected blueprints."""
+
+ # Aggregate all valid test cases from organized modules
+ VALID_TEST_CASES: ClassVar[List[Tuple[str, str, PipelexBundleBlueprint]]] = [
+ # Domain tests
+ *DOMAIN_TEST_CASES,
+ # Concept tests
+ *STRUCTURED_CONCEPT_TEST_CASES,
+ *REFINING_CONCEPT_TEST_CASES,
+ *SIMPLE_CONCEPT_TEST_CASES,
+ # Pipe operator tests
+ *PIPE_LLM_TEST_CASES,
+ *PIPE_OCR_TEST_CASES,
+ *PIPE_FUNC_TEST_CASES,
+ *PIPE_IMG_GEN_TEST_CASES,
+ *PIPE_JINJA2_TEST_CASES,
+ # Pipe controller tests
+ *PIPE_SEQUENCE_TEST_CASES,
+ *PIPE_CONDITION_TEST_CASES,
+ *PIPE_PARALLEL_TEST_CASES,
+ *PIPE_BATCH_TEST_CASES,
+ # Complex tests
+ *COMPLEX_TEST_CASES,
+ ]
+
+ # Error test cases
+ ERROR_TEST_CASES: ClassVar[List[Tuple[str, str]]] = ERROR_TEST_CASES
diff --git a/tests/unit/pipelex/core/test_data/complex/__init__.py b/tests/unit/pipelex/core/test_data/complex/__init__.py
new file mode 100644
index 000000000..bbba9d0b6
--- /dev/null
+++ b/tests/unit/pipelex/core/test_data/complex/__init__.py
@@ -0,0 +1 @@
+"""Complex test scenarios for PipelexInterpreter tests."""
diff --git a/tests/unit/pipelex/core/test_data/complex/multi_feature.py b/tests/unit/pipelex/core/test_data/complex/multi_feature.py
new file mode 100644
index 000000000..fac28db5b
--- /dev/null
+++ b/tests/unit/pipelex/core/test_data/complex/multi_feature.py
@@ -0,0 +1,67 @@
+"""Complex test cases combining multiple features."""
+
+from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint
+from pipelex.core.pipes.pipe_input_spec import InputRequirementBlueprint
+from pipelex.pipe_controllers.pipe_sequence_factory import PipeSequenceBlueprint
+from pipelex.pipe_controllers.sub_pipe_factory import SubPipeBlueprint
+from pipelex.pipe_operators.pipe_llm_factory import PipeLLMBlueprint
+
+COMPLEX_PIPES = (
+ "complex_pipes",
+ """domain = "complex_domain"
+definition = "Domain with multiple pipe types"
+
+[concept]
+InputData = "Input data concept"
+ProcessedData = "Processed data concept"
+
+[pipe.llm_pipe]
+type = "PipeLLM"
+definition = "Generate content"
+output = "ProcessedData"
+inputs = { data = "InputData" }
+prompt_template = "Process this data: @data"
+
+[pipe.sequence_pipe]
+type = "PipeSequence"
+definition = "Sequential processing"
+output = "ProcessedData"
+inputs = { input_data = "InputData" }
+steps = [
+ { pipe = "llm_pipe", result = "llm_result" },
+ { pipe = "final_step", result = "final_output" },
+]
+""",
+ PipelexBundleBlueprint(
+ domain="complex_domain",
+ definition="Domain with multiple pipe types",
+ concept={
+ "InputData": "Input data concept",
+ "ProcessedData": "Processed data concept",
+ },
+ pipe={
+ "llm_pipe": PipeLLMBlueprint(
+ type="PipeLLM",
+ definition="Generate content",
+ inputs={"data": InputRequirementBlueprint(concept_code="InputData")},
+ output="ProcessedData",
+ prompt_template="Process this data: @data",
+ ),
+ "sequence_pipe": PipeSequenceBlueprint(
+ type="PipeSequence",
+ definition="Sequential processing",
+ inputs={"input_data": InputRequirementBlueprint(concept_code="InputData")},
+ output="ProcessedData",
+ steps=[
+ SubPipeBlueprint(pipe="llm_pipe", result="llm_result"),
+ SubPipeBlueprint(pipe="final_step", result="final_output"),
+ ],
+ ),
+ },
+ ),
+)
+
+# Export all complex test cases
+COMPLEX_TEST_CASES = [
+ COMPLEX_PIPES,
+]
diff --git a/tests/unit/pipelex/core/test_data/concepts/__init__.py b/tests/unit/pipelex/core/test_data/concepts/__init__.py
new file mode 100644
index 000000000..7d431fc5a
--- /dev/null
+++ b/tests/unit/pipelex/core/test_data/concepts/__init__.py
@@ -0,0 +1 @@
+"""Concept-related test data for PipelexInterpreter tests."""
diff --git a/tests/unit/pipelex/core/test_data/concepts/refining_concepts.py b/tests/unit/pipelex/core/test_data/concepts/refining_concepts.py
new file mode 100644
index 000000000..7f486319d
--- /dev/null
+++ b/tests/unit/pipelex/core/test_data/concepts/refining_concepts.py
@@ -0,0 +1,33 @@
+"""Refining concept test cases."""
+
+from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint
+from pipelex.core.concepts.concept_blueprint import ConceptBlueprint
+
+CONCEPTS_WITH_REFINES = (
+ "concepts_with_refines",
+ """domain = "refining_concepts"
+definition = "Domain with concepts that refine others"
+
+[concept]
+Concept1 = "A concept1"
+Concept2 = "A concept2"
+
+[concept.Concept3]
+definition = "A concept3"
+refines = "Concept2"
+""",
+ PipelexBundleBlueprint(
+ domain="refining_concepts",
+ definition="Domain with concepts that refine others",
+ concept={
+ "Concept1": "A concept1",
+ "Concept2": "A concept2",
+ "Concept3": ConceptBlueprint(definition="A concept3", refines="Concept2"),
+ },
+ ),
+)
+
+# Export all refining concept test cases
+REFINING_CONCEPT_TEST_CASES = [
+ CONCEPTS_WITH_REFINES,
+]
diff --git a/tests/unit/pipelex/core/test_data/concepts/simple_concepts.py b/tests/unit/pipelex/core/test_data/concepts/simple_concepts.py
new file mode 100644
index 000000000..ff4f54df1
--- /dev/null
+++ b/tests/unit/pipelex/core/test_data/concepts/simple_concepts.py
@@ -0,0 +1,36 @@
+"""Simple concept test cases."""
+
+from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint
+
+SIMPLE_CONCEPTS = (
+ "simple_concept",
+ """domain = "simple_concept"
+definition = "A simple concept"
+
+[concept]
+Concept1 = "A concept"
+Concept2 = "A concept"
+""",
+ PipelexBundleBlueprint(
+ domain="simple_concept",
+ definition="A simple concept",
+ concept={
+ "Concept1": "A concept",
+ "Concept2": "A concept",
+ },
+ ),
+)
+
+EMPTY_CONCEPTS = (
+ "empty_concepts",
+ """domain = "empty_concepts"
+definition = "Domain with empty concept section"
+""",
+ PipelexBundleBlueprint(domain="empty_concepts", definition="Domain with empty concept section", concept={}),
+)
+
+# Export all simple concept test cases
+SIMPLE_CONCEPT_TEST_CASES = [
+ SIMPLE_CONCEPTS,
+ EMPTY_CONCEPTS,
+]
diff --git a/tests/unit/pipelex/core/test_data/concepts/structured_concepts.py b/tests/unit/pipelex/core/test_data/concepts/structured_concepts.py
new file mode 100644
index 000000000..61c1f6fc4
--- /dev/null
+++ b/tests/unit/pipelex/core/test_data/concepts/structured_concepts.py
@@ -0,0 +1,80 @@
+"""Structured concept test cases."""
+
+from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint
+from pipelex.core.concepts.concept_blueprint import ConceptBlueprint, ConceptStructureBlueprint, ConceptStructureBlueprintFieldType
+
+CONCEPTS_WITH_STRUCTURES = (
+ "concepts_with_structures",
+ """domain = "structured_concepts"
+definition = "Domain with structured concepts"
+
+[concept]
+SimpleData = "Simple data concept"
+
+[concept.PersonInfo]
+definition = "Information about a person"
+
+[concept.PersonInfo.structure]
+name = "The name of the person"
+age = { type = "number", definition = "The age of the person", required = true }
+birthdate = { type = "date", definition = "The birthdate of the person", required = true }
+""",
+ PipelexBundleBlueprint(
+ domain="structured_concepts",
+ definition="Domain with structured concepts",
+ concept={
+ "SimpleData": "Simple data concept",
+ "PersonInfo": ConceptBlueprint(
+ definition="Information about a person",
+ structure={
+ "name": "The name of the person",
+ "age": ConceptStructureBlueprint(
+ type=ConceptStructureBlueprintFieldType.NUMBER, definition="The age of the person", required=True
+ ),
+ "birthdate": ConceptStructureBlueprint(
+ type=ConceptStructureBlueprintFieldType.DATE, definition="The birthdate of the person", required=True
+ ),
+ },
+ ),
+ },
+ ),
+)
+
+CONCEPTS_WITH_NAMED_STRUCTURES = (
+ "concepts_with_named_structures",
+ """domain = "named_structures"
+definition = "Domain with concepts using named structure references"
+
+[concept]
+BasicInfo = "Basic information concept"
+
+[concept.ProductInfo]
+definition = "Information about a product"
+structure = "ProductData"
+
+[concept.OrderInfo]
+definition = "Information about an order"
+structure = "OrderData"
+""",
+ PipelexBundleBlueprint(
+ domain="named_structures",
+ definition="Domain with concepts using named structure references",
+ concept={
+ "BasicInfo": "Basic information concept",
+ "ProductInfo": ConceptBlueprint(
+ definition="Information about a product",
+ structure="ProductData",
+ ),
+ "OrderInfo": ConceptBlueprint(
+ definition="Information about an order",
+ structure="OrderData",
+ ),
+ },
+ ),
+)
+
+# Export all structured concept test cases
+STRUCTURED_CONCEPT_TEST_CASES = [
+ CONCEPTS_WITH_STRUCTURES,
+ CONCEPTS_WITH_NAMED_STRUCTURES,
+]
diff --git a/tests/unit/pipelex/core/test_data/domain/__init__.py b/tests/unit/pipelex/core/test_data/domain/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/unit/pipelex/core/test_data/domain/simple_domains.py b/tests/unit/pipelex/core/test_data/domain/simple_domains.py
new file mode 100644
index 000000000..3dc8c2fd5
--- /dev/null
+++ b/tests/unit/pipelex/core/test_data/domain/simple_domains.py
@@ -0,0 +1,37 @@
+"""Simple domain test cases."""
+
+from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint
+
+SIMPLE_DOMAIN = (
+ "simple_domain",
+ """domain = "simple_test"
+definition = "A simple test domain"
+""",
+ PipelexBundleBlueprint(
+ domain="simple_test",
+ definition="A simple test domain",
+ ),
+)
+
+DOMAIN_WITH_SYSTEM_PROMPTS = (
+ "domain_with_system_prompts",
+ """domain = "system_prompt_test"
+definition = "A domain with system prompts"
+system_prompt = "You are an expert assistant"
+system_prompt_to_structure = "Generate structured output"
+prompt_template_to_structure = "Structure the following data: ..."
+""",
+ PipelexBundleBlueprint(
+ domain="system_prompt_test",
+ definition="A domain with system prompts",
+ system_prompt="You are an expert assistant",
+ system_prompt_to_structure="Generate structured output",
+ prompt_template_to_structure="Structure the following data: ...",
+ ),
+)
+
+# Export all domain test cases
+DOMAIN_TEST_CASES = [
+ SIMPLE_DOMAIN,
+ DOMAIN_WITH_SYSTEM_PROMPTS,
+]
diff --git a/tests/unit/pipelex/core/test_data/errors/__init__.py b/tests/unit/pipelex/core/test_data/errors/__init__.py
new file mode 100644
index 000000000..011b80c50
--- /dev/null
+++ b/tests/unit/pipelex/core/test_data/errors/__init__.py
@@ -0,0 +1 @@
+"""Error test cases for PipelexInterpreter tests."""
diff --git a/tests/unit/pipelex/core/test_data/errors/invalid_toml.py b/tests/unit/pipelex/core/test_data/errors/invalid_toml.py
new file mode 100644
index 000000000..26fc011a8
--- /dev/null
+++ b/tests/unit/pipelex/core/test_data/errors/invalid_toml.py
@@ -0,0 +1,30 @@
+"""Invalid TOML test cases."""
+
+from typing import List, Tuple
+
+INVALID_TOML_SYNTAX = (
+ "invalid_toml_syntax",
+ """domain = "invalid_syntax"
+definition = "Domain with invalid TOML syntax"
+
+[concept]
+InvalidConcept = "This is missing a closing quote""",
+ None, # Should raise an exception
+)
+
+MISSING_DOMAIN = (
+ "missing_domain",
+ """# Missing domain field
+definition = "Domain without domain field"
+
+[concept]
+TestConcept = "A test concept"
+""",
+ None, # Should raise an exception
+)
+
+# Export all error test cases
+ERROR_TEST_CASES: List[Tuple[str, str]] = [
+ (INVALID_TOML_SYNTAX[0], INVALID_TOML_SYNTAX[1]),
+ (MISSING_DOMAIN[0], MISSING_DOMAIN[1]),
+]
diff --git a/tests/unit/pipelex/core/test_data/pipes/__init__.py b/tests/unit/pipelex/core/test_data/pipes/__init__.py
new file mode 100644
index 000000000..44aeec3e0
--- /dev/null
+++ b/tests/unit/pipelex/core/test_data/pipes/__init__.py
@@ -0,0 +1 @@
+"""Pipe-related test data for PipelexInterpreter tests."""
diff --git a/tests/unit/pipelex/core/test_data/pipes/controllers/__init__.py b/tests/unit/pipelex/core/test_data/pipes/controllers/__init__.py
new file mode 100644
index 000000000..b9b684798
--- /dev/null
+++ b/tests/unit/pipelex/core/test_data/pipes/controllers/__init__.py
@@ -0,0 +1 @@
+"""Pipe controller test data for PipelexInterpreter tests."""
diff --git a/tests/unit/pipelex/core/test_data/pipes/controllers/batch/__init__.py b/tests/unit/pipelex/core/test_data/pipes/controllers/batch/__init__.py
new file mode 100644
index 000000000..22a23e3cb
--- /dev/null
+++ b/tests/unit/pipelex/core/test_data/pipes/controllers/batch/__init__.py
@@ -0,0 +1 @@
+"""PipeBatch test data for PipelexInterpreter tests."""
diff --git a/tests/unit/pipelex/core/test_data/pipes/controllers/batch/pipe_batch.py b/tests/unit/pipelex/core/test_data/pipes/controllers/batch/pipe_batch.py
new file mode 100644
index 000000000..b493cb218
--- /dev/null
+++ b/tests/unit/pipelex/core/test_data/pipes/controllers/batch/pipe_batch.py
@@ -0,0 +1,38 @@
+"""PipeBatch test cases."""
+
+from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint
+from pipelex.pipe_controllers.pipe_batch_factory import PipeBatchBlueprint
+
+PIPE_BATCH = (
+ "pipe_batch",
+ """domain = "test_pipes"
+definition = "Domain with batch pipe"
+
+[pipe.batch_process]
+type = "PipeBatch"
+definition = "Process items in batch"
+output = "ProcessedData"
+branch_pipe_code = "process_item"
+input_list_name = "items"
+input_item_name = "current_item"
+""",
+ PipelexBundleBlueprint(
+ domain="test_pipes",
+ definition="Domain with batch pipe",
+ pipe={
+ "batch_process": PipeBatchBlueprint(
+ type="PipeBatch",
+ definition="Process items in batch",
+ output="ProcessedData",
+ branch_pipe_code="process_item",
+ input_list_name="items",
+ input_item_name="current_item",
+ ),
+ },
+ ),
+)
+
+# Export all PipeBatch test cases
+PIPE_BATCH_TEST_CASES = [
+ PIPE_BATCH,
+]
diff --git a/tests/unit/pipelex/core/test_data/pipes/controllers/condition/__init__.py b/tests/unit/pipelex/core/test_data/pipes/controllers/condition/__init__.py
new file mode 100644
index 000000000..55ed0effd
--- /dev/null
+++ b/tests/unit/pipelex/core/test_data/pipes/controllers/condition/__init__.py
@@ -0,0 +1 @@
+"""PipeCondition test data for PipelexInterpreter tests."""
diff --git a/tests/unit/pipelex/core/test_data/pipes/controllers/condition/pipe_condition.py b/tests/unit/pipelex/core/test_data/pipes/controllers/condition/pipe_condition.py
new file mode 100644
index 000000000..973bb8f9e
--- /dev/null
+++ b/tests/unit/pipelex/core/test_data/pipes/controllers/condition/pipe_condition.py
@@ -0,0 +1,39 @@
+"""PipeCondition test cases."""
+
+from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint
+from pipelex.pipe_controllers.pipe_condition_factory import PipeConditionBlueprint
+
+PIPE_CONDITION = (
+ "pipe_condition",
+ """domain = "test_pipes"
+definition = "Domain with conditional pipe"
+
+[pipe.conditional_process]
+type = "PipeCondition"
+definition = "Process based on condition"
+output = "ProcessedData"
+pipe_map = { small = "process_small", large = "process_large" }
+expression = "input_data.category"
+""",
+ PipelexBundleBlueprint(
+ domain="test_pipes",
+ definition="Domain with conditional pipe",
+ pipe={
+ "conditional_process": PipeConditionBlueprint(
+ type="PipeCondition",
+ definition="Process based on condition",
+ output="ProcessedData",
+ expression="input_data.category",
+ pipe_map={
+ "small": "process_small",
+ "large": "process_large",
+ },
+ ),
+ },
+ ),
+)
+
+# Export all PipeCondition test cases
+PIPE_CONDITION_TEST_CASES = [
+ PIPE_CONDITION,
+]
diff --git a/tests/unit/pipelex/core/test_data/pipes/controllers/parallel/__init__.py b/tests/unit/pipelex/core/test_data/pipes/controllers/parallel/__init__.py
new file mode 100644
index 000000000..e15534b6c
--- /dev/null
+++ b/tests/unit/pipelex/core/test_data/pipes/controllers/parallel/__init__.py
@@ -0,0 +1 @@
+"""PipeParallel test data for PipelexInterpreter tests."""
diff --git a/tests/unit/pipelex/core/test_data/pipes/controllers/parallel/pipe_parallel.py b/tests/unit/pipelex/core/test_data/pipes/controllers/parallel/pipe_parallel.py
new file mode 100644
index 000000000..bbce89315
--- /dev/null
+++ b/tests/unit/pipelex/core/test_data/pipes/controllers/parallel/pipe_parallel.py
@@ -0,0 +1,41 @@
+"""PipeParallel test cases."""
+
+from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint
+from pipelex.pipe_controllers.pipe_parallel_factory import PipeParallelBlueprint
+from pipelex.pipe_controllers.sub_pipe_factory import SubPipeBlueprint
+
+PIPE_PARALLEL = (
+ "pipe_parallel",
+ """domain = "test_pipes"
+definition = "Domain with parallel pipe"
+
+[pipe.parallel_process]
+type = "PipeParallel"
+definition = "Process data in parallel"
+output = "ProcessedData"
+parallels = [
+ { pipe = "process_a", result = "result_a" },
+ { pipe = "process_b", result = "result_b" },
+]
+""",
+ PipelexBundleBlueprint(
+ domain="test_pipes",
+ definition="Domain with parallel pipe",
+ pipe={
+ "parallel_process": PipeParallelBlueprint(
+ type="PipeParallel",
+ definition="Process data in parallel",
+ output="ProcessedData",
+ parallels=[
+ SubPipeBlueprint(pipe="process_a", result="result_a"),
+ SubPipeBlueprint(pipe="process_b", result="result_b"),
+ ],
+ ),
+ },
+ ),
+)
+
+# Export all PipeParallel test cases
+PIPE_PARALLEL_TEST_CASES = [
+ PIPE_PARALLEL,
+]
diff --git a/tests/unit/pipelex/core/test_data/pipes/controllers/sequence/__init__.py b/tests/unit/pipelex/core/test_data/pipes/controllers/sequence/__init__.py
new file mode 100644
index 000000000..c0ec60837
--- /dev/null
+++ b/tests/unit/pipelex/core/test_data/pipes/controllers/sequence/__init__.py
@@ -0,0 +1 @@
+"""PipeSequence test data for PipelexInterpreter tests."""
diff --git a/tests/unit/pipelex/core/test_data/pipes/controllers/sequence/pipe_sequence.py b/tests/unit/pipelex/core/test_data/pipes/controllers/sequence/pipe_sequence.py
new file mode 100644
index 000000000..4bb4d95f8
--- /dev/null
+++ b/tests/unit/pipelex/core/test_data/pipes/controllers/sequence/pipe_sequence.py
@@ -0,0 +1,41 @@
+"""PipeSequence test cases."""
+
+from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint
+from pipelex.pipe_controllers.pipe_sequence_factory import PipeSequenceBlueprint
+from pipelex.pipe_controllers.sub_pipe_factory import SubPipeBlueprint
+
+PIPE_SEQUENCE = (
+ "pipe_sequence",
+ """domain = "test_pipes"
+definition = "Domain with sequence pipe"
+
+[pipe.process_sequence]
+type = "PipeSequence"
+definition = "Process data in sequence"
+output = "ProcessedData"
+steps = [
+ { pipe = "step1", result = "intermediate1" },
+ { pipe = "step2", result = "final_result" },
+]
+""",
+ PipelexBundleBlueprint(
+ domain="test_pipes",
+ definition="Domain with sequence pipe",
+ pipe={
+ "process_sequence": PipeSequenceBlueprint(
+ type="PipeSequence",
+ definition="Process data in sequence",
+ output="ProcessedData",
+ steps=[
+ SubPipeBlueprint(pipe="step1", result="intermediate1"),
+ SubPipeBlueprint(pipe="step2", result="final_result"),
+ ],
+ ),
+ },
+ ),
+)
+
+# Export all PipeSequence test cases
+PIPE_SEQUENCE_TEST_CASES = [
+ PIPE_SEQUENCE,
+]
diff --git a/tests/unit/pipelex/core/test_data/pipes/operators/__init__.py b/tests/unit/pipelex/core/test_data/pipes/operators/__init__.py
new file mode 100644
index 000000000..d9191620f
--- /dev/null
+++ b/tests/unit/pipelex/core/test_data/pipes/operators/__init__.py
@@ -0,0 +1 @@
+"""Pipe operator test data for PipelexInterpreter tests."""
diff --git a/tests/unit/pipelex/core/test_data/pipes/operators/func/__init__.py b/tests/unit/pipelex/core/test_data/pipes/operators/func/__init__.py
new file mode 100644
index 000000000..dac31fc83
--- /dev/null
+++ b/tests/unit/pipelex/core/test_data/pipes/operators/func/__init__.py
@@ -0,0 +1 @@
+"""PipeFunc test data for PipelexInterpreter tests."""
diff --git a/tests/unit/pipelex/core/test_data/pipes/operators/func/pipe_func.py b/tests/unit/pipelex/core/test_data/pipes/operators/func/pipe_func.py
new file mode 100644
index 000000000..cd395b0eb
--- /dev/null
+++ b/tests/unit/pipelex/core/test_data/pipes/operators/func/pipe_func.py
@@ -0,0 +1,34 @@
+"""PipeFunc test cases."""
+
+from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint
+from pipelex.pipe_operators.pipe_func_factory import PipeFuncBlueprint
+
+PIPE_FUNC = (
+ "pipe_func",
+ """domain = "test_pipes"
+definition = "Domain with function pipe"
+
+[pipe.process_data]
+type = "PipeFunc"
+definition = "Process data with function"
+output = "ProcessedData"
+function_name = "process_data_function"
+""",
+ PipelexBundleBlueprint(
+ domain="test_pipes",
+ definition="Domain with function pipe",
+ pipe={
+ "process_data": PipeFuncBlueprint(
+ type="PipeFunc",
+ definition="Process data with function",
+ output="ProcessedData",
+ function_name="process_data_function",
+ ),
+ },
+ ),
+)
+
+# Export all PipeFunc test cases
+PIPE_FUNC_TEST_CASES = [
+ PIPE_FUNC,
+]
diff --git a/tests/unit/pipelex/core/test_data/pipes/operators/img_gen/__init__.py b/tests/unit/pipelex/core/test_data/pipes/operators/img_gen/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/unit/pipelex/core/test_data/pipes/operators/img_gen/pipe_img_gen.py b/tests/unit/pipelex/core/test_data/pipes/operators/img_gen/pipe_img_gen.py
new file mode 100644
index 000000000..f9a1e2d6b
--- /dev/null
+++ b/tests/unit/pipelex/core/test_data/pipes/operators/img_gen/pipe_img_gen.py
@@ -0,0 +1,34 @@
+"""PipeImgGen test cases."""
+
+from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint
+from pipelex.pipe_operators.pipe_img_gen_factory import PipeImgGenBlueprint
+
+PIPE_IMG_GEN = (
+ "pipe_img_gen",
+ """domain = "test_pipes"
+definition = "Domain with image generation pipe"
+
+[pipe.generate_image]
+type = "PipeImgGen"
+definition = "Generate an image from a prompt"
+output = "Image"
+img_gen_prompt = "A beautiful landscape"
+""",
+ PipelexBundleBlueprint(
+ domain="test_pipes",
+ definition="Domain with image generation pipe",
+ pipe={
+ "generate_image": PipeImgGenBlueprint(
+ type="PipeImgGen",
+ definition="Generate an image from a prompt",
+ output="Image",
+ img_gen_prompt="A beautiful landscape",
+ ),
+ },
+ ),
+)
+
+# Export all PipeImgGen test cases
+PIPE_IMG_GEN_TEST_CASES = [
+ PIPE_IMG_GEN,
+]
diff --git a/tests/unit/pipelex/core/test_data/pipes/operators/jinja2/__init__.py b/tests/unit/pipelex/core/test_data/pipes/operators/jinja2/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/unit/pipelex/core/test_data/pipes/operators/jinja2/pipe_jinja2.py b/tests/unit/pipelex/core/test_data/pipes/operators/jinja2/pipe_jinja2.py
new file mode 100644
index 000000000..af5642070
--- /dev/null
+++ b/tests/unit/pipelex/core/test_data/pipes/operators/jinja2/pipe_jinja2.py
@@ -0,0 +1,34 @@
+"""PipeJinja2 test cases."""
+
+from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint
+from pipelex.pipe_operators.pipe_jinja2_factory import PipeJinja2Blueprint
+
+PIPE_JINJA2 = (
+ "pipe_jinja2",
+ """domain = "test_pipes"
+definition = "Domain with template processing pipe"
+
+[pipe.process_template]
+type = "PipeJinja2"
+definition = "Process a Jinja2 template"
+output = "Text"
+jinja2 = "Hello {{ name }}!"
+""",
+ PipelexBundleBlueprint(
+ domain="test_pipes",
+ definition="Domain with template processing pipe",
+ pipe={
+ "process_template": PipeJinja2Blueprint(
+ type="PipeJinja2",
+ definition="Process a Jinja2 template",
+ output="Text",
+ jinja2="Hello {{ name }}!",
+ ),
+ },
+ ),
+)
+
+# Export all PipeJinja2 test cases
+PIPE_JINJA2_TEST_CASES = [
+ PIPE_JINJA2,
+]
diff --git a/tests/unit/pipelex/core/test_data/pipes/operators/llm/__init__.py b/tests/unit/pipelex/core/test_data/pipes/operators/llm/__init__.py
new file mode 100644
index 000000000..a91704a7e
--- /dev/null
+++ b/tests/unit/pipelex/core/test_data/pipes/operators/llm/__init__.py
@@ -0,0 +1 @@
+"""PipeLLM test data for PipelexInterpreter tests."""
diff --git a/tests/unit/pipelex/core/test_data/pipes/operators/llm/pipe_llm.py b/tests/unit/pipelex/core/test_data/pipes/operators/llm/pipe_llm.py
new file mode 100644
index 000000000..9daf44c54
--- /dev/null
+++ b/tests/unit/pipelex/core/test_data/pipes/operators/llm/pipe_llm.py
@@ -0,0 +1,34 @@
+"""PipeLLM test cases."""
+
+from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint
+from pipelex.pipe_operators.pipe_llm_factory import PipeLLMBlueprint
+
+PIPE_LLM = (
+ "pipe_llm",
+ """domain = "test_pipes"
+definition = "Domain with pipe definitions"
+
+[pipe.generate_text]
+type = "PipeLLM"
+definition = "Generate text using LLM"
+output = "Text"
+prompt_template = "Generate a story about a programmer"
+""",
+ PipelexBundleBlueprint(
+ domain="test_pipes",
+ definition="Domain with pipe definitions",
+ pipe={
+ "generate_text": PipeLLMBlueprint(
+ type="PipeLLM",
+ definition="Generate text using LLM",
+ output="Text",
+ prompt_template="Generate a story about a programmer",
+ ),
+ },
+ ),
+)
+
+# Export all PipeLLM test cases
+PIPE_LLM_TEST_CASES = [
+ PIPE_LLM,
+]
diff --git a/tests/unit/pipelex/core/test_data/pipes/operators/ocr/__init__.py b/tests/unit/pipelex/core/test_data/pipes/operators/ocr/__init__.py
new file mode 100644
index 000000000..a5ef8b7a3
--- /dev/null
+++ b/tests/unit/pipelex/core/test_data/pipes/operators/ocr/__init__.py
@@ -0,0 +1 @@
+"""PipeOcr test data for PipelexInterpreter tests."""
diff --git a/tests/unit/pipelex/core/test_data/pipes/operators/ocr/pipe_ocr.py b/tests/unit/pipelex/core/test_data/pipes/operators/ocr/pipe_ocr.py
new file mode 100644
index 000000000..c654f42f3
--- /dev/null
+++ b/tests/unit/pipelex/core/test_data/pipes/operators/ocr/pipe_ocr.py
@@ -0,0 +1,32 @@
+"""PipeOcr test cases."""
+
+from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint
+from pipelex.pipe_operators.pipe_ocr_factory import PipeOcrBlueprint
+
+PIPE_OCR = (
+ "pipe_ocr",
+ """domain = "test_pipes"
+definition = "Domain with OCR pipe"
+
+[pipe.extract_text]
+type = "PipeOcr"
+definition = "Extract text from document"
+output = "Page"
+""",
+ PipelexBundleBlueprint(
+ domain="test_pipes",
+ definition="Domain with OCR pipe",
+ pipe={
+ "extract_text": PipeOcrBlueprint(
+ type="PipeOcr",
+ definition="Extract text from document",
+ output="Page",
+ ),
+ },
+ ),
+)
+
+# Export all PipeOcr test cases
+PIPE_OCR_TEST_CASES = [
+ PIPE_OCR,
+]
diff --git a/tests/unit/pipelex/core/test_interpreter.py b/tests/unit/pipelex/core/test_interpreter.py
new file mode 100644
index 000000000..6a6861d1e
--- /dev/null
+++ b/tests/unit/pipelex/core/test_interpreter.py
@@ -0,0 +1,42 @@
+from pathlib import Path
+
+import pytest
+
+from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint
+from pipelex.core.interpreter import PipelexInterpreter
+from tests.unit.pipelex.core.test_data import InterpreterTestCases
+
+
+class TestPipelexInterpreter:
+ """Test the PipelexInterpreter class with various TOML configurations."""
+
+ def test_init_with_both_file_path_and_content(self, tmp_path: Path):
+ """Test initialization with both file_path and file_content."""
+ test_file = tmp_path / "test.toml"
+ test_file.write_text("domain = 'test'\n[concept]\n")
+ content = "domain = 'other'\n[concept]\n"
+
+ converter = PipelexInterpreter(file_path=test_file, file_content=content)
+ assert converter.file_path == test_file
+ assert converter.file_content == content
+
+ @pytest.mark.parametrize("test_name,toml_content,expected_blueprint", InterpreterTestCases.VALID_TEST_CASES)
+ def test_make_pipelex_bundle_blueprint(self, test_name: str, toml_content: str, expected_blueprint: PipelexBundleBlueprint):
+ """Test making blueprint from various valid TOML content."""
+ converter = PipelexInterpreter(file_content=toml_content)
+ from pipelex import pretty_print
+
+ blueprint = converter.make_pipelex_bundle_blueprint()
+ pretty_print(blueprint, title=f"Blueprint {test_name}")
+ pretty_print(expected_blueprint, title=f"Expected blueprint {test_name}")
+ assert blueprint == expected_blueprint
+
+ @pytest.mark.parametrize("test_name,toml_content,expected_blueprint", InterpreterTestCases.VALID_TEST_CASES)
+ def test_make_toml_content(self, test_name: str, toml_content: str, expected_blueprint: PipelexBundleBlueprint):
+ """Test making blueprint from various valid TOML content."""
+ get_toml_content = PipelexInterpreter.make_toml_content(blueprint=expected_blueprint)
+ from pipelex import pretty_print
+
+ pretty_print(get_toml_content, title=f"TOML content {test_name}")
+ pretty_print(toml_content, title=f"Expected TOML content {test_name}")
+ assert get_toml_content == toml_content
diff --git a/tests/unit/pipelex/create/test_structured_output_generator.py b/tests/unit/pipelex/create/test_structured_output_generator.py
index e7838c1d5..635f88a4d 100644
--- a/tests/unit/pipelex/create/test_structured_output_generator.py
+++ b/tests/unit/pipelex/create/test_structured_output_generator.py
@@ -31,7 +31,7 @@ active = { type = "boolean", definition = "Active status", default = true }
# Check that basic structure is correct
assert "from typing import Optional" in result
- assert "from pipelex.core.stuff_content import StructuredContent" in result
+ assert "from pipelex.core.stuffs.stuff_content import StructuredContent" in result
assert "from pydantic import Field" in result
assert "class TestModel(StructuredContent):" in result
assert '"""A test model"""' in result
diff --git a/tests/unit/pipelex/tools/misc/test_toml_utils.py b/tests/unit/pipelex/tools/misc/test_toml_utils.py
index b9edca0c5..7edcdcda1 100644
--- a/tests/unit/pipelex/tools/misc/test_toml_utils.py
+++ b/tests/unit/pipelex/tools/misc/test_toml_utils.py
@@ -13,8 +13,7 @@ from pipelex.tools.misc.toml_utils import (
class TestTomlUtils:
def test_load_toml_from_path_valid_file(self, tmp_path: Path) -> None:
"""Test loading a valid TOML file without issues."""
- toml_content = """
-domain = "test_domain"
+ toml_content = """domain = "test_domain"
definition = "Test definition"
[concept]
@@ -22,10 +21,11 @@ TestConcept = "A test concept"
[pipe]
[pipe.test_pipe]
-PipeLLM = "Test pipe definition"
+type = "PipeLLM"
+definition = "Test pipe definition"
prompt_template = '''
This is a test prompt
-'''
+'''
"""
toml_file = tmp_path / "valid.toml"
toml_file.write_text(toml_content)
@@ -194,7 +194,8 @@ CategoryInput = "Input with a category field"
[pipe]
[pipe.basic_condition_by_category_2]
-PipeCondition = "Route based on category field using expression"
+type = "PipeCondition"
+definition = "Route based on category field using expression"
inputs = { input_data = "CategoryInput" }
output = "native.Text"
expression = "input_data.category"
@@ -205,7 +206,8 @@ medium = "process_medium_2"
large = "process_large_2"
[pipe.process_large_2]
-PipeLLM = "Generate random text for large items"
+type = "PipeLLM"
+definition = "Generate random text for large items"
output = "native.Text"
prompt_template = """
Output this only: "large"
@@ -218,7 +220,7 @@ Output this only: "large"
error_msg = str(exc_info.value)
assert "Trailing whitespace after triple quotes" in error_msg
- assert "Line 24" in error_msg # The line with """ followed by space
+ assert "Line 26" in error_msg # The line with """ followed by space
def test_validate_toml_file_actual_problematic_file(self) -> None:
"""Test validation on the actual problematic file from the codebase."""
diff --git a/tests/unit/pipelex/tools/typing/test_type_inspector.py b/tests/unit/pipelex/tools/typing/test_type_inspector.py
index 399c195d9..5ea82b6c8 100644
--- a/tests/unit/pipelex/tools/typing/test_type_inspector.py
+++ b/tests/unit/pipelex/tools/typing/test_type_inspector.py
@@ -4,7 +4,7 @@ from typing import List, Literal, Optional
from pydantic import BaseModel, Field, field_validator
from pytest import FixtureRequest
-from pipelex.core.stuff_content import ListContent, StructuredContent, TextContent
+from pipelex.core.stuffs.stuff_content import ListContent, StructuredContent, TextContent
from pipelex.tools.typing.type_inspector import get_type_structure
from pipelex.types import StrEnum
diff --git a/uv.lock b/uv.lock
index d7b5a32d9..6f6a5a622 100644
--- a/uv.lock
+++ b/uv.lock
@@ -1748,7 +1748,7 @@ wheels = [
[[package]]
name = "pipelex"
-version = "0.6.10"
+version = "0.7.0"
source = { editable = "." }
dependencies = [
{ name = "aiofiles" },