Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

Start Epic Library blueprint with refactor loading library using a blueprint

Louis Choquelb37aa1c460a5
diff --git a/pipelex/libraries/library_manager.py b/pipelex/libraries/library_manager.py
index 2f654e57b..12adc0d48 100644
--- a/pipelex/libraries/library_manager.py
+++ b/pipelex/libraries/library_manager.py
@@ -27,6 +27,12 @@ from pipelex.exceptions import (
)
from pipelex.libraries.library_config import LibraryConfig
from pipelex.libraries.library_manager_abstract import LibraryManagerAbstract
+from pipelex.libraries.pipeline_blueprint import (
+ ConceptBlueprintError,
+ PipeBlueprintError,
+ PipelineBlueprintValidationError,
+ PipelineLibraryBlueprint,
+)
from pipelex.tools.class_registry_utils import ClassRegistryUtils
from pipelex.tools.misc.file_utils import find_files_in_dir
from pipelex.tools.misc.json_utils import deep_update
@@ -171,145 +177,102 @@ class LibraryManager(LibraryManagerAbstract):
# First pass: load all domains
for toml_path in library_paths:
- library_dict = load_toml_from_path(path=str(toml_path))
- library_name = toml_path.stem
- domain_code = library_dict.get("domain")
- if domain_code is None:
- raise LibraryParsingError(
- f"Error loading library '{library_name}' which has no domain set at '{toml_path}'. "
- "Just write 'domain = \"my_domain\"' at the top of the file."
- )
- domain_definition = library_dict.get("definition")
- system_prompt = library_dict.get("system_prompt")
- system_prompt_to_structure = library_dict.get("system_prompt_to_structure")
- prompt_template_to_structure = library_dict.get("prompt_template_to_structure")
+ blueprint = self._load_blueprint_from_file(toml_path)
domain = Domain(
- code=domain_code,
- definition=domain_definition,
- system_prompt=system_prompt,
- system_prompt_to_structure=system_prompt_to_structure,
- prompt_template_to_structure=prompt_template_to_structure,
+ code=blueprint.domain,
+ definition=blueprint.definition,
+ system_prompt=blueprint.system_prompt,
+ system_prompt_to_structure=blueprint.system_prompt_to_structure,
+ prompt_template_to_structure=blueprint.prompt_template_to_structure,
)
self.domain_library.add_domain_details(domain=domain)
# Second pass: load all concepts
for toml_path in library_paths:
nb_concepts_before = len(self.concept_library.root)
- library_dict = load_toml_from_path(path=str(toml_path))
- library_name = toml_path.stem
+ blueprint = self._load_blueprint_from_file(toml_path)
try:
- self._load_library_dict(library_name=library_name, library_dict=library_dict, component_type=LibraryComponent.CONCEPT)
- except ConceptLibraryError as exc:
- raise LibraryError(f"Error loading concepts from library '{library_name}' at '{toml_path}': {exc}") from exc
+ self._load_concepts_from_blueprint(blueprint=blueprint, file_path=str(toml_path))
+ except ConceptBlueprintError:
+ raise # Re-raise with detailed error message
nb_concepts_loaded = len(self.concept_library.root) - nb_concepts_before
log.verbose(f"Loaded {nb_concepts_loaded} concepts from '{toml_path.name}'")
# Third pass: load all pipes
for toml_path in library_paths:
nb_pipes_before = len(self.pipe_library.root)
+ blueprint = self._load_blueprint_from_file(toml_path)
try:
- library_dict = load_toml_from_path(path=str(toml_path))
- except Exception as exc:
- log.error(f"Failed to load TOML file '{toml_path}': {exc}")
- continue
- library_name = toml_path.stem
- try:
- self._load_library_dict(library_name=library_name, library_dict=library_dict, component_type=LibraryComponent.PIPE)
- except StaticValidationError as static_validation_error:
- static_validation_error.file_path = str(toml_path)
- log.error(static_validation_error.desc())
- raise static_validation_error
- except PipeLibraryError as pipe_library_error:
- raise LibraryError(
- f"Error loading pipes from library '{library_name}' at '{toml_path}': {pipe_library_error}"
- ) from pipe_library_error
+ self._load_pipes_from_blueprint(blueprint=blueprint, file_path=str(toml_path))
+ except (PipeBlueprintError, StaticValidationError) as pipe_error:
+ if isinstance(pipe_error, StaticValidationError):
+ pipe_error.file_path = str(toml_path)
+ log.error(pipe_error.desc())
+ raise pipe_error
nb_pipes_loaded = len(self.pipe_library.root) - nb_pipes_before
log.verbose(f"Loaded {nb_pipes_loaded} pipes from '{toml_path.name}'")
- def _load_library_dict(self, library_name: str, library_dict: Dict[str, Any], component_type: LibraryComponent):
- if domain_code := library_dict.pop("domain", None):
- # domain is set at the root of the library
- self._load_library_components_from_recursive_dict(
- domain_code=domain_code,
- recursive_dict=library_dict,
- component_type=component_type,
- )
- else:
- raise LibraryParsingError(f"Library '{library_name}' has no domain set")
-
- def _load_library_components_from_recursive_dict(
- self,
- domain_code: str,
- recursive_dict: Dict[str, Any],
- component_type: LibraryComponent,
- ):
- for key, obj in recursive_dict.items():
- # root of domain
- if not isinstance(obj, dict):
- if not isinstance(obj, str):
- raise LibraryError(f"Only a dict or a string is expected at the root of domain but '{domain_code}' got type '{type(obj)}'")
- if key not in self.allowed_root_attributes:
- raise LibraryParsingError(f"Domain '{domain_code}' has an unexpected root attribute '{key}'")
- continue
-
- # definitions within the domain
- obj_dict: Dict[str, Any] = obj
- if key == component_type:
- if key == LibraryComponent.CONCEPT:
- self._load_concepts(domain_code=domain_code, obj_dict=obj_dict)
- elif key == LibraryComponent.PIPE:
- self._load_pipes(domain_code=domain_code, obj_dict=obj_dict)
+ def _load_blueprint_from_file(self, toml_path: Path) -> PipelineLibraryBlueprint:
+ """Load and validate a pipeline blueprint from a TOML file."""
+ try:
+ toml_data = load_toml_from_path(path=str(toml_path))
+ blueprint = PipelineLibraryBlueprint.model_validate(toml_data)
+ return blueprint
+ except ValidationError as exc:
+ error_msg = format_pydantic_validation_error(exc)
+ raise PipelineBlueprintValidationError(
+ file_path=str(toml_path),
+ validation_error_msg=error_msg,
+ ) from exc
+ except Exception as exc:
+ raise LibraryError(f"Failed to load TOML file '{toml_path}': {exc}") from exc
+
+ def _load_concepts_from_blueprint(self, blueprint: PipelineLibraryBlueprint, file_path: str):
+ """Load concepts from a validated blueprint."""
+ for concept_name, concept_data in blueprint.concept.items():
+ try:
+ if isinstance(concept_data, str):
+ # Simple string definition
+ concept_from_def = ConceptFactory.make_concept_from_definition_str(
+ domain_code=blueprint.domain,
+ concept_str=concept_name,
+ definition=concept_data,
+ )
+ self.concept_library.add_new_concept(concept=concept_from_def)
else:
- continue
- elif key not in [LibraryComponent.CONCEPT, LibraryComponent.PIPE]:
- # Not a concept but a subdomain
- self._load_library_components_from_recursive_dict(domain_code=domain_code, recursive_dict=obj_dict, component_type=component_type)
- else:
- # Skip keys that don't match our criteria
- continue
-
- def _load_concepts(self, domain_code: str, obj_dict: Dict[str, Any]):
- for concept_str, concept_obj in obj_dict.items():
- if isinstance(concept_obj, str):
- # we only have a definition
- definition = concept_obj
- concept_from_def = ConceptFactory.make_concept_from_definition_str(
- domain_code=domain_code,
- concept_str=concept_str,
- definition=definition,
- )
- self.concept_library.add_new_concept(concept=concept_from_def)
- elif isinstance(concept_obj, dict):
- # blueprint dict definition
- concept_obj_dict: Dict[str, Any] = concept_obj
- try:
+ # Complex dictionary blueprint - guaranteed by blueprint schema
concept_from_dict = ConceptFactory.make_from_details_dict(
- domain_code=domain_code, code=concept_str, details_dict=concept_obj_dict
+ domain_code=blueprint.domain,
+ code=concept_name,
+ details_dict=concept_data,
)
- except ValidationError as exc:
- error_msg = format_pydantic_validation_error(exc)
- raise ConceptLibraryError(f"Error loading concept '{concept_str}' because of: {error_msg}") from exc
- self.concept_library.add_new_concept(concept=concept_from_dict)
- else:
- raise ConceptLibraryError(f"Unexpected type for concept_code '{concept_str}' in domain '{domain_code}': {type(concept_obj)}")
-
- def _load_pipes(self, domain_code: str, obj_dict: Dict[str, Any]):
- for pipe_code, pipe_obj in obj_dict.items():
- if isinstance(pipe_obj, str):
- # TODO: handle one-liner
- pass
- elif isinstance(pipe_obj, dict):
- pipe_obj_dict: Dict[str, Any] = pipe_obj.copy()
- try:
- pipe = LibraryManager.make_pipe_from_details_dict(
- domain_code=domain_code,
- pipe_code=pipe_code,
- details_dict=pipe_obj_dict,
- )
- except ValidationError as exc:
- error_msg = format_pydantic_validation_error(exc)
- raise PipeLibraryError(f"Error loading pipe '{pipe_code}' because of: {error_msg}") from exc
+ self.concept_library.add_new_concept(concept=concept_from_dict)
+ except ValidationError as exc:
+ error_msg = format_pydantic_validation_error(exc)
+ raise ConceptBlueprintError(
+ file_path=file_path,
+ concept_name=concept_name,
+ error_msg=error_msg,
+ ) from exc
+
+ def _load_pipes_from_blueprint(self, blueprint: PipelineLibraryBlueprint, file_path: str):
+ """Load pipes from a validated blueprint."""
+ for pipe_name, pipe_data in blueprint.pipe.items():
+ try:
+ # pipe_data is guaranteed to be Dict[str, Any] by the blueprint schema
+ pipe = LibraryManager.make_pipe_from_details_dict(
+ domain_code=blueprint.domain,
+ pipe_code=pipe_name,
+ details_dict=pipe_data.copy(),
+ )
self.pipe_library.add_new_pipe(pipe=pipe)
+ except ValidationError as exc:
+ error_msg = format_pydantic_validation_error(exc)
+ raise PipeBlueprintError(
+ file_path=file_path,
+ pipe_name=pipe_name,
+ error_msg=error_msg,
+ ) from exc
def validate_libraries(self):
log.debug("LibraryManager validating libraries")
diff --git a/pipelex/libraries/pipeline_blueprint.py b/pipelex/libraries/pipeline_blueprint.py
new file mode 100644
index 000000000..186aba66a
--- /dev/null
+++ b/pipelex/libraries/pipeline_blueprint.py
@@ -0,0 +1,92 @@
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Union
+
+from pydantic import BaseModel, Field
+
+from pipelex.exceptions import LibraryError
+from pipelex.tools.misc.toml_utils import dict_to_toml_string, save_toml_to_path
+
+
+class PipelineBlueprintValidationError(LibraryError):
+ """Error raised when a pipeline blueprint fails validation."""
+
+ def __init__(self, file_path: str, validation_error_msg: str):
+ self.file_path = file_path
+ self.validation_error_msg = validation_error_msg
+ super().__init__(f"Pipeline blueprint validation failed for '{file_path}': {validation_error_msg}")
+
+
+class ConceptBlueprintError(LibraryError):
+ """Error raised when loading concept blueprints."""
+
+ def __init__(self, file_path: str, concept_name: str, error_msg: str):
+ self.file_path = file_path
+ self.concept_name = concept_name
+ super().__init__(f"Error loading concept '{concept_name}' from '{file_path}': {error_msg}")
+
+
+class PipeBlueprintError(LibraryError):
+ """Error raised when loading pipe blueprints."""
+
+ def __init__(self, file_path: str, pipe_name: str, error_msg: str):
+ self.file_path = file_path
+ self.pipe_name = pipe_name
+ super().__init__(f"Error loading pipe '{pipe_name}' from '{file_path}': {error_msg}")
+
+
+class PipeStepBlueprint(BaseModel):
+ """Blueprint of a step in a pipe sequence."""
+
+ pipe: str
+ result: str
+
+
+class PipelineLibraryBlueprint(BaseModel):
+ """Complete blueprint of a pipeline library TOML file."""
+
+ # Domain information (required)
+ domain: str
+ definition: Optional[str] = None
+ system_prompt: Optional[str] = None
+ system_prompt_to_structure: Optional[str] = None
+ prompt_template_to_structure: Optional[str] = None
+
+ # Concepts section - concept_name -> definition (string) or blueprint (dict)
+ concept: Dict[str, Union[str, Dict[str, Any]]] = Field(default_factory=dict)
+
+ # Pipes section - pipe_name -> blueprint dict
+ pipe: Dict[str, Dict[str, Any]] = Field(default_factory=dict)
+
+ def to_toml_dict(self) -> Dict[str, Any]:
+ """Convert blueprint back to TOML-compatible dictionary structure."""
+ result: Dict[str, Any] = {"domain": self.domain}
+
+ # Add optional domain-level fields
+ if self.definition:
+ result["definition"] = self.definition
+ if self.system_prompt:
+ result["system_prompt"] = self.system_prompt
+ if self.system_prompt_to_structure:
+ result["system_prompt_to_structure"] = self.system_prompt_to_structure
+ if self.prompt_template_to_structure:
+ result["prompt_template_to_structure"] = self.prompt_template_to_structure
+
+ # Add concepts section if not empty
+ if self.concept:
+ result["concept"] = self.concept
+
+ # Add pipes section if not empty
+ if self.pipe:
+ result["pipe"] = self.pipe
+
+ return result
+
+ def to_toml_string(self) -> str:
+ """Convert blueprint to TOML string format."""
+ return dict_to_toml_string(self.to_toml_dict())
+
+ def save_to_file(self, file_path: Union[str, Path]) -> None:
+ """Save blueprint to a TOML file."""
+ path = Path(file_path)
+ path.parent.mkdir(parents=True, exist_ok=True)
+ save_toml_to_path(data=self.to_toml_dict(), path=str(path))
diff --git a/pipelex/tools/misc/toml_utils.py b/pipelex/tools/misc/toml_utils.py
index 584350f61..2316100cf 100644
--- a/pipelex/tools/misc/toml_utils.py
+++ b/pipelex/tools/misc/toml_utils.py
@@ -120,3 +120,29 @@ def failable_load_toml_from_path(path: str) -> Optional[Dict[str, Any]]:
except toml.TomlDecodeError as exc:
print(f"Failed to parse TOML file '{path}': {exc}")
return None
+
+
+def save_toml_to_path(data: Dict[str, Any], path: str) -> None:
+ """Save dictionary as TOML to file path.
+
+ Args:
+ data: Dictionary to save as TOML
+ path: Path where to save the TOML file
+ """
+ with open(path, "w", encoding="utf-8") as file:
+ toml_content = toml.dumps(data)
+ cleaned_content = _clean_trailing_whitespace(toml_content)
+ file.write(cleaned_content)
+
+
+def dict_to_toml_string(data: Dict[str, Any]) -> str:
+ """Convert dictionary to TOML string format.
+
+ Args:
+ data: Dictionary to convert to TOML
+
+ Returns:
+ TOML formatted string
+ """
+ toml_content = toml.dumps(data)
+ return _clean_trailing_whitespace(toml_content)