Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
Control pipeline TOML
Louis Choquel2ee601af4576
diff --git a/pipelex/create/build_blueprint.py b/pipelex/create/build_blueprint.pyindex a3d427ba6..d73b72b1e 100644--- a/pipelex/create/build_blueprint.py+++ b/pipelex/create/build_blueprint.py@@ -9,6 +9,7 @@ from pydantic import ValidationErrorfrom pipelex import pretty_printfrom pipelex.create.helpers import get_pipeline_creation_rules+from pipelex.create.pipeline_toml import save_pipeline_blueprint_toml_to_path, save_toml_to_pathfrom pipelex.exceptions import PipeDefinitionError, PipelexCLIErrorfrom pipelex.hub import get_library_managerfrom pipelex.libraries.library_manager import LibraryManager@@ -16,7 +17,6 @@ from pipelex.libraries.pipeline_blueprint import PipelineLibraryBlueprintfrom pipelex.pipe_works.pipe_dry import dry_run_pipe_codesfrom pipelex.pipeline.execute import execute_pipelinefrom pipelex.tools.misc.json_utils import save_as_json_to_path-from pipelex.tools.misc.toml_utils import save_toml_to_pathfrom pipelex.tools.typing.pydantic_utils import format_pydantic_validation_error@@ -39,10 +39,11 @@ async def do_build_blueprint(# Save or display resultoutput_path = output_path or "pipelex/libraries/pipelines/temp/generated_blueprint.toml"- blueprint_dict = blueprint.smart_dump()- save_toml_to_path(data=blueprint_dict, path=output_path)+ # blueprint_dict = blueprint.smart_dump()+ # save_toml_to_path(data=blueprint_dict, path=output_path)+ save_pipeline_blueprint_toml_to_path(blueprint=blueprint, path=output_path)json_path = output_path.replace(Path(output_path).suffix, ".json")- save_as_json_to_path(object_to_save=blueprint_dict, path=json_path)+ save_as_json_to_path(object_to_save=blueprint, path=json_path)typer.echo(f"✅ Blueprint saved to '{output_path}'")if validate:diff --git a/pipelex/create/draft_pipeline.py b/pipelex/create/draft_pipeline.pyindex 8ecfd7893..bccf24e74 100644--- a/pipelex/create/draft_pipeline.py+++ b/pipelex/create/draft_pipeline.py@@ -6,6 +6,7 @@ import typerfrom pipelex import pretty_printfrom pipelex.create.helpers import get_pipeline_creation_rules+from pipelex.create.pipeline_toml import dict_to_toml, save_toml_to_pathfrom pipelex.libraries.pipelines.meta.pipeline_draft import PipelineDraftfrom pipelex.pipeline.execute import execute_pipelinefrom pipelex.tools.misc.file_utils import save_text_to_path@@ -48,9 +49,10 @@ async def do_draft_pipeline()draft = pipe_output.main_stuff_as(content_type=PipelineDraft)pretty_print(draft, title="Pipeline Draft")- pretty_print(draft.to_toml_dict(), title="Pipeline Draft (TOML)")+ draft_dict = draft.smart_dump()+ pretty_print(dict_to_toml(data=draft_dict), title="Pipeline Draft (TOML)")# Save or display resultoutput_path = output_path or "pipelex/libraries/pipelines/temp/generated_blueprint.toml"- draft.save_to_file(output_path)+ save_toml_to_path(data=draft_dict, path=output_path)typer.echo(f"✅ Blueprint saved to '{output_path}'")diff --git a/pipelex/create/pipeline_toml.py b/pipelex/create/pipeline_toml.pynew file mode 100644index 000000000..232746c54--- /dev/null+++ b/pipelex/create/pipeline_toml.py@@ -0,0 +1,94 @@+from __future__ import annotations++from typing import Any, Dict, List, Mapping, Optional, cast++import tomlkit+from tomlkit import array, document, inline_table, table+from tomlkit import string as tk_string++from pipelex.libraries.pipeline_blueprint import PipelineLibraryBlueprint+from pipelex.tools.misc.file_utils import save_text_to_path+from pipelex.tools.misc.json_utils import remove_none_values_from_dict+from pipelex.tools.misc.toml_utils import clean_trailing_whitespace+++def make_toml_string(text: str, prefer_literal: bool = False, force_multiline: bool = False):+ """Return a tomlkit String configured for triple quotes when appropriate."""+ needs_multiline = force_multiline or ("\n" in text)+ # Prefer literal '''...''' when you do not want escapes, as long as it+ # does not contain a triple single-quote sequence (which would be invalid).+ use_literal = prefer_literal and ("'''" not in text)+ return tk_string(text, multiline=needs_multiline, literal=use_literal)+++def _convert_to_inline(value: Any):+ """Recursively convert Python values; dicts -> inline tables; lists kept as arrays."""+ if isinstance(value, Mapping):+ value = cast(Mapping[str, Any], value)+ inline_table_obj = inline_table()+ for key, value_item in value.items():+ inline_table_obj[key] = _convert_to_inline(value_item)+ return inline_table_obj++ if isinstance(value, list):+ value = cast(List[Any], value)+ array_obj = array()+ array_obj.multiline(True) # set to False for single-line arrays+ for element in value:+ if isinstance(element, Mapping):+ element = cast(Mapping[str, Any], element)+ inline_element = inline_table()+ for inner_key, inner_value in element.items():+ inline_element[inner_key] = _convert_to_inline(inner_value)+ array_obj.append(inline_element) # pyright: ignore[reportUnknownMemberType]+ else:+ array_obj.append(_convert_to_inline(element)) # pyright: ignore[reportUnknownMemberType]+ return array_obj++ if isinstance(value, str):+ # Emit """...""" when there are newlines; flip prefer_literal=True to get '''...'''.+ return make_toml_string(value, prefer_literal=False)++ return value+++def dict_to_toml(data: Mapping[str, Any]) -> str:+ """Top-level keys become tables; nested dicts become inline tables."""+ data = remove_none_values_from_dict(data=data)+ document_root = document()+ for section_key, section_value in data.items():+ if isinstance(section_value, Mapping):+ section_value = cast(Mapping[str, Any], section_value)+ table_obj = table()+ for field_key, field_value in section_value.items():+ table_obj.add(field_key, _convert_to_inline(field_value))+ document_root.add(section_key, table_obj) # `[section_key]` section+ else:+ document_root.add(section_key, _convert_to_inline(section_value))+ dumped_content = tomlkit.dumps(document_root) # pyright: ignore[reportUnknownMemberType]+ return dumped_content+++def pipeline_blueprint_to_toml(blueprint: PipelineLibraryBlueprint) -> str:+ blueprint_dict = blueprint.smart_dump()+ return dict_to_toml(data=blueprint_dict)+++def save_toml_to_path(data: Dict[str, Any], path: str) -> None:+ """Save dictionary as TOML to file path.++ Args:+ data: Dictionary to save as TOML+ path: Path where to save the TOML file+ """+ # data_cleaned = remove_none_values_from_dict(data=data)+ data_cleaned = data+ with open(path, "w", encoding="utf-8") as file:+ toml_content: str = dict_to_toml(data=data_cleaned)+ cleaned_content = clean_trailing_whitespace(toml_content)+ file.write(cleaned_content)+++def save_pipeline_blueprint_toml_to_path(blueprint: PipelineLibraryBlueprint, path: str) -> None:+ pipeline_blueprint_toml = pipeline_blueprint_to_toml(blueprint=blueprint)+ save_text_to_path(text=pipeline_blueprint_toml, path=path)diff --git a/pipelex/libraries/pipelines/meta/pipeline_draft.py b/pipelex/libraries/pipelines/meta/pipeline_draft.pyindex a1769dfe0..26e5ab970 100644--- a/pipelex/libraries/pipelines/meta/pipeline_draft.py+++ b/pipelex/libraries/pipelines/meta/pipeline_draft.py@@ -1,10 +1,8 @@-from pathlib import Path-from typing import Any, Dict, Union+from typing import Dictfrom pydantic import Fieldfrom pipelex.core.stuff_content import StructuredContent-from pipelex.tools.misc.toml_utils import dict_to_toml_string, save_toml_to_pathclass PipeDraft(StructuredContent):@@ -27,31 +25,3 @@ class PipelineDraft(StructuredContent):# Pipes section - pipe_name -> blueprint dictpipe: Dict[str, PipeDraft] = Field(default_factory=dict)-- def to_toml_dict(self) -> Dict[str, Any]:- """Convert blueprint back to TOML-compatible dictionary structure."""- result: Dict[str, Any] = {"domain": self.domain}-- # Add optional domain-level fields- if self.definition:- result["definition"] = self.definition-- # Add concepts section if not empty- if self.concept:- result["concept"] = self.concept-- # Add pipes section if not empty- if self.pipe:- result["pipe"] = self.pipe-- return result-- def to_toml_string(self) -> str:- """Convert blueprint to TOML string format."""- return dict_to_toml_string(self.to_toml_dict())-- def save_to_file(self, file_path: Union[str, Path]) -> None:- """Save blueprint to a TOML file."""- path = Path(file_path)- path.parent.mkdir(parents=True, exist_ok=True)- save_toml_to_path(data=self.to_toml_dict(), path=str(path))diff --git a/pipelex/tools/misc/json_utils.py b/pipelex/tools/misc/json_utils.pyindex 5d331f369..a1c528128 100644--- a/pipelex/tools/misc/json_utils.py+++ b/pipelex/tools/misc/json_utils.py@@ -1,5 +1,5 @@import json-from typing import Any, Dict, List, Optional, Tuple, Union, cast+from typing import Any, Dict, List, Mapping, Optional, Tuple, Union, castfrom kajson import kajsonfrom pydantic import BaseModel@@ -227,6 +227,14 @@ def remove_none_values(json_content: JsonContent | Any) -> JsonContent | Any:return json_content+def remove_none_values_from_dict(data: Mapping[str, Any]) -> Dict[str, Any]:+ processed = remove_none_values(json_content=data)+ if not isinstance(processed, dict):+ raise JsonTypeError("Removing None values from a dict, we expected a dict in return")+ processed = cast(Dict[str, Any], processed) # pyright: ignore[reportUnnecessaryCast]+ return processed++def purify_json(data: Any,indent: Optional[int] = None,diff --git a/pipelex/tools/misc/toml_utils.py b/pipelex/tools/misc/toml_utils.pyindex 22b060b4e..f8c486f1b 100644--- a/pipelex/tools/misc/toml_utils.py+++ b/pipelex/tools/misc/toml_utils.py@@ -1,10 +1,13 @@-from typing import Any, Dict, List, Optional+from __future__ import annotations++from typing import Any, Dict, List, Mapping, Optional, castimport tomlimport tomlkit+from tomlkit import array, document, inline_table, tablefrom pipelex.tools.misc.file_utils import path_exists-from pipelex.tools.misc.json_utils import remove_none_values+from pipelex.tools.misc.json_utils import remove_none_values_from_dictclass TOMLValidationError(Exception):@@ -55,7 +58,7 @@ def validate_toml_file(path: str) -> None:_validate_toml_content(content, path)-def _clean_trailing_whitespace(content: str) -> str:+def clean_trailing_whitespace(content: str) -> str:"""Clean trailing whitespace from TOML content.This function:@@ -100,7 +103,7 @@ def load_toml_from_path(path: str) -> Dict[str, Any]:with open(path, "r", encoding="utf-8") as file:content = file.read()- cleaned_content = _clean_trailing_whitespace(content)+ cleaned_content = clean_trailing_whitespace(content)# If content changed, write it backif content != cleaned_content:@@ -122,32 +125,3 @@ def failable_load_toml_from_path(path: str) -> Optional[Dict[str, Any]]:except toml.TomlDecodeError as exc:print(f"Failed to parse TOML file '{path}': {exc}")return None---def save_toml_to_path(data: Dict[str, Any], path: str) -> None:- """Save dictionary as TOML to file path.-- Args:- data: Dictionary to save as TOML- path: Path where to save the TOML file- """- data_cleaned = remove_none_values(data)- if not isinstance(data_cleaned, dict):- raise RuntimeError("Data must be a dictionary")- with open(path, "w", encoding="utf-8") as file:- toml_content: str = tomlkit.dumps(data_cleaned) # type: ignore- cleaned_content = _clean_trailing_whitespace(toml_content)- file.write(cleaned_content)---def dict_to_toml_string(data: Dict[str, Any]) -> str:- """Convert dictionary to TOML string format.-- Args:- data: Dictionary to convert to TOML-- Returns:- TOML formatted string- """- toml_content = toml.dumps(data)- return _clean_trailing_whitespace(toml_content)