Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

Control pipeline TOML

Louis Choquel2ee601af4576
diff --git a/pipelex/create/build_blueprint.py b/pipelex/create/build_blueprint.py
index a3d427ba6..d73b72b1e 100644
--- a/pipelex/create/build_blueprint.py
+++ b/pipelex/create/build_blueprint.py
@@ -9,6 +9,7 @@ from pydantic import ValidationError
from pipelex import pretty_print
from pipelex.create.helpers import get_pipeline_creation_rules
+from pipelex.create.pipeline_toml import save_pipeline_blueprint_toml_to_path, save_toml_to_path
from pipelex.exceptions import PipeDefinitionError, PipelexCLIError
from pipelex.hub import get_library_manager
from pipelex.libraries.library_manager import LibraryManager
@@ -16,7 +17,6 @@ from pipelex.libraries.pipeline_blueprint import PipelineLibraryBlueprint
from pipelex.pipe_works.pipe_dry import dry_run_pipe_codes
from pipelex.pipeline.execute import execute_pipeline
from pipelex.tools.misc.json_utils import save_as_json_to_path
-from pipelex.tools.misc.toml_utils import save_toml_to_path
from pipelex.tools.typing.pydantic_utils import format_pydantic_validation_error
@@ -39,10 +39,11 @@ async def do_build_blueprint(
# Save or display result
output_path = output_path or "pipelex/libraries/pipelines/temp/generated_blueprint.toml"
- blueprint_dict = blueprint.smart_dump()
- save_toml_to_path(data=blueprint_dict, path=output_path)
+ # blueprint_dict = blueprint.smart_dump()
+ # save_toml_to_path(data=blueprint_dict, path=output_path)
+ save_pipeline_blueprint_toml_to_path(blueprint=blueprint, path=output_path)
json_path = output_path.replace(Path(output_path).suffix, ".json")
- save_as_json_to_path(object_to_save=blueprint_dict, path=json_path)
+ save_as_json_to_path(object_to_save=blueprint, path=json_path)
typer.echo(f"✅ Blueprint saved to '{output_path}'")
if validate:
diff --git a/pipelex/create/draft_pipeline.py b/pipelex/create/draft_pipeline.py
index 8ecfd7893..bccf24e74 100644
--- a/pipelex/create/draft_pipeline.py
+++ b/pipelex/create/draft_pipeline.py
@@ -6,6 +6,7 @@ import typer
from pipelex import pretty_print
from pipelex.create.helpers import get_pipeline_creation_rules
+from pipelex.create.pipeline_toml import dict_to_toml, save_toml_to_path
from pipelex.libraries.pipelines.meta.pipeline_draft import PipelineDraft
from pipelex.pipeline.execute import execute_pipeline
from pipelex.tools.misc.file_utils import save_text_to_path
@@ -48,9 +49,10 @@ async def do_draft_pipeline(
)
draft = pipe_output.main_stuff_as(content_type=PipelineDraft)
pretty_print(draft, title="Pipeline Draft")
- pretty_print(draft.to_toml_dict(), title="Pipeline Draft (TOML)")
+ draft_dict = draft.smart_dump()
+ pretty_print(dict_to_toml(data=draft_dict), title="Pipeline Draft (TOML)")
# Save or display result
output_path = output_path or "pipelex/libraries/pipelines/temp/generated_blueprint.toml"
- draft.save_to_file(output_path)
+ save_toml_to_path(data=draft_dict, path=output_path)
typer.echo(f"✅ Blueprint saved to '{output_path}'")
diff --git a/pipelex/create/pipeline_toml.py b/pipelex/create/pipeline_toml.py
new file mode 100644
index 000000000..232746c54
--- /dev/null
+++ b/pipelex/create/pipeline_toml.py
@@ -0,0 +1,94 @@
+from __future__ import annotations
+
+from typing import Any, Dict, List, Mapping, Optional, cast
+
+import tomlkit
+from tomlkit import array, document, inline_table, table
+from tomlkit import string as tk_string
+
+from pipelex.libraries.pipeline_blueprint import PipelineLibraryBlueprint
+from pipelex.tools.misc.file_utils import save_text_to_path
+from pipelex.tools.misc.json_utils import remove_none_values_from_dict
+from pipelex.tools.misc.toml_utils import clean_trailing_whitespace
+
+
+def make_toml_string(text: str, prefer_literal: bool = False, force_multiline: bool = False):
+ """Return a tomlkit String configured for triple quotes when appropriate."""
+ needs_multiline = force_multiline or ("\n" in text)
+ # Prefer literal '''...''' when you do not want escapes, as long as it
+ # does not contain a triple single-quote sequence (which would be invalid).
+ use_literal = prefer_literal and ("'''" not in text)
+ return tk_string(text, multiline=needs_multiline, literal=use_literal)
+
+
+def _convert_to_inline(value: Any):
+ """Recursively convert Python values; dicts -> inline tables; lists kept as arrays."""
+ if isinstance(value, Mapping):
+ value = cast(Mapping[str, Any], value)
+ inline_table_obj = inline_table()
+ for key, value_item in value.items():
+ inline_table_obj[key] = _convert_to_inline(value_item)
+ return inline_table_obj
+
+ if isinstance(value, list):
+ value = cast(List[Any], value)
+ array_obj = array()
+ array_obj.multiline(True) # set to False for single-line arrays
+ for element in value:
+ if isinstance(element, Mapping):
+ element = cast(Mapping[str, Any], element)
+ inline_element = inline_table()
+ for inner_key, inner_value in element.items():
+ inline_element[inner_key] = _convert_to_inline(inner_value)
+ array_obj.append(inline_element) # pyright: ignore[reportUnknownMemberType]
+ else:
+ array_obj.append(_convert_to_inline(element)) # pyright: ignore[reportUnknownMemberType]
+ return array_obj
+
+ if isinstance(value, str):
+ # Emit """...""" when there are newlines; flip prefer_literal=True to get '''...'''.
+ return make_toml_string(value, prefer_literal=False)
+
+ return value
+
+
+def dict_to_toml(data: Mapping[str, Any]) -> str:
+ """Top-level keys become tables; nested dicts become inline tables."""
+ data = remove_none_values_from_dict(data=data)
+ document_root = document()
+ for section_key, section_value in data.items():
+ if isinstance(section_value, Mapping):
+ section_value = cast(Mapping[str, Any], section_value)
+ table_obj = table()
+ for field_key, field_value in section_value.items():
+ table_obj.add(field_key, _convert_to_inline(field_value))
+ document_root.add(section_key, table_obj) # `[section_key]` section
+ else:
+ document_root.add(section_key, _convert_to_inline(section_value))
+ dumped_content = tomlkit.dumps(document_root) # pyright: ignore[reportUnknownMemberType]
+ return dumped_content
+
+
+def pipeline_blueprint_to_toml(blueprint: PipelineLibraryBlueprint) -> str:
+ blueprint_dict = blueprint.smart_dump()
+ return dict_to_toml(data=blueprint_dict)
+
+
+def save_toml_to_path(data: Dict[str, Any], path: str) -> None:
+ """Save dictionary as TOML to file path.
+
+ Args:
+ data: Dictionary to save as TOML
+ path: Path where to save the TOML file
+ """
+ # data_cleaned = remove_none_values_from_dict(data=data)
+ data_cleaned = data
+ with open(path, "w", encoding="utf-8") as file:
+ toml_content: str = dict_to_toml(data=data_cleaned)
+ cleaned_content = clean_trailing_whitespace(toml_content)
+ file.write(cleaned_content)
+
+
+def save_pipeline_blueprint_toml_to_path(blueprint: PipelineLibraryBlueprint, path: str) -> None:
+ pipeline_blueprint_toml = pipeline_blueprint_to_toml(blueprint=blueprint)
+ save_text_to_path(text=pipeline_blueprint_toml, path=path)
diff --git a/pipelex/libraries/pipelines/meta/pipeline_draft.py b/pipelex/libraries/pipelines/meta/pipeline_draft.py
index a1769dfe0..26e5ab970 100644
--- a/pipelex/libraries/pipelines/meta/pipeline_draft.py
+++ b/pipelex/libraries/pipelines/meta/pipeline_draft.py
@@ -1,10 +1,8 @@
-from pathlib import Path
-from typing import Any, Dict, Union
+from typing import Dict
from pydantic import Field
from pipelex.core.stuff_content import StructuredContent
-from pipelex.tools.misc.toml_utils import dict_to_toml_string, save_toml_to_path
class PipeDraft(StructuredContent):
@@ -27,31 +25,3 @@ class PipelineDraft(StructuredContent):
# Pipes section - pipe_name -> blueprint dict
pipe: Dict[str, PipeDraft] = Field(default_factory=dict)
-
- def to_toml_dict(self) -> Dict[str, Any]:
- """Convert blueprint back to TOML-compatible dictionary structure."""
- result: Dict[str, Any] = {"domain": self.domain}
-
- # Add optional domain-level fields
- if self.definition:
- result["definition"] = self.definition
-
- # Add concepts section if not empty
- if self.concept:
- result["concept"] = self.concept
-
- # Add pipes section if not empty
- if self.pipe:
- result["pipe"] = self.pipe
-
- return result
-
- def to_toml_string(self) -> str:
- """Convert blueprint to TOML string format."""
- return dict_to_toml_string(self.to_toml_dict())
-
- def save_to_file(self, file_path: Union[str, Path]) -> None:
- """Save blueprint to a TOML file."""
- path = Path(file_path)
- path.parent.mkdir(parents=True, exist_ok=True)
- save_toml_to_path(data=self.to_toml_dict(), path=str(path))
diff --git a/pipelex/tools/misc/json_utils.py b/pipelex/tools/misc/json_utils.py
index 5d331f369..a1c528128 100644
--- a/pipelex/tools/misc/json_utils.py
+++ b/pipelex/tools/misc/json_utils.py
@@ -1,5 +1,5 @@
import json
-from typing import Any, Dict, List, Optional, Tuple, Union, cast
+from typing import Any, Dict, List, Mapping, Optional, Tuple, Union, cast
from kajson import kajson
from pydantic import BaseModel
@@ -227,6 +227,14 @@ def remove_none_values(json_content: JsonContent | Any) -> JsonContent | Any:
return json_content
+def remove_none_values_from_dict(data: Mapping[str, Any]) -> Dict[str, Any]:
+ processed = remove_none_values(json_content=data)
+ if not isinstance(processed, dict):
+ raise JsonTypeError("Removing None values from a dict, we expected a dict in return")
+ processed = cast(Dict[str, Any], processed) # pyright: ignore[reportUnnecessaryCast]
+ return processed
+
+
def purify_json(
data: Any,
indent: Optional[int] = None,
diff --git a/pipelex/tools/misc/toml_utils.py b/pipelex/tools/misc/toml_utils.py
index 22b060b4e..f8c486f1b 100644
--- a/pipelex/tools/misc/toml_utils.py
+++ b/pipelex/tools/misc/toml_utils.py
@@ -1,10 +1,13 @@
-from typing import Any, Dict, List, Optional
+from __future__ import annotations
+
+from typing import Any, Dict, List, Mapping, Optional, cast
import toml
import tomlkit
+from tomlkit import array, document, inline_table, table
from pipelex.tools.misc.file_utils import path_exists
-from pipelex.tools.misc.json_utils import remove_none_values
+from pipelex.tools.misc.json_utils import remove_none_values_from_dict
class TOMLValidationError(Exception):
@@ -55,7 +58,7 @@ def validate_toml_file(path: str) -> None:
_validate_toml_content(content, path)
-def _clean_trailing_whitespace(content: str) -> str:
+def clean_trailing_whitespace(content: str) -> str:
"""Clean trailing whitespace from TOML content.
This function:
@@ -100,7 +103,7 @@ def load_toml_from_path(path: str) -> Dict[str, Any]:
with open(path, "r", encoding="utf-8") as file:
content = file.read()
- cleaned_content = _clean_trailing_whitespace(content)
+ cleaned_content = clean_trailing_whitespace(content)
# If content changed, write it back
if content != cleaned_content:
@@ -122,32 +125,3 @@ def failable_load_toml_from_path(path: str) -> Optional[Dict[str, Any]]:
except toml.TomlDecodeError as exc:
print(f"Failed to parse TOML file '{path}': {exc}")
return None
-
-
-def save_toml_to_path(data: Dict[str, Any], path: str) -> None:
- """Save dictionary as TOML to file path.
-
- Args:
- data: Dictionary to save as TOML
- path: Path where to save the TOML file
- """
- data_cleaned = remove_none_values(data)
- if not isinstance(data_cleaned, dict):
- raise RuntimeError("Data must be a dictionary")
- with open(path, "w", encoding="utf-8") as file:
- toml_content: str = tomlkit.dumps(data_cleaned) # type: ignore
- cleaned_content = _clean_trailing_whitespace(toml_content)
- file.write(cleaned_content)
-
-
-def dict_to_toml_string(data: Dict[str, Any]) -> str:
- """Convert dictionary to TOML string format.
-
- Args:
- data: Dictionary to convert to TOML
-
- Returns:
- TOML formatted string
- """
- toml_content = toml.dumps(data)
- return _clean_trailing_whitespace(toml_content)