Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

Fixed remove_none_values() Added dep to tomlkit for better toml writing

Louis Choquel365231fe704f
diff --git a/pipelex/create/build_blueprint.py b/pipelex/create/build_blueprint.py
index 5fd074806..a3d427ba6 100644
--- a/pipelex/create/build_blueprint.py
+++ b/pipelex/create/build_blueprint.py
@@ -2,7 +2,7 @@ from __future__ import annotations
import asyncio
from pathlib import Path
-from typing import Annotated, Any, Dict, Optional, cast
+from typing import Any, Dict, Optional, cast
import typer
from pydantic import ValidationError
@@ -13,13 +13,10 @@ from pipelex.exceptions import PipeDefinitionError, PipelexCLIError
from pipelex.hub import get_library_manager
from pipelex.libraries.library_manager import LibraryManager
from pipelex.libraries.pipeline_blueprint import PipelineLibraryBlueprint
-from pipelex.libraries.pipelines.meta.pipeline_draft import PipelineDraft
from pipelex.pipe_works.pipe_dry import dry_run_pipe_codes
-from pipelex.pipelex import Pipelex
from pipelex.pipeline.execute import execute_pipeline
-from pipelex.tools.misc.file_utils import load_text_from_path
from pipelex.tools.misc.json_utils import save_as_json_to_path
-from pipelex.tools.misc.toml_utils import dict_to_toml_string, save_toml_to_path
+from pipelex.tools.misc.toml_utils import save_toml_to_path
from pipelex.tools.typing.pydantic_utils import format_pydantic_validation_error
@@ -42,9 +39,10 @@ async def do_build_blueprint(
# Save or display result
output_path = output_path or "pipelex/libraries/pipelines/temp/generated_blueprint.toml"
- save_toml_to_path(blueprint.smart_dump(), output_path)
+ blueprint_dict = blueprint.smart_dump()
+ save_toml_to_path(data=blueprint_dict, path=output_path)
json_path = output_path.replace(Path(output_path).suffix, ".json")
- save_as_json_to_path(blueprint.smart_dump(), json_path)
+ save_as_json_to_path(object_to_save=blueprint_dict, path=json_path)
typer.echo(f"✅ Blueprint saved to '{output_path}'")
if validate:
diff --git a/pipelex/create/draft_pipeline.py b/pipelex/create/draft_pipeline.py
index 86cb78f1a..8ecfd7893 100644
--- a/pipelex/create/draft_pipeline.py
+++ b/pipelex/create/draft_pipeline.py
@@ -1,23 +1,14 @@
from __future__ import annotations
-import asyncio
-from typing import Annotated, Any, Dict, Optional, cast
+from typing import Optional
import typer
-from pydantic import ValidationError
from pipelex import pretty_print
from pipelex.create.helpers import get_pipeline_creation_rules
-from pipelex.exceptions import PipeDefinitionError, PipelexCLIError
-from pipelex.hub import get_library_manager
-from pipelex.libraries.library_manager import LibraryManager
-from pipelex.libraries.pipeline_blueprint import PipelineLibraryBlueprint
from pipelex.libraries.pipelines.meta.pipeline_draft import PipelineDraft
-from pipelex.pipe_works.pipe_dry import dry_run_pipe_codes
-from pipelex.pipelex import Pipelex
from pipelex.pipeline.execute import execute_pipeline
-from pipelex.tools.misc.file_utils import load_text_from_path, save_text_to_path
-from pipelex.tools.typing.pydantic_utils import format_pydantic_validation_error
+from pipelex.tools.misc.file_utils import save_text_to_path
async def do_draft_pipeline_text(
diff --git a/pipelex/libraries/pipeline_blueprint.py b/pipelex/libraries/pipeline_blueprint.py
index dab7a7e5b..85e3c8ea7 100644
--- a/pipelex/libraries/pipeline_blueprint.py
+++ b/pipelex/libraries/pipeline_blueprint.py
@@ -1,4 +1,3 @@
-from pathlib import Path
from typing import Any, Dict, Optional, Union
from pydantic import Field
@@ -6,7 +5,6 @@ from pydantic import Field
from pipelex.core.concept_factory import ConceptBlueprint
from pipelex.core.stuff_content import StructuredContent
from pipelex.exceptions import LibraryError
-from pipelex.tools.misc.toml_utils import dict_to_toml_string, save_toml_to_path
class PipelineBlueprintValidationError(LibraryError):
diff --git a/pipelex/tools/misc/json_utils.py b/pipelex/tools/misc/json_utils.py
index 630937d93..d7c74a499 100644
--- a/pipelex/tools/misc/json_utils.py
+++ b/pipelex/tools/misc/json_utils.py
@@ -1,5 +1,5 @@
import json
-from typing import Any, Dict, List, Optional, Tuple, Union, cast
+from typing import Any, Dict, List, Optional, Tuple, Union
from kajson import kajson
from pydantic import BaseModel
@@ -8,7 +8,7 @@ from pipelex.tools.exceptions import ToolException
from pipelex.tools.misc.file_utils import save_text_to_path
from pipelex.tools.typing.pydantic_utils import CustomBaseModel
-JsonContent = Union[Dict[Any, Any], List[Any]]
+JsonContent = Union[Dict[str, Any], List[Any]]
class ArgumentTypeError(ToolException):
@@ -182,7 +182,7 @@ def deep_update(target_dict: Dict[str, Any], updates: Dict[str, Any]):
target_dict[key] = value
-def remove_none_values(json_content: JsonContent) -> JsonContent:
+def remove_none_values(json_content: JsonContent | Any) -> JsonContent | Any:
"""
Recursively removes all None values from a JSON-compatible data structure.
@@ -214,10 +214,15 @@ def remove_none_values(json_content: JsonContent) -> JsonContent:
}
"""
if isinstance(json_content, dict):
- json_content = cast(Dict[str, Any], json_content)
- return {key: remove_none_values(json_content=value) for key, value in json_content.items() if value is not None}
- else:
+ cleaned_dict: Dict[str, Any] = {}
+ for key, value in json_content.items():
+ if value is not None:
+ cleaned_dict[key] = remove_none_values(json_content=value)
+ return cleaned_dict
+ elif isinstance(json_content, list):
return [remove_none_values(item) for item in json_content]
+ else:
+ return json_content
def purify_json(
diff --git a/pipelex/tools/misc/toml_utils.py b/pipelex/tools/misc/toml_utils.py
index 2316100cf..1fb0a69a8 100644
--- a/pipelex/tools/misc/toml_utils.py
+++ b/pipelex/tools/misc/toml_utils.py
@@ -1,8 +1,10 @@
from typing import Any, Dict, List, Optional
import toml
+import tomlkit
from pipelex.tools.misc.file_utils import path_exists
+from pipelex.tools.misc.json_utils import remove_none_values
class TOMLValidationError(Exception):
@@ -129,8 +131,11 @@ def save_toml_to_path(data: Dict[str, Any], path: str) -> None:
data: Dictionary to save as TOML
path: Path where to save the TOML file
"""
+ data_cleaned = remove_none_values(data)
+ if not isinstance(data_cleaned, dict):
+ raise RuntimeError("Data must be a dictionary")
with open(path, "w", encoding="utf-8") as file:
- toml_content = toml.dumps(data)
+ toml_content = tomlkit.dumps(data_cleaned) # type: ignore
cleaned_content = _clean_trailing_whitespace(toml_content)
file.write(cleaned_content)
diff --git a/pyproject.toml b/pyproject.toml
index 2c7636cd0..8d2103775 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -39,6 +39,7 @@ dependencies = [
"rich>=13.8.1",
"shortuuid>=1.0.13",
"toml>=0.10.2",
+ "tomlkit>=0.13.2",
"typer>=0.16.0",
"typing-extensions>=4.13.2",
"yattag>=1.15.2",
diff --git a/uv.lock b/uv.lock
index 573525370..ae4ea3189 100644
--- a/uv.lock
+++ b/uv.lock
@@ -1773,6 +1773,7 @@ dependencies = [
{ name = "rich" },
{ name = "shortuuid" },
{ name = "toml" },
+ { name = "tomlkit" },
{ name = "typer" },
{ name = "typing-extensions" },
{ name = "yattag" },
@@ -1874,6 +1875,7 @@ requires-dist = [
{ name = "ruff", marker = "extra == 'dev'", specifier = ">=0.6.8" },
{ name = "shortuuid", specifier = ">=1.0.13" },
{ name = "toml", specifier = ">=0.10.2" },
+ { name = "tomlkit", specifier = ">=0.13.2" },
{ name = "typer", specifier = ">=0.16.0" },
{ name = "types-aioboto3", extras = ["bedrock", "bedrock-runtime"], marker = "extra == 'dev'", specifier = ">=13.4.0" },
{ name = "types-aiofiles", marker = "extra == 'dev'", specifier = ">=24.1.0.20240626" },
@@ -2596,6 +2598,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/6e/c2/61d3e0f47e2b74ef40a68b9e6ad5984f6241a942f7cd3bbfbdbd03861ea9/tomli-2.2.1-py3-none-any.whl", hash = "sha256:cb55c73c5f4408779d0cf3eef9f762b9c9f147a77de7b258bef0a5628adc85cc", size = 14257, upload-time = "2024-11-27T22:38:35.385Z" },
]
+[[package]]
+name = "tomlkit"
+version = "0.13.3"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/cc/18/0bbf3884e9eaa38819ebe46a7bd25dcd56b67434402b66a58c4b8e552575/tomlkit-0.13.3.tar.gz", hash = "sha256:430cf247ee57df2b94ee3fbe588e71d362a941ebb545dec29b53961d61add2a1", size = 185207, upload-time = "2025-06-05T07:13:44.947Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/bd/75/8539d011f6be8e29f339c42e633aae3cb73bffa95dd0f9adec09b9c58e85/tomlkit-0.13.3-py3-none-any.whl", hash = "sha256:c89c649d79ee40629a9fda55f8ace8c6a1b42deb912b2a8fd8d942ddadb606b0", size = 38901, upload-time = "2025-06-05T07:13:43.546Z" },
+]
+
[[package]]
name = "tqdm"
version = "4.67.1"