Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

Merge branch 'dev' into refactor/Use-proper-tomlkit-to-serialize-blueprints

Louis Choqueld2e9ea1e0aec
diff --cc pipelex/core/interpreter.py
index 03c7b226e,0d6d9509b..383ce55e0
--- a/pipelex/core/interpreter.py
+++ b/pipelex/core/interpreter.py
@@@ -1,32 -1,32 +1,13 @@@
-from collections.abc import Mapping
from pathlib import Path
- from typing import Any, Dict, Mapping, Optional, Union, cast
-from typing import Any, cast
++from typing import Any
import toml
from pydantic import BaseModel, model_validator
- from typing_extensions import Self
from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint
- from pipelex.core.concepts.concept_blueprint import ConceptBlueprint
-from pipelex.core.concepts.concept_blueprint import ConceptBlueprint, ConceptStructureBlueprint
from pipelex.core.exceptions import (
PipelexConfigurationError,
-- PipelexUnknownPipeError,
)
--from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
--from pipelex.pipe_controllers.batch.pipe_batch_blueprint import PipeBatchBlueprint
--from pipelex.pipe_controllers.condition.pipe_condition_blueprint import (
-- PipeConditionBlueprint,
--)
--from pipelex.pipe_controllers.parallel.pipe_parallel_blueprint import (
-- PipeParallelBlueprint,
--)
--from pipelex.pipe_controllers.sequence.pipe_sequence_blueprint import (
-- PipeSequenceBlueprint,
--)
--from pipelex.pipe_operators.func.pipe_func_blueprint import PipeFuncBlueprint
--from pipelex.pipe_operators.img_gen.pipe_img_gen_blueprint import PipeImgGenBlueprint
--from pipelex.pipe_operators.jinja2.pipe_jinja2_blueprint import PipeJinja2Blueprint
--from pipelex.pipe_operators.llm.pipe_llm_blueprint import PipeLLMBlueprint
--from pipelex.pipe_operators.ocr.pipe_ocr_blueprint import PipeOcrBlueprint
from pipelex.tools.misc.toml_utils import (
clean_trailing_whitespace,
validate_toml_content,
diff --cc pipelex/tools/misc/toml_utils.py
index 1afa2f93e,d20b5bec8..23b06db9e
--- a/pipelex/tools/misc/toml_utils.py
+++ b/pipelex/tools/misc/toml_utils.py
@@@ -1,16 -1,17 +1,10 @@@
from __future__ import annotations
--import re
- from typing import Any, Dict, List, Mapping, Optional, cast
-from collections.abc import Mapping
-from typing import Any, cast
++from typing import Any
import toml
--import tomlkit
--from pydantic import BaseModel
--from tomlkit import array, document, inline_table, table
--from tomlkit import string as tk_string
from pipelex.tools.misc.file_utils import path_exists
--from pipelex.tools.misc.json_utils import remove_none_values_from_dict
class TOMLValidationError(Exception):
diff --cc pipelex/tools/plx/plx_utils.py
index cec1937aa,000000000..021c96543
mode 100644,000000..100644
--- a/pipelex/tools/plx/plx_utils.py
+++ b/pipelex/tools/plx/plx_utils.py
@@@ -1,114 -1,0 +1,112 @@@
+from __future__ import annotations
+
- from typing import Any, List, Mapping, cast
++from typing import TYPE_CHECKING, Any, Mapping, cast
+
+import tomlkit
+from tomlkit import array, document, inline_table, table
+from tomlkit import string as tomlkit_string
+
- from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint
- from pipelex.tools.misc.file_utils import save_text_to_path
+from pipelex.tools.misc.json_utils import remove_none_values_from_dict
+
++if TYPE_CHECKING:
++ from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint
++
+
+def _format_tomlkit_string(
+ text: str,
+ prefer_literal: bool = False,
+ force_multiline: bool = False,
+ ensure_trailing_newline: bool = True,
+ ensure_leading_blank_line: bool = True,
+) -> Any: # Can't type this because of tomlkit
- """
- Build a tomlkit string node.
++ r"""Build a tomlkit string node.
+ - If `force_multiline` or the text contains '\\n', we emit a triple-quoted multiline string.
+ - When multiline, `ensure_trailing_newline` puts the closing quotes on their own line.
+ - When multiline, `ensure_leading_blank_line` inserts a real blank line at the start of the string.
+ """
+ needs_multiline = force_multiline or ("\n" in text)
+ normalized = text
+
+ if needs_multiline:
+ if ensure_leading_blank_line and not normalized.startswith("\n"):
+ normalized = "\n" + normalized
+ if ensure_trailing_newline and not normalized.endswith("\n"):
+ normalized = normalized + "\n"
+
+ use_literal = prefer_literal and ("'''" not in normalized)
+ return tomlkit_string(normalized, multiline=needs_multiline, literal=use_literal)
+
+
+def _convert_dicts_to_inline_tables(value: Any) -> Any: # Can't type this because of tomlkit
+ """Recursively convert Python values; dicts -> inline tables; lists kept as arrays."""
+ if isinstance(value, Mapping):
- value = cast(Mapping[str, Any], value)
++ value = cast("Mapping[str, Any]", value)
+ inline_table_obj = inline_table()
+ for key, value_item in value.items():
+ inline_table_obj[key] = _convert_dicts_to_inline_tables(value_item)
+ return inline_table_obj
+
+ if isinstance(value, list):
- value = cast(List[Any], value)
++ value = cast("list[Any]", value)
+ array_obj = array()
+ array_obj.multiline(True) # set to False for single-line arrays
+ for element in value:
+ if isinstance(element, Mapping):
- element = cast(Mapping[str, Any], element)
++ element = cast("Mapping[str, Any]", element)
+ inline_element = inline_table()
+ for inner_key, inner_value in element.items():
+ inline_element[inner_key] = _convert_dicts_to_inline_tables(inner_value)
+ array_obj.append(inline_element) # pyright: ignore[reportUnknownMemberType]
+ else:
+ array_obj.append(_convert_dicts_to_inline_tables(element)) # pyright: ignore[reportUnknownMemberType]
+ return array_obj
+
+ if isinstance(value, str):
+ return _format_tomlkit_string(value)
+ return value
+
+
+def _convert_mapping_to_table(mapping: Mapping[str, Any]) -> Any: # Can't type this because of tomlkit
- """
- Convert a mapping into a TOML Table where any nested mappings (third level+)
++ """Convert a mapping into a TOML Table where any nested mappings (third level+)
+ are converted to inline tables.
+
+ This creates a second-level standard table, and only uses inline tables for
+ third level and deeper structures.
+ """
+ tbl = table()
+ for field_key, field_value in mapping.items():
+ if isinstance(field_value, Mapping):
+ # Third-level mapping -> inline table
+ tbl.add(field_key, _convert_dicts_to_inline_tables(field_value))
+ else:
+ tbl.add(field_key, _convert_dicts_to_inline_tables(field_value))
+ return tbl
+
+
+def dict_to_plx_styled_toml(data: Mapping[str, Any]) -> str:
+ """Top-level keys become tables; second-level mappings become tables; inline tables start at third level."""
+ data = remove_none_values_from_dict(data=data)
+ document_root = document()
+ for section_key, section_value in data.items():
+ if isinstance(section_value, Mapping):
- section_value = cast(Mapping[str, Any], section_value)
++ section_value = cast("Mapping[str, Any]", section_value)
+ # Skip empty mappings (empty concept and pipe sections)
+ if not section_value:
+ continue
+ table_obj = table()
+ for field_key, field_value in section_value.items():
+ if isinstance(field_value, Mapping):
+ # Second level mapping -> standard table [section.field]
- sub_table = _convert_mapping_to_table(cast(Mapping[str, Any], field_value))
++ sub_table = _convert_mapping_to_table(cast("Mapping[str, Any]", field_value))
+ table_obj.add(field_key, sub_table)
+ else:
+ table_obj.add(field_key, _convert_dicts_to_inline_tables(field_value))
+ document_root.add(section_key, table_obj) # `[section_key]` section
+ else:
+ document_root.add(section_key, _convert_dicts_to_inline_tables(section_value))
- dumped_content = tomlkit.dumps(document_root) # pyright: ignore[reportUnknownMemberType]
- return dumped_content
++ return tomlkit.dumps(document_root) # pyright: ignore[reportUnknownMemberType]
+
+
+def make_plx_content(blueprint: PipelexBundleBlueprint) -> str:
+ blueprint_dict = blueprint.model_dump(serialize_as_any=True)
+ return dict_to_plx_styled_toml(data=blueprint_dict)
diff --cc tests/unit/pipelex/core/interpreter/test_interpreter.py
index 023730c59,12bf546de..443b560df
--- a/tests/unit/pipelex/core/interpreter/test_interpreter.py
+++ b/tests/unit/pipelex/core/interpreter/test_interpreter.py
@@@ -3,10 -2,9 +3,10 @@@ from typing import Typ
import pytest
--from pipelex import pretty_print
++from pipelex import log, pretty_print
from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint
from pipelex.core.interpreter import PipelexInterpreter
+from pipelex.tools.plx.plx_utils import make_plx_content
from tests.unit.pipelex.core.test_data import InterpreterTestCases
@@@ -31,18 -28,17 +31,19 @@@ class TestPipelexInterpreter
pretty_print(expected_blueprint, title=f"Expected blueprint {test_name}")
assert blueprint == expected_blueprint
- @pytest.mark.parametrize("test_name,plx_content,expected_blueprint", InterpreterTestCases.VALID_TEST_CASES)
- def test_make_plx_content(self, test_name: str, plx_content: str, expected_blueprint: PipelexBundleBlueprint):
- get_plx_content = PipelexInterpreter.make_plx_content(blueprint=expected_blueprint)
-
- pretty_print(get_plx_content, title=f"PLx content {test_name}")
- pretty_print(plx_content, title=f"Expected PLX content {test_name}")
- assert get_plx_content == plx_content
+ @pytest.mark.parametrize("test_name,expected_plx_content,blueprint", InterpreterTestCases.VALID_TEST_CASES)
+ def test_make_plx_content(self, test_name: str, expected_plx_content: str, blueprint: PipelexBundleBlueprint):
+ """Test making blueprint from various valid PLX content."""
+ plx_content = make_plx_content(blueprint=blueprint)
+ pretty_print(plx_content, title=f"PLx content {test_name}")
+ pretty_print(expected_plx_content, title=f"Expected PLX content {test_name}")
+ assert plx_content == expected_plx_content
@pytest.mark.parametrize("test_name,invalid_plx_content,expected_exception", InterpreterTestCases.ERROR_TEST_CASES)
- def test_invalid_plx_should_raise_exception(self, test_name: str, invalid_plx_content: str, expected_exception: type[Exception]):
+ def test_invalid_plx_should_raise_exception(self, test_name: str, invalid_plx_content: str, expected_exception: Type[Exception]):
+ """Test that invalid PLX content raises appropriate exceptions."""
converter = PipelexInterpreter(file_content=invalid_plx_content)
++ log.verbose(f"Testing invalid PLX content: {test_name}")
with pytest.raises(expected_exception):
converter.make_pipelex_bundle_blueprint()