Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
Merge branch 'dev' into refactor/Use-proper-tomlkit-to-serialize-blueprints
Louis Choqueld2e9ea1e0aec
diff --cc pipelex/core/interpreter.pyindex 03c7b226e,0d6d9509b..383ce55e0--- a/pipelex/core/interpreter.py+++ b/pipelex/core/interpreter.py@@@ -1,32 -1,32 +1,13 @@@-from collections.abc import Mappingfrom pathlib import Path- from typing import Any, Dict, Mapping, Optional, Union, cast-from typing import Any, cast++from typing import Anyimport tomlfrom pydantic import BaseModel, model_validator- from typing_extensions import Selffrom pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint- from pipelex.core.concepts.concept_blueprint import ConceptBlueprint-from pipelex.core.concepts.concept_blueprint import ConceptBlueprint, ConceptStructureBlueprintfrom pipelex.core.exceptions import (PipelexConfigurationError,-- PipelexUnknownPipeError,)--from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint--from pipelex.pipe_controllers.batch.pipe_batch_blueprint import PipeBatchBlueprint--from pipelex.pipe_controllers.condition.pipe_condition_blueprint import (-- PipeConditionBlueprint,--)--from pipelex.pipe_controllers.parallel.pipe_parallel_blueprint import (-- PipeParallelBlueprint,--)--from pipelex.pipe_controllers.sequence.pipe_sequence_blueprint import (-- PipeSequenceBlueprint,--)--from pipelex.pipe_operators.func.pipe_func_blueprint import PipeFuncBlueprint--from pipelex.pipe_operators.img_gen.pipe_img_gen_blueprint import PipeImgGenBlueprint--from pipelex.pipe_operators.jinja2.pipe_jinja2_blueprint import PipeJinja2Blueprint--from pipelex.pipe_operators.llm.pipe_llm_blueprint import PipeLLMBlueprint--from pipelex.pipe_operators.ocr.pipe_ocr_blueprint import PipeOcrBlueprintfrom pipelex.tools.misc.toml_utils import (clean_trailing_whitespace,validate_toml_content,diff --cc pipelex/tools/misc/toml_utils.pyindex 1afa2f93e,d20b5bec8..23b06db9e--- a/pipelex/tools/misc/toml_utils.py+++ b/pipelex/tools/misc/toml_utils.py@@@ -1,16 -1,17 +1,10 @@@from __future__ import annotations--import re- from typing import Any, Dict, List, Mapping, Optional, cast-from collections.abc import Mapping-from typing import Any, cast++from typing import Anyimport toml--import tomlkit--from pydantic import BaseModel--from tomlkit import array, document, inline_table, table--from tomlkit import string as tk_stringfrom pipelex.tools.misc.file_utils import path_exists--from pipelex.tools.misc.json_utils import remove_none_values_from_dictclass TOMLValidationError(Exception):diff --cc pipelex/tools/plx/plx_utils.pyindex cec1937aa,000000000..021c96543mode 100644,000000..100644--- a/pipelex/tools/plx/plx_utils.py+++ b/pipelex/tools/plx/plx_utils.py@@@ -1,114 -1,0 +1,112 @@@+from __future__ import annotations+- from typing import Any, List, Mapping, cast++from typing import TYPE_CHECKING, Any, Mapping, cast++import tomlkit+from tomlkit import array, document, inline_table, table+from tomlkit import string as tomlkit_string+- from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint- from pipelex.tools.misc.file_utils import save_text_to_path+from pipelex.tools.misc.json_utils import remove_none_values_from_dict+++if TYPE_CHECKING:++ from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint++++def _format_tomlkit_string(+ text: str,+ prefer_literal: bool = False,+ force_multiline: bool = False,+ ensure_trailing_newline: bool = True,+ ensure_leading_blank_line: bool = True,+) -> Any: # Can't type this because of tomlkit- """- Build a tomlkit string node.++ r"""Build a tomlkit string node.+ - If `force_multiline` or the text contains '\\n', we emit a triple-quoted multiline string.+ - When multiline, `ensure_trailing_newline` puts the closing quotes on their own line.+ - When multiline, `ensure_leading_blank_line` inserts a real blank line at the start of the string.+ """+ needs_multiline = force_multiline or ("\n" in text)+ normalized = text++ if needs_multiline:+ if ensure_leading_blank_line and not normalized.startswith("\n"):+ normalized = "\n" + normalized+ if ensure_trailing_newline and not normalized.endswith("\n"):+ normalized = normalized + "\n"++ use_literal = prefer_literal and ("'''" not in normalized)+ return tomlkit_string(normalized, multiline=needs_multiline, literal=use_literal)+++def _convert_dicts_to_inline_tables(value: Any) -> Any: # Can't type this because of tomlkit+ """Recursively convert Python values; dicts -> inline tables; lists kept as arrays."""+ if isinstance(value, Mapping):- value = cast(Mapping[str, Any], value)++ value = cast("Mapping[str, Any]", value)+ inline_table_obj = inline_table()+ for key, value_item in value.items():+ inline_table_obj[key] = _convert_dicts_to_inline_tables(value_item)+ return inline_table_obj++ if isinstance(value, list):- value = cast(List[Any], value)++ value = cast("list[Any]", value)+ array_obj = array()+ array_obj.multiline(True) # set to False for single-line arrays+ for element in value:+ if isinstance(element, Mapping):- element = cast(Mapping[str, Any], element)++ element = cast("Mapping[str, Any]", element)+ inline_element = inline_table()+ for inner_key, inner_value in element.items():+ inline_element[inner_key] = _convert_dicts_to_inline_tables(inner_value)+ array_obj.append(inline_element) # pyright: ignore[reportUnknownMemberType]+ else:+ array_obj.append(_convert_dicts_to_inline_tables(element)) # pyright: ignore[reportUnknownMemberType]+ return array_obj++ if isinstance(value, str):+ return _format_tomlkit_string(value)+ return value+++def _convert_mapping_to_table(mapping: Mapping[str, Any]) -> Any: # Can't type this because of tomlkit- """- Convert a mapping into a TOML Table where any nested mappings (third level+)++ """Convert a mapping into a TOML Table where any nested mappings (third level+)+ are converted to inline tables.++ This creates a second-level standard table, and only uses inline tables for+ third level and deeper structures.+ """+ tbl = table()+ for field_key, field_value in mapping.items():+ if isinstance(field_value, Mapping):+ # Third-level mapping -> inline table+ tbl.add(field_key, _convert_dicts_to_inline_tables(field_value))+ else:+ tbl.add(field_key, _convert_dicts_to_inline_tables(field_value))+ return tbl+++def dict_to_plx_styled_toml(data: Mapping[str, Any]) -> str:+ """Top-level keys become tables; second-level mappings become tables; inline tables start at third level."""+ data = remove_none_values_from_dict(data=data)+ document_root = document()+ for section_key, section_value in data.items():+ if isinstance(section_value, Mapping):- section_value = cast(Mapping[str, Any], section_value)++ section_value = cast("Mapping[str, Any]", section_value)+ # Skip empty mappings (empty concept and pipe sections)+ if not section_value:+ continue+ table_obj = table()+ for field_key, field_value in section_value.items():+ if isinstance(field_value, Mapping):+ # Second level mapping -> standard table [section.field]- sub_table = _convert_mapping_to_table(cast(Mapping[str, Any], field_value))++ sub_table = _convert_mapping_to_table(cast("Mapping[str, Any]", field_value))+ table_obj.add(field_key, sub_table)+ else:+ table_obj.add(field_key, _convert_dicts_to_inline_tables(field_value))+ document_root.add(section_key, table_obj) # `[section_key]` section+ else:+ document_root.add(section_key, _convert_dicts_to_inline_tables(section_value))- dumped_content = tomlkit.dumps(document_root) # pyright: ignore[reportUnknownMemberType]- return dumped_content++ return tomlkit.dumps(document_root) # pyright: ignore[reportUnknownMemberType]+++def make_plx_content(blueprint: PipelexBundleBlueprint) -> str:+ blueprint_dict = blueprint.model_dump(serialize_as_any=True)+ return dict_to_plx_styled_toml(data=blueprint_dict)diff --cc tests/unit/pipelex/core/interpreter/test_interpreter.pyindex 023730c59,12bf546de..443b560df--- a/tests/unit/pipelex/core/interpreter/test_interpreter.py+++ b/tests/unit/pipelex/core/interpreter/test_interpreter.py@@@ -3,10 -2,9 +3,10 @@@ from typing import Typimport pytest--from pipelex import pretty_print++from pipelex import log, pretty_printfrom pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprintfrom pipelex.core.interpreter import PipelexInterpreter+from pipelex.tools.plx.plx_utils import make_plx_contentfrom tests.unit.pipelex.core.test_data import InterpreterTestCases@@@ -31,18 -28,17 +31,19 @@@ class TestPipelexInterpreterpretty_print(expected_blueprint, title=f"Expected blueprint {test_name}")assert blueprint == expected_blueprint- @pytest.mark.parametrize("test_name,plx_content,expected_blueprint", InterpreterTestCases.VALID_TEST_CASES)- def test_make_plx_content(self, test_name: str, plx_content: str, expected_blueprint: PipelexBundleBlueprint):- get_plx_content = PipelexInterpreter.make_plx_content(blueprint=expected_blueprint)-- pretty_print(get_plx_content, title=f"PLx content {test_name}")- pretty_print(plx_content, title=f"Expected PLX content {test_name}")- assert get_plx_content == plx_content+ @pytest.mark.parametrize("test_name,expected_plx_content,blueprint", InterpreterTestCases.VALID_TEST_CASES)+ def test_make_plx_content(self, test_name: str, expected_plx_content: str, blueprint: PipelexBundleBlueprint):+ """Test making blueprint from various valid PLX content."""+ plx_content = make_plx_content(blueprint=blueprint)+ pretty_print(plx_content, title=f"PLx content {test_name}")+ pretty_print(expected_plx_content, title=f"Expected PLX content {test_name}")+ assert plx_content == expected_plx_content@pytest.mark.parametrize("test_name,invalid_plx_content,expected_exception", InterpreterTestCases.ERROR_TEST_CASES)- def test_invalid_plx_should_raise_exception(self, test_name: str, invalid_plx_content: str, expected_exception: type[Exception]):+ def test_invalid_plx_should_raise_exception(self, test_name: str, invalid_plx_content: str, expected_exception: Type[Exception]):+ """Test that invalid PLX content raises appropriate exceptions."""converter = PipelexInterpreter(file_content=invalid_plx_content)++ log.verbose(f"Testing invalid PLX content: {test_name}")with pytest.raises(expected_exception):converter.make_pipelex_bundle_blueprint()