Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
Remove useless toml validations
Louis Choquel585b692fadfb
diff --git a/pipelex/cogt/model_routing/routing_profile_library.py b/pipelex/cogt/model_routing/routing_profile_library.pyindex 949e8a60c..1fb265580 100644--- a/pipelex/cogt/model_routing/routing_profile_library.py+++ b/pipelex/cogt/model_routing/routing_profile_library.py@@ -9,7 +9,7 @@ from pipelex.cogt.model_routing.routing_profile_factory import (RoutingProfileLibraryBlueprint,)from pipelex.config import get_config-from pipelex.tools.misc.toml_utils import TOMLValidationError, load_toml_from_path+from pipelex.tools.misc.toml_utils import load_toml_from_pathfrom pipelex.types import SelfRoutingProfileLibraryRoot = dict[str, RoutingProfile]@@ -47,9 +47,6 @@ class RoutingProfileLibrary(RootModel[RoutingProfileLibraryRoot]):except FileNotFoundError as not_found_exc:msg = f"Failed to load routing profile library from file '{routing_profile_library_path}': {not_found_exc}"raise RoutingProfileLibraryNotFoundError(msg) from not_found_exc- except TOMLValidationError as toml_validation_exc:- msg = f"Failed to load routing profile library from file '{routing_profile_library_path}': {toml_validation_exc}"- raise RoutingProfileLibraryError(msg) from toml_validation_exctry:catalog_blueprint = RoutingProfileLibraryBlueprint.model_validate(catalog_dict)diff --git a/pipelex/cogt/models/model_manager.py b/pipelex/cogt/models/model_manager.pyindex a314ac2ff..37f5dbaa2 100644--- a/pipelex/cogt/models/model_manager.py+++ b/pipelex/cogt/models/model_manager.py@@ -13,9 +13,8 @@ from pipelex.cogt.model_routing.routing_profile_library import RoutingProfileLibfrom pipelex.cogt.models.model_deck import ModelDeck, ModelDeckBlueprintfrom pipelex.cogt.models.model_manager_abstract import ModelManagerAbstractfrom pipelex.config import get_config-from pipelex.exceptions import LibraryErrorfrom pipelex.tools.misc.json_utils import deep_update-from pipelex.tools.misc.toml_utils import TOMLValidationError, load_toml_from_path, validate_toml_file+from pipelex.tools.misc.toml_utils import load_toml_from_pathclass ModelManager(ModelManagerAbstract):@@ -43,21 +42,6 @@ class ModelManager(ModelManagerAbstract):deck_blueprint = self.load_deck_blueprint()self.model_deck = self.build_deck(model_deck_blueprint=deck_blueprint)- @classmethod- def _validate_toml_files(cls) -> None:- log.debug("LibraryManager deck TOML file formatting")-- # Validation of LLM deck paths- deck_paths = get_config().cogt.inference_config.get_model_deck_paths()- for deck_path in deck_paths:- if os.path.exists(deck_path):- try:- validate_toml_file(deck_path)- except TOMLValidationError as exc:- msg = f"TOML validation failed for LLM deck file '{deck_path}': {exc}"- log.error(msg)- raise LibraryError(msg) from exc-@classmethoddef load_deck_blueprint(cls) -> ModelDeckBlueprint:deck_paths = get_config().cogt.inference_config.get_model_deck_paths()diff --git a/pipelex/core/concepts/concept.py b/pipelex/core/concepts/concept.pyindex e8630ed25..b7dcc7357 100644--- a/pipelex/core/concepts/concept.py+++ b/pipelex/core/concepts/concept.py@@ -80,7 +80,7 @@ class Concept(BaseModel):except TypeError:pass- # Check if concept_1 has a field that is of type concept_2_class+ # Check if concept_1 has compatible fields with concept_2return ClassRegistryUtils.has_compatible_field(concept_1_class, concept_2_class)return Falsediff --git a/pipelex/core/interpreter.py b/pipelex/core/interpreter.pyindex 383ce55e0..036e550f3 100644--- a/pipelex/core/interpreter.py+++ b/pipelex/core/interpreter.py@@ -8,11 +8,7 @@ from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprintfrom pipelex.core.exceptions import (PipelexConfigurationError,)-from pipelex.tools.misc.toml_utils import (- clean_trailing_whitespace,- validate_toml_content,- validate_toml_file,-)+from pipelex.tools.misc.file_utils import load_text_from_pathfrom pipelex.types import Self@@ -48,31 +44,15 @@ class PipelexInterpreter(BaseModel):raise PipelexConfigurationError(msg)return self- @model_validator(mode="after")- def validate_file_path(self) -> Self:- if self.file_path:- validate_toml_file(path=str(self.file_path))- if self.file_content:- validate_toml_content(content=self.file_content, file_path=str(self.file_path))- return self-- def _read_and_reformat(self) -> str:+ def _get_content(self) -> str:"""Load PLX content from file_path or use file_content directly."""if self.file_path:- with open(self.file_path, encoding="utf-8") as file:- file_content = file.read()-- # Clean trailing whitespace and write back if needed- cleaned_content = clean_trailing_whitespace(file_content)- if file_content != cleaned_content:- with open(self.file_path, "w", encoding="utf-8") as file:- file.write(cleaned_content)- return cleaned_content- return file_content- if self.file_content is None:+ return load_text_from_path(path=str(self.file_path))+ elif self.file_content:+ return self.file_content+ else:msg = "file_content must be provided if file_path is not provided"raise PipelexConfigurationError(msg)- return self.file_content@staticmethoddef is_pipelex_file(file_path: Path) -> bool:@@ -120,6 +100,6 @@ class PipelexInterpreter(BaseModel):def make_pipelex_bundle_blueprint(self) -> PipelexBundleBlueprint:"""Make a PipelexBundleBlueprint from the file_path or file_content"""- file_content = self._read_and_reformat()+ file_content = self._get_content()plx_data = self._parse_plx_content(file_content)return PipelexBundleBlueprint.model_validate(plx_data)diff --git a/pipelex/core/pipes/pipe_abstract.py b/pipelex/core/pipes/pipe_abstract.pyindex ff94528b8..5e543a183 100644--- a/pipelex/core/pipes/pipe_abstract.py+++ b/pipelex/core/pipes/pipe_abstract.py@@ -25,8 +25,8 @@ class PipeAbstract(ABC, BaseModel):output: Concept@field_validator("code", mode="before")- @staticmethod- def validate_pipe_code_syntax(code: str) -> str:+ @classmethod+ def validate_pipe_code_syntax(cls, code: str) -> str:PipeBlueprint.validate_pipe_code_syntax(pipe_code=code)return codediff --git a/pipelex/libraries/library_manager.py b/pipelex/libraries/library_manager.pyindex 4c43a1706..e2e38d3e9 100644--- a/pipelex/libraries/library_manager.py+++ b/pipelex/libraries/library_manager.py@@ -1,4 +1,3 @@-import osfrom pathlib import Pathfrom typing import ClassVar@@ -37,7 +36,6 @@ from pipelex.libraries.library_manager_abstract import LibraryManagerAbstractfrom pipelex.tools.class_registry_utils import ClassRegistryUtilsfrom pipelex.tools.func_registry_utils import FuncRegistryUtilsfrom pipelex.tools.misc.file_utils import find_files_in_dir-from pipelex.tools.misc.toml_utils import TOMLValidationError, validate_toml_filefrom pipelex.tools.runtime_manager import runtime_managerfrom pipelex.tools.typing.pydantic_utils import format_pydantic_validation_errorfrom pipelex.types import StrEnum@@ -85,21 +83,6 @@ class LibraryManager(LibraryManagerAbstract):self.pipe_library.validate_with_libraries()self.domain_library.validate_with_libraries()- def _validate_toml_files(self):- """Validate all TOML files used by the library manager for formatting issues."""- log.debug("LibraryManager validating TOML file formatting")-- # Validation of template paths- template_paths = self.library_config.get_templates_paths()- for template_path in template_paths:- if os.path.exists(template_path):- try:- validate_toml_file(template_path)- except TOMLValidationError as exc:- msg = f"PLX validation failed for template file '{template_path}': {exc}"- log.error(msg)- raise LibraryError(msg) from exc-@overridedef setup(self) -> None:self.concept_library.setup()diff --git a/pipelex/pipe_controllers/parallel/pipe_parallel_blueprint.py b/pipelex/pipe_controllers/parallel/pipe_parallel_blueprint.pyindex 9351f0814..d8174a418 100644--- a/pipelex/pipe_controllers/parallel/pipe_parallel_blueprint.py+++ b/pipelex/pipe_controllers/parallel/pipe_parallel_blueprint.py@@ -17,8 +17,8 @@ class PipeParallelBlueprint(PipeBlueprint):combined_output: str | None = None@field_validator("combined_output", mode="before")- @staticmethod- def validate_combined_output(combined_output: str) -> str:+ @classmethod+ def validate_combined_output(cls, combined_output: str) -> str:if combined_output:ConceptBlueprint.validate_concept_string_or_code(concept_string_or_code=combined_output)return combined_outputdiff --git a/pipelex/pipe_operators/llm/pipe_llm_blueprint.py b/pipelex/pipe_operators/llm/pipe_llm_blueprint.pyindex f58c5a59d..b8457c2e0 100644--- a/pipelex/pipe_operators/llm/pipe_llm_blueprint.py+++ b/pipelex/pipe_operators/llm/pipe_llm_blueprint.py@@ -39,11 +39,11 @@ class PipeLLMBlueprint(PipeBlueprint):multiple_output: bool | None = None@field_validator("nb_output", mode="after")- @staticmethod- def validate_nb_output(value: int | None = None) -> int | None:- if value and value < 1:- msg = "PipeLLMBlueprint nb_output must be greater than 0"- raise PipeDefinitionError(msg)+ @classmethod+ def validate_nb_output(cls, value: int | None = None) -> int | None:+ if value and value < 2:+ msg = "PipeLLMBlueprint nb_output must be at least 2"+ raise PipeDefinitionError(message=msg)return value@model_validator(mode="after")diff --git a/pipelex/tools/misc/toml_utils.py b/pipelex/tools/misc/toml_utils.pyindex ea6b4eff8..b01ea4515 100644--- a/pipelex/tools/misc/toml_utils.py+++ b/pipelex/tools/misc/toml_utils.py@@ -7,84 +7,6 @@ import tomlfrom pipelex.tools.misc.file_utils import path_exists-class TOMLValidationError(Exception):- """Raised when TOML file has formatting issues that could cause problems."""---def validate_toml_content(content: str, file_path: str | None = None) -> None:- """Validate TOML content for common formatting issues."""- # lines = content.splitlines()- # issues: list[str] = []-- # for line_num, line in enumerate(lines, 1):- # # Check for trailing whitespace- # if line.rstrip() != line:- # trailing_chars = line[len(line.rstrip()) :]- # trailing_repr = repr(trailing_chars)- # issues.append(f"Line {line_num}: Trailing whitespace detected: {trailing_repr}")-- # # Check for trailing whitespace after triple quotes (common issue)- # if line.strip().endswith('"""') and line != line.rstrip():- # issues.append(f"Line {line_num}: Trailing whitespace after triple quotes - this can cause TOML parsing issues")-- # # Check for mixed line endings- # has_crlf = "\r\n" in content- # content_without_crlf = content.replace("\r\n", "")- # has_standalone_lf = "\n" in content_without_crlf- # if has_crlf and has_standalone_lf:- # issues.append("Mixed line endings detected (both CRLF and LF)")-- # if issues:- # error_msg = f"TOML formatting issues in '{file_path}':\n" + "\n".join(f" - {issue}" for issue in issues)- # raise TOMLValidationError(error_msg)---def validate_toml_file(path: str) -> None:- """Validate TOML file for formatting issues.-- Args:- path: Path to the TOML file to validate-- Raises:- TOMLValidationError: If formatting issues are detected-- """- with open(path, encoding="utf-8") as file:- content = file.read()- validate_toml_content(content, path)---def clean_trailing_whitespace(content: str) -> str:- # """Clean trailing whitespace from TOML content.-- # This function:- # 1. Removes trailing whitespace from all lines- # 2. Ensures exactly one empty line at EOF (two newline characters)- # 3. If no empty line at EOF, removes trailing whitespace from last non-empty line-- # Args:- # content: The TOML content to clean-- # Returns:- # The cleaned TOML content with trailing whitespace removed and an empty line at EOF-- # """- # # Split into lines and clean each line- # lines = [line.rstrip() for line in content.splitlines()]-- # # Remove trailing empty lines- # while lines and not lines[-1]:- # lines.pop()-- # # If we have lines and the last line has trailing whitespace, remove it- # if lines:- # lines[-1] = lines[-1].rstrip()-- # # Join with newlines and ensure an empty line at EOF (two newlines)- # return "\n".join(lines) + "\n\n"- return content--def load_toml_from_path(path: str) -> dict[str, Any]:"""Load TOML from path.@@ -102,15 +24,8 @@ def load_toml_from_path(path: str) -> dict[str, Any]:with open(path, encoding="utf-8") as file:content = file.read()- cleaned_content = clean_trailing_whitespace(content)-- # If content changed, write it back- if content != cleaned_content:- with open(path, "w", encoding="utf-8") as file:- file.write(cleaned_content)-# Parse TOML first- return toml.loads(cleaned_content)+ return toml.loads(content)except toml.TomlDecodeError as exc:msg = f"TOML parsing error in file '{path}': {exc}"raise toml.TomlDecodeError(msg, exc.doc, exc.pos) from excdiff --git a/tests/test_pipelines/pipe_controllers/pipe_sequence/pipe_sequence_2.plx b/tests/test_pipelines/pipe_controllers/pipe_sequence/pipe_sequence_2.plxindex 689564787..60f3465a8 100644--- a/tests/test_pipelines/pipe_controllers/pipe_sequence/pipe_sequence_2.plx+++ b/tests/test_pipelines/pipe_controllers/pipe_sequence/pipe_sequence_2.plx@@ -25,7 +25,7 @@ type = "PipeLLM"definition = "Extract individual reviews from document"inputs = { document = "Document" }output = "CustomerReview"-nb_output = 1+multiple_output = truellm = "llm_for_testing_gen_text"prompt_template = """Extract each individual customer review from this document as separate items:diff --git a/tests/unit/pipelex/tools/misc/test_toml_utils.py b/tests/unit/pipelex/tools/misc/test_toml_utils.pyindex d9ca70087..a394d6b27 100644--- a/tests/unit/pipelex/tools/misc/test_toml_utils.py+++ b/tests/unit/pipelex/tools/misc/test_toml_utils.py@@ -3,10 +3,8 @@ from pathlib import Pathimport pytestfrom pipelex.tools.misc.toml_utils import (- TOMLValidationError,load_toml_from_path_if_exists,load_toml_from_path,- validate_toml_file,)@@ -46,15 +44,6 @@ definition = "Test"toml_file = tmp_path / "trailing_space.toml"toml_file.write_text(toml_content)- # with pytest.raises(TOMLValidationError) as exc_info:- # validate_toml_file(str(toml_file))-- # error_msg = str(exc_info.value)- # assert "Trailing whitespace detected" in error_msg- # assert "Line 1" in error_msg- # assert "Line 2" in error_msg- validate_toml_file(str(toml_file))-def test_validate_toml_file_trailing_space_after_triple_quotes(self, tmp_path: Path) -> None:"""Test detection of trailing whitespace after triple quotes."""toml_content = '''domain = "test"@@ -69,14 +58,6 @@ Output this only: "test"toml_file = tmp_path / "trailing_quotes.toml"toml_file.write_text(toml_content)- # with pytest.raises(TOMLValidationError) as exc_info:- # validate_toml_file(str(toml_file))-- # error_msg = str(exc_info.value)- # assert "Trailing whitespace after triple quotes" in error_msg- # assert "Line 8" in error_msg- validate_toml_file(str(toml_file))-@pytest.mark.skip(reason="Mixed line ending detection needs refinement - focusing on trailing whitespace detection")def test_validate_toml_file_mixed_line_endings(self, tmp_path: Path) -> None:"""Test detection of mixed line endings."""@@ -86,13 +67,6 @@ Output this only: "test"mixed_content = b'domain = "test"\r\ndefinition = "Test"\nextra = "value"\n'toml_file.write_bytes(mixed_content)- # with pytest.raises(TOMLValidationError) as exc_info:- # validate_toml_file(str(toml_file))-- # error_msg = str(exc_info.value)- # assert "Mixed line endings detected" in error_msg- validate_toml_file(str(toml_file))-def test_load_toml_from_path_no_validation_by_default(self, tmp_path: Path) -> None:"""Test that loading works normally without validation."""toml_content = """domain = "test"@@ -148,9 +122,6 @@ TestConcept = "A test concept"toml_file = tmp_path / "valid.toml"toml_file.write_text(toml_content)- # Should not raise- validate_toml_file(str(toml_file))-def test_validate_toml_file_multiple_validation_issues(self, tmp_path: Path) -> None:"""Test that multiple validation issues are all reported."""toml_content = """domain = "test"@@ -164,17 +135,6 @@ Output: "test"toml_file = tmp_path / "multiple_issues.toml"toml_file.write_text(toml_content)- # with pytest.raises(TOMLValidationError) as exc_info:- # validate_toml_file(str(toml_file))-- # error_msg = str(exc_info.value)- # # Should detect multiple lines with trailing whitespace- # assert "Line 1" in error_msg- # assert "Line 2" in error_msg- # assert "Line 7" in error_msg- # assert "Trailing whitespace after triple quotes" in error_msg- validate_toml_file(str(toml_file))-def test_validate_toml_file_error_contains_file_path(self, tmp_path: Path) -> None:"""Test that validation error includes the file path."""toml_content = """domain = "test"@@ -182,14 +142,6 @@ Output: "test"toml_file = tmp_path / "path_test.toml"toml_file.write_text(toml_content)- # with pytest.raises(TOMLValidationError) as exc_info:- # validate_toml_file(str(toml_file))-- # error_msg = str(exc_info.value)- # assert str(toml_file) in error_msg- # assert "TOML formatting issues" in error_msg- validate_toml_file(str(toml_file))-def test_validate_toml_file_pipe_condition_real_case(self, tmp_path: Path) -> None:"""Test the exact scenario from pipe_condition_2.toml with trailing space after triple quotes."""@@ -222,23 +174,8 @@ Output this only: "large"toml_file = tmp_path / "pipe_condition_real_case.toml"toml_file.write_text(toml_content)- # with pytest.raises(TOMLValidationError) as exc_info:- # validate_toml_file(str(toml_file))-- # error_msg = str(exc_info.value)- # assert "Trailing whitespace after triple quotes" in error_msg- # assert "Line 26" in error_msg # The line with """ followed by space- validate_toml_file(str(toml_file))-def test_validate_toml_file_actual_problematic_file(self) -> None:"""Test validation on the actual problematic file from the codebase."""- problematic_file = "tests/data/tools_data/problematic_test_cases.toml"-- # # This should catch multiple trailing whitespace issues- # with pytest.raises(TOMLValidationError) as exc_info:- # validate_toml_file(problematic_file)+ # problematic_file = "tests/data/tools_data/problematic_test_cases.toml"- # error_msg = str(exc_info.value)- # assert "Trailing whitespace" in error_msg- # assert problematic_file in error_msg- validate_toml_file(problematic_file)+ # validate_toml_file(problematic_file)