Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

Remove useless toml validations

Louis Choquel585b692fadfb
diff --git a/pipelex/cogt/model_routing/routing_profile_library.py b/pipelex/cogt/model_routing/routing_profile_library.py
index 949e8a60c..1fb265580 100644
--- a/pipelex/cogt/model_routing/routing_profile_library.py
+++ b/pipelex/cogt/model_routing/routing_profile_library.py
@@ -9,7 +9,7 @@ from pipelex.cogt.model_routing.routing_profile_factory import (
RoutingProfileLibraryBlueprint,
)
from pipelex.config import get_config
-from pipelex.tools.misc.toml_utils import TOMLValidationError, load_toml_from_path
+from pipelex.tools.misc.toml_utils import load_toml_from_path
from pipelex.types import Self
RoutingProfileLibraryRoot = dict[str, RoutingProfile]
@@ -47,9 +47,6 @@ class RoutingProfileLibrary(RootModel[RoutingProfileLibraryRoot]):
except FileNotFoundError as not_found_exc:
msg = f"Failed to load routing profile library from file '{routing_profile_library_path}': {not_found_exc}"
raise RoutingProfileLibraryNotFoundError(msg) from not_found_exc
- except TOMLValidationError as toml_validation_exc:
- msg = f"Failed to load routing profile library from file '{routing_profile_library_path}': {toml_validation_exc}"
- raise RoutingProfileLibraryError(msg) from toml_validation_exc
try:
catalog_blueprint = RoutingProfileLibraryBlueprint.model_validate(catalog_dict)
diff --git a/pipelex/cogt/models/model_manager.py b/pipelex/cogt/models/model_manager.py
index a314ac2ff..37f5dbaa2 100644
--- a/pipelex/cogt/models/model_manager.py
+++ b/pipelex/cogt/models/model_manager.py
@@ -13,9 +13,8 @@ from pipelex.cogt.model_routing.routing_profile_library import RoutingProfileLib
from pipelex.cogt.models.model_deck import ModelDeck, ModelDeckBlueprint
from pipelex.cogt.models.model_manager_abstract import ModelManagerAbstract
from pipelex.config import get_config
-from pipelex.exceptions import LibraryError
from pipelex.tools.misc.json_utils import deep_update
-from pipelex.tools.misc.toml_utils import TOMLValidationError, load_toml_from_path, validate_toml_file
+from pipelex.tools.misc.toml_utils import load_toml_from_path
class ModelManager(ModelManagerAbstract):
@@ -43,21 +42,6 @@ class ModelManager(ModelManagerAbstract):
deck_blueprint = self.load_deck_blueprint()
self.model_deck = self.build_deck(model_deck_blueprint=deck_blueprint)
- @classmethod
- def _validate_toml_files(cls) -> None:
- log.debug("LibraryManager deck TOML file formatting")
-
- # Validation of LLM deck paths
- deck_paths = get_config().cogt.inference_config.get_model_deck_paths()
- for deck_path in deck_paths:
- if os.path.exists(deck_path):
- try:
- validate_toml_file(deck_path)
- except TOMLValidationError as exc:
- msg = f"TOML validation failed for LLM deck file '{deck_path}': {exc}"
- log.error(msg)
- raise LibraryError(msg) from exc
-
@classmethod
def load_deck_blueprint(cls) -> ModelDeckBlueprint:
deck_paths = get_config().cogt.inference_config.get_model_deck_paths()
diff --git a/pipelex/core/concepts/concept.py b/pipelex/core/concepts/concept.py
index e8630ed25..b7dcc7357 100644
--- a/pipelex/core/concepts/concept.py
+++ b/pipelex/core/concepts/concept.py
@@ -80,7 +80,7 @@ class Concept(BaseModel):
except TypeError:
pass
- # Check if concept_1 has a field that is of type concept_2_class
+ # Check if concept_1 has compatible fields with concept_2
return ClassRegistryUtils.has_compatible_field(concept_1_class, concept_2_class)
return False
diff --git a/pipelex/core/interpreter.py b/pipelex/core/interpreter.py
index 383ce55e0..036e550f3 100644
--- a/pipelex/core/interpreter.py
+++ b/pipelex/core/interpreter.py
@@ -8,11 +8,7 @@ from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint
from pipelex.core.exceptions import (
PipelexConfigurationError,
)
-from pipelex.tools.misc.toml_utils import (
- clean_trailing_whitespace,
- validate_toml_content,
- validate_toml_file,
-)
+from pipelex.tools.misc.file_utils import load_text_from_path
from pipelex.types import Self
@@ -48,31 +44,15 @@ class PipelexInterpreter(BaseModel):
raise PipelexConfigurationError(msg)
return self
- @model_validator(mode="after")
- def validate_file_path(self) -> Self:
- if self.file_path:
- validate_toml_file(path=str(self.file_path))
- if self.file_content:
- validate_toml_content(content=self.file_content, file_path=str(self.file_path))
- return self
-
- def _read_and_reformat(self) -> str:
+ def _get_content(self) -> str:
"""Load PLX content from file_path or use file_content directly."""
if self.file_path:
- with open(self.file_path, encoding="utf-8") as file:
- file_content = file.read()
-
- # Clean trailing whitespace and write back if needed
- cleaned_content = clean_trailing_whitespace(file_content)
- if file_content != cleaned_content:
- with open(self.file_path, "w", encoding="utf-8") as file:
- file.write(cleaned_content)
- return cleaned_content
- return file_content
- if self.file_content is None:
+ return load_text_from_path(path=str(self.file_path))
+ elif self.file_content:
+ return self.file_content
+ else:
msg = "file_content must be provided if file_path is not provided"
raise PipelexConfigurationError(msg)
- return self.file_content
@staticmethod
def is_pipelex_file(file_path: Path) -> bool:
@@ -120,6 +100,6 @@ class PipelexInterpreter(BaseModel):
def make_pipelex_bundle_blueprint(self) -> PipelexBundleBlueprint:
"""Make a PipelexBundleBlueprint from the file_path or file_content"""
- file_content = self._read_and_reformat()
+ file_content = self._get_content()
plx_data = self._parse_plx_content(file_content)
return PipelexBundleBlueprint.model_validate(plx_data)
diff --git a/pipelex/core/pipes/pipe_abstract.py b/pipelex/core/pipes/pipe_abstract.py
index ff94528b8..5e543a183 100644
--- a/pipelex/core/pipes/pipe_abstract.py
+++ b/pipelex/core/pipes/pipe_abstract.py
@@ -25,8 +25,8 @@ class PipeAbstract(ABC, BaseModel):
output: Concept
@field_validator("code", mode="before")
- @staticmethod
- def validate_pipe_code_syntax(code: str) -> str:
+ @classmethod
+ def validate_pipe_code_syntax(cls, code: str) -> str:
PipeBlueprint.validate_pipe_code_syntax(pipe_code=code)
return code
diff --git a/pipelex/libraries/library_manager.py b/pipelex/libraries/library_manager.py
index 4c43a1706..e2e38d3e9 100644
--- a/pipelex/libraries/library_manager.py
+++ b/pipelex/libraries/library_manager.py
@@ -1,4 +1,3 @@
-import os
from pathlib import Path
from typing import ClassVar
@@ -37,7 +36,6 @@ from pipelex.libraries.library_manager_abstract import LibraryManagerAbstract
from pipelex.tools.class_registry_utils import ClassRegistryUtils
from pipelex.tools.func_registry_utils import FuncRegistryUtils
from pipelex.tools.misc.file_utils import find_files_in_dir
-from pipelex.tools.misc.toml_utils import TOMLValidationError, validate_toml_file
from pipelex.tools.runtime_manager import runtime_manager
from pipelex.tools.typing.pydantic_utils import format_pydantic_validation_error
from pipelex.types import StrEnum
@@ -85,21 +83,6 @@ class LibraryManager(LibraryManagerAbstract):
self.pipe_library.validate_with_libraries()
self.domain_library.validate_with_libraries()
- def _validate_toml_files(self):
- """Validate all TOML files used by the library manager for formatting issues."""
- log.debug("LibraryManager validating TOML file formatting")
-
- # Validation of template paths
- template_paths = self.library_config.get_templates_paths()
- for template_path in template_paths:
- if os.path.exists(template_path):
- try:
- validate_toml_file(template_path)
- except TOMLValidationError as exc:
- msg = f"PLX validation failed for template file '{template_path}': {exc}"
- log.error(msg)
- raise LibraryError(msg) from exc
-
@override
def setup(self) -> None:
self.concept_library.setup()
diff --git a/pipelex/pipe_controllers/parallel/pipe_parallel_blueprint.py b/pipelex/pipe_controllers/parallel/pipe_parallel_blueprint.py
index 9351f0814..d8174a418 100644
--- a/pipelex/pipe_controllers/parallel/pipe_parallel_blueprint.py
+++ b/pipelex/pipe_controllers/parallel/pipe_parallel_blueprint.py
@@ -17,8 +17,8 @@ class PipeParallelBlueprint(PipeBlueprint):
combined_output: str | None = None
@field_validator("combined_output", mode="before")
- @staticmethod
- def validate_combined_output(combined_output: str) -> str:
+ @classmethod
+ def validate_combined_output(cls, combined_output: str) -> str:
if combined_output:
ConceptBlueprint.validate_concept_string_or_code(concept_string_or_code=combined_output)
return combined_output
diff --git a/pipelex/pipe_operators/llm/pipe_llm_blueprint.py b/pipelex/pipe_operators/llm/pipe_llm_blueprint.py
index f58c5a59d..b8457c2e0 100644
--- a/pipelex/pipe_operators/llm/pipe_llm_blueprint.py
+++ b/pipelex/pipe_operators/llm/pipe_llm_blueprint.py
@@ -39,11 +39,11 @@ class PipeLLMBlueprint(PipeBlueprint):
multiple_output: bool | None = None
@field_validator("nb_output", mode="after")
- @staticmethod
- def validate_nb_output(value: int | None = None) -> int | None:
- if value and value < 1:
- msg = "PipeLLMBlueprint nb_output must be greater than 0"
- raise PipeDefinitionError(msg)
+ @classmethod
+ def validate_nb_output(cls, value: int | None = None) -> int | None:
+ if value and value < 2:
+ msg = "PipeLLMBlueprint nb_output must be at least 2"
+ raise PipeDefinitionError(message=msg)
return value
@model_validator(mode="after")
diff --git a/pipelex/tools/misc/toml_utils.py b/pipelex/tools/misc/toml_utils.py
index ea6b4eff8..b01ea4515 100644
--- a/pipelex/tools/misc/toml_utils.py
+++ b/pipelex/tools/misc/toml_utils.py
@@ -7,84 +7,6 @@ import toml
from pipelex.tools.misc.file_utils import path_exists
-class TOMLValidationError(Exception):
- """Raised when TOML file has formatting issues that could cause problems."""
-
-
-def validate_toml_content(content: str, file_path: str | None = None) -> None:
- """Validate TOML content for common formatting issues."""
- # lines = content.splitlines()
- # issues: list[str] = []
-
- # for line_num, line in enumerate(lines, 1):
- # # Check for trailing whitespace
- # if line.rstrip() != line:
- # trailing_chars = line[len(line.rstrip()) :]
- # trailing_repr = repr(trailing_chars)
- # issues.append(f"Line {line_num}: Trailing whitespace detected: {trailing_repr}")
-
- # # Check for trailing whitespace after triple quotes (common issue)
- # if line.strip().endswith('"""') and line != line.rstrip():
- # issues.append(f"Line {line_num}: Trailing whitespace after triple quotes - this can cause TOML parsing issues")
-
- # # Check for mixed line endings
- # has_crlf = "\r\n" in content
- # content_without_crlf = content.replace("\r\n", "")
- # has_standalone_lf = "\n" in content_without_crlf
- # if has_crlf and has_standalone_lf:
- # issues.append("Mixed line endings detected (both CRLF and LF)")
-
- # if issues:
- # error_msg = f"TOML formatting issues in '{file_path}':\n" + "\n".join(f" - {issue}" for issue in issues)
- # raise TOMLValidationError(error_msg)
-
-
-def validate_toml_file(path: str) -> None:
- """Validate TOML file for formatting issues.
-
- Args:
- path: Path to the TOML file to validate
-
- Raises:
- TOMLValidationError: If formatting issues are detected
-
- """
- with open(path, encoding="utf-8") as file:
- content = file.read()
- validate_toml_content(content, path)
-
-
-def clean_trailing_whitespace(content: str) -> str:
- # """Clean trailing whitespace from TOML content.
-
- # This function:
- # 1. Removes trailing whitespace from all lines
- # 2. Ensures exactly one empty line at EOF (two newline characters)
- # 3. If no empty line at EOF, removes trailing whitespace from last non-empty line
-
- # Args:
- # content: The TOML content to clean
-
- # Returns:
- # The cleaned TOML content with trailing whitespace removed and an empty line at EOF
-
- # """
- # # Split into lines and clean each line
- # lines = [line.rstrip() for line in content.splitlines()]
-
- # # Remove trailing empty lines
- # while lines and not lines[-1]:
- # lines.pop()
-
- # # If we have lines and the last line has trailing whitespace, remove it
- # if lines:
- # lines[-1] = lines[-1].rstrip()
-
- # # Join with newlines and ensure an empty line at EOF (two newlines)
- # return "\n".join(lines) + "\n\n"
- return content
-
-
def load_toml_from_path(path: str) -> dict[str, Any]:
"""Load TOML from path.
@@ -102,15 +24,8 @@ def load_toml_from_path(path: str) -> dict[str, Any]:
with open(path, encoding="utf-8") as file:
content = file.read()
- cleaned_content = clean_trailing_whitespace(content)
-
- # If content changed, write it back
- if content != cleaned_content:
- with open(path, "w", encoding="utf-8") as file:
- file.write(cleaned_content)
-
# Parse TOML first
- return toml.loads(cleaned_content)
+ return toml.loads(content)
except toml.TomlDecodeError as exc:
msg = f"TOML parsing error in file '{path}': {exc}"
raise toml.TomlDecodeError(msg, exc.doc, exc.pos) from exc
diff --git a/tests/test_pipelines/pipe_controllers/pipe_sequence/pipe_sequence_2.plx b/tests/test_pipelines/pipe_controllers/pipe_sequence/pipe_sequence_2.plx
index 689564787..60f3465a8 100644
--- a/tests/test_pipelines/pipe_controllers/pipe_sequence/pipe_sequence_2.plx
+++ b/tests/test_pipelines/pipe_controllers/pipe_sequence/pipe_sequence_2.plx
@@ -25,7 +25,7 @@ type = "PipeLLM"
definition = "Extract individual reviews from document"
inputs = { document = "Document" }
output = "CustomerReview"
-nb_output = 1
+multiple_output = true
llm = "llm_for_testing_gen_text"
prompt_template = """
Extract each individual customer review from this document as separate items:
diff --git a/tests/unit/pipelex/tools/misc/test_toml_utils.py b/tests/unit/pipelex/tools/misc/test_toml_utils.py
index d9ca70087..a394d6b27 100644
--- a/tests/unit/pipelex/tools/misc/test_toml_utils.py
+++ b/tests/unit/pipelex/tools/misc/test_toml_utils.py
@@ -3,10 +3,8 @@ from pathlib import Path
import pytest
from pipelex.tools.misc.toml_utils import (
- TOMLValidationError,
load_toml_from_path_if_exists,
load_toml_from_path,
- validate_toml_file,
)
@@ -46,15 +44,6 @@ definition = "Test"
toml_file = tmp_path / "trailing_space.toml"
toml_file.write_text(toml_content)
- # with pytest.raises(TOMLValidationError) as exc_info:
- # validate_toml_file(str(toml_file))
-
- # error_msg = str(exc_info.value)
- # assert "Trailing whitespace detected" in error_msg
- # assert "Line 1" in error_msg
- # assert "Line 2" in error_msg
- validate_toml_file(str(toml_file))
-
def test_validate_toml_file_trailing_space_after_triple_quotes(self, tmp_path: Path) -> None:
"""Test detection of trailing whitespace after triple quotes."""
toml_content = '''domain = "test"
@@ -69,14 +58,6 @@ Output this only: "test"
toml_file = tmp_path / "trailing_quotes.toml"
toml_file.write_text(toml_content)
- # with pytest.raises(TOMLValidationError) as exc_info:
- # validate_toml_file(str(toml_file))
-
- # error_msg = str(exc_info.value)
- # assert "Trailing whitespace after triple quotes" in error_msg
- # assert "Line 8" in error_msg
- validate_toml_file(str(toml_file))
-
@pytest.mark.skip(reason="Mixed line ending detection needs refinement - focusing on trailing whitespace detection")
def test_validate_toml_file_mixed_line_endings(self, tmp_path: Path) -> None:
"""Test detection of mixed line endings."""
@@ -86,13 +67,6 @@ Output this only: "test"
mixed_content = b'domain = "test"\r\ndefinition = "Test"\nextra = "value"\n'
toml_file.write_bytes(mixed_content)
- # with pytest.raises(TOMLValidationError) as exc_info:
- # validate_toml_file(str(toml_file))
-
- # error_msg = str(exc_info.value)
- # assert "Mixed line endings detected" in error_msg
- validate_toml_file(str(toml_file))
-
def test_load_toml_from_path_no_validation_by_default(self, tmp_path: Path) -> None:
"""Test that loading works normally without validation."""
toml_content = """domain = "test"
@@ -148,9 +122,6 @@ TestConcept = "A test concept"
toml_file = tmp_path / "valid.toml"
toml_file.write_text(toml_content)
- # Should not raise
- validate_toml_file(str(toml_file))
-
def test_validate_toml_file_multiple_validation_issues(self, tmp_path: Path) -> None:
"""Test that multiple validation issues are all reported."""
toml_content = """domain = "test"
@@ -164,17 +135,6 @@ Output: "test"
toml_file = tmp_path / "multiple_issues.toml"
toml_file.write_text(toml_content)
- # with pytest.raises(TOMLValidationError) as exc_info:
- # validate_toml_file(str(toml_file))
-
- # error_msg = str(exc_info.value)
- # # Should detect multiple lines with trailing whitespace
- # assert "Line 1" in error_msg
- # assert "Line 2" in error_msg
- # assert "Line 7" in error_msg
- # assert "Trailing whitespace after triple quotes" in error_msg
- validate_toml_file(str(toml_file))
-
def test_validate_toml_file_error_contains_file_path(self, tmp_path: Path) -> None:
"""Test that validation error includes the file path."""
toml_content = """domain = "test"
@@ -182,14 +142,6 @@ Output: "test"
toml_file = tmp_path / "path_test.toml"
toml_file.write_text(toml_content)
- # with pytest.raises(TOMLValidationError) as exc_info:
- # validate_toml_file(str(toml_file))
-
- # error_msg = str(exc_info.value)
- # assert str(toml_file) in error_msg
- # assert "TOML formatting issues" in error_msg
- validate_toml_file(str(toml_file))
-
def test_validate_toml_file_pipe_condition_real_case(self, tmp_path: Path) -> None:
"""Test the exact scenario from pipe_condition_2.toml with trailing space after triple quotes."""
@@ -222,23 +174,8 @@ Output this only: "large"
toml_file = tmp_path / "pipe_condition_real_case.toml"
toml_file.write_text(toml_content)
- # with pytest.raises(TOMLValidationError) as exc_info:
- # validate_toml_file(str(toml_file))
-
- # error_msg = str(exc_info.value)
- # assert "Trailing whitespace after triple quotes" in error_msg
- # assert "Line 26" in error_msg # The line with """ followed by space
- validate_toml_file(str(toml_file))
-
def test_validate_toml_file_actual_problematic_file(self) -> None:
"""Test validation on the actual problematic file from the codebase."""
- problematic_file = "tests/data/tools_data/problematic_test_cases.toml"
-
- # # This should catch multiple trailing whitespace issues
- # with pytest.raises(TOMLValidationError) as exc_info:
- # validate_toml_file(problematic_file)
+ # problematic_file = "tests/data/tools_data/problematic_test_cases.toml"
- # error_msg = str(exc_info.value)
- # assert "Trailing whitespace" in error_msg
- # assert problematic_file in error_msg
- validate_toml_file(problematic_file)
+ # validate_toml_file(problematic_file)