Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

Base feature to generate BaseModels from toml

Louis Choquel5392dcb31093
diff --git a/pipelex/create/structured_output_generator.py b/pipelex/create/structured_output_generator.py
new file mode 100644
index 000000000..d275afa37
--- /dev/null
+++ b/pipelex/create/structured_output_generator.py
@@ -0,0 +1,233 @@
+"""Generate Pydantic BaseModel classes from TOML definitions for structured outputs."""
+
+from __future__ import annotations
+
+import textwrap
+from typing import Any, ClassVar, Dict, List, Optional, Union
+
+import tomlkit
+
+from pipelex.core.stuff_content import StructuredContent
+from pipelex.types import StrEnum
+
+
+class FieldType(StrEnum):
+ """Supported field types for structured outputs."""
+ TEXT = "text"
+ NUMBER = "number"
+ INTEGER = "integer"
+ BOOLEAN = "boolean"
+ LIST = "list"
+ DICT = "dict"
+
+
+class StructuredOutputGenerator:
+ """Generate Pydantic BaseModel classes from TOML structured output definitions."""
+
+ # Map high-level type names to Python types
+ TYPE_MAPPING: ClassVar[Dict[FieldType, str]] = {
+ FieldType.TEXT: "str",
+ FieldType.NUMBER: "float",
+ FieldType.INTEGER: "int",
+ FieldType.BOOLEAN: "bool",
+ FieldType.LIST: "List",
+ FieldType.DICT: "Dict",
+ }
+
+ def __init__(self):
+ self.imports = {
+ "from typing import Optional, List, Dict, Any",
+ "from pipelex.core.stuff_content import StructuredContent",
+ "from pydantic import Field",
+ }
+
+ def generate_from_toml(self, toml_content: str) -> str:
+ """Generate Python module content from TOML structured output definitions.
+
+ Args:
+ toml_content: TOML content containing schema definitions
+
+ Returns:
+ Generated Python module content
+ """
+ data = tomlkit.parse(toml_content)
+
+ if "schema" not in data:
+ raise ValueError("TOML must contain a 'schema' section")
+
+ schemas = data["schema"]
+ classes: List[str] = []
+
+ for class_name, schema_def in schemas.items(): # type: ignore[attr-defined,union-attr]
+ class_code = self._generate_class(str(class_name), dict(schema_def)) # type: ignore[arg-type]
+ classes.append(class_code)
+
+ # Generate the complete module
+ imports_section = "\n".join(sorted(self.imports))
+ classes_section = "\n\n\n".join(classes)
+
+ return f"{imports_section}\n\n\n{classes_section}\n"
+
+ def _generate_class(self, class_name: str, schema_def: Dict[str, Any]) -> str:
+ """Generate a single class definition.
+
+ Args:
+ class_name: Name of the class
+ schema_def: Schema definition from TOML
+
+ Returns:
+ Generated class code
+ """
+ definition = schema_def.get("definition", f"Generated {class_name} class")
+ fields = schema_def.get("fields", {})
+
+ # Generate class header
+ class_header = f'class {class_name}(StructuredContent):\n """{definition}"""\n'
+
+ # Generate fields
+ field_definitions: List[str] = []
+ for field_name, field_def in fields.items():
+ field_code = self._generate_field(str(field_name), field_def) # type: ignore[arg-type]
+ field_definitions.append(field_code)
+
+ if not field_definitions:
+ # Empty class with just pass
+ return class_header + "\n pass"
+
+ fields_code = "\n".join(field_definitions)
+ return class_header + "\n" + fields_code
+
+ def _generate_field(self, field_name: str, field_def: Union[Dict[str, Any], str]) -> str:
+ """Generate a single field definition.
+
+ Args:
+ field_name: Name of the field
+ field_def: Field definition (dict or string for simple types)
+
+ Returns:
+ Generated field code
+ """
+ # Handle simple string definitions (just the definition text)
+ if isinstance(field_def, str):
+ field_def = {"type": FieldType.TEXT, "definition": field_def}
+
+ field_type = field_def.get("type", FieldType.TEXT)
+ definition = field_def.get("definition", f"{field_name} field")
+ required = field_def.get("required", False)
+ default_value = field_def.get("default")
+
+ # Handle complex types
+ python_type = self._get_python_type(field_type, field_def)
+
+ # Make optional if not required
+ if not required:
+ python_type = f"Optional[{python_type}]"
+
+ # Generate Field parameters
+ field_params = [f'description="{definition}"']
+
+ if required:
+ if default_value is not None:
+ field_params.insert(0, f'default={repr(default_value)}')
+ else:
+ field_params.insert(0, "...")
+ else:
+ if default_value is not None:
+ field_params.insert(0, f'default={repr(default_value)}')
+ else:
+ field_params.insert(0, "default=None")
+
+ field_call = f"Field({', '.join(field_params)})"
+
+ return f" {field_name}: {python_type} = {field_call}"
+
+ def _get_python_type(self, field_type: Any, field_def: Dict[str, Any]) -> str:
+ """Convert high-level type to Python type annotation.
+
+ Args:
+ field_type: High-level type name or FieldType enum
+ field_def: Complete field definition
+
+ Returns:
+ Python type annotation string
+ """
+ # Convert string to FieldType if needed
+ if isinstance(field_type, str):
+ try:
+ field_type_enum = FieldType(field_type)
+ except ValueError:
+ # Unknown type, assume it's a custom type or class reference
+ return field_type
+ field_type = field_type_enum
+
+ # Use match/case for type handling
+ match field_type:
+ case FieldType.TEXT:
+ return "str"
+ case FieldType.NUMBER:
+ return "float"
+ case FieldType.INTEGER:
+ return "int"
+ case FieldType.BOOLEAN:
+ return "bool"
+ case FieldType.LIST:
+ item_type = field_def.get("item_type", "Any")
+ # Recursively handle item types
+ if isinstance(item_type, str):
+ try:
+ item_type_enum = FieldType(item_type)
+ item_type = self._get_python_type(item_type_enum, {})
+ except ValueError:
+ # Keep as string if not a known FieldType
+ pass
+ return f"List[{item_type}]"
+ case FieldType.DICT:
+ key_type = field_def.get("key_type", "str")
+ value_type = field_def.get("value_type", "Any")
+ # Recursively handle key and value types
+ if isinstance(key_type, str):
+ try:
+ key_type_enum = FieldType(key_type)
+ key_type = self._get_python_type(key_type_enum, {})
+ except ValueError:
+ pass
+ if isinstance(value_type, str):
+ try:
+ value_type_enum = FieldType(value_type)
+ value_type = self._get_python_type(value_type_enum, {})
+ except ValueError:
+ pass
+ return f"Dict[{key_type}, {value_type}]"
+ case _:
+ # Unknown FieldType, assume it's a custom type
+ return str(field_type)
+
+
+def generate_structured_outputs_from_toml_file(toml_file_path: str, output_file_path: str) -> None:
+ """Generate structured output Python module from TOML file.
+
+ Args:
+ toml_file_path: Path to input TOML file
+ output_file_path: Path to output Python file
+ """
+ with open(toml_file_path, "r", encoding="utf-8") as f:
+ toml_content = f.read()
+
+ generator = StructuredOutputGenerator()
+ python_code = generator.generate_from_toml(toml_content)
+
+ with open(output_file_path, "w", encoding="utf-8") as f:
+ f.write(python_code)
+
+
+def generate_structured_outputs_from_toml_string(toml_content: str) -> str:
+ """Generate structured output Python code from TOML string.
+
+ Args:
+ toml_content: TOML content as string
+
+ Returns:
+ Generated Python module content
+ """
+ generator = StructuredOutputGenerator()
+ return generator.generate_from_toml(toml_content)
\ No newline at end of file
diff --git a/tests/unit/pipelex/create/__init__.py b/tests/unit/pipelex/create/__init__.py
new file mode 100644
index 000000000..7bb4a8e50
--- /dev/null
+++ b/tests/unit/pipelex/create/__init__.py
@@ -0,0 +1 @@
+# Init file for create tests
\ No newline at end of file
diff --git a/tests/unit/pipelex/create/test_structured_output_generator.py b/tests/unit/pipelex/create/test_structured_output_generator.py
new file mode 100644
index 000000000..7512b94f4
--- /dev/null
+++ b/tests/unit/pipelex/create/test_structured_output_generator.py
@@ -0,0 +1,248 @@
+"""Tests for structured output generator."""
+
+import pytest
+
+from pipelex import pretty_print
+from pipelex.create.structured_output_generator import (
+ FieldType,
+ StructuredOutputGenerator,
+ generate_structured_outputs_from_toml_string,
+)
+
+
+class TestStructuredOutputGenerator:
+ def test_simple_schema_generation(self):
+ """Test generation of a simple schema with basic fields."""
+ toml_content = """
+[schema.TestModel]
+definition = "A test model"
+
+[schema.TestModel.fields]
+name = { type = "text", definition = "Name field", required = true }
+age = { type = "integer", definition = "Age field" }
+active = { type = "boolean", definition = "Active status", default = true }
+"""
+
+ generator = StructuredOutputGenerator()
+ result = generator.generate_from_toml(toml_content)
+
+ pretty_print(toml_content, title="Source TOML")
+ pretty_print(result, title="Generated Result")
+
+ # Check that basic structure is correct
+ assert "from typing import Optional" in result
+ assert "from pipelex.core.stuff_content import StructuredContent" in result
+ assert "from pydantic import Field" in result
+ assert "class TestModel(StructuredContent):" in result
+ assert '"""A test model"""' in result
+ assert 'name: str = Field(..., description="Name field")' in result
+ assert 'age: Optional[int] = Field(default=None, description="Age field")' in result
+ assert 'active: Optional[bool] = Field(default=True, description="Active status")' in result
+
+ def test_simplified_field_syntax(self):
+ """Test the simplified field syntax where a field is just a string definition."""
+ toml_content = """
+[schema.SimpleModel]
+definition = "A model with simplified field syntax"
+
+[schema.SimpleModel.fields]
+dominant_feature = "The most important or dominant feature in the image"
+simple_text = "Just some text"
+"""
+
+ generator = StructuredOutputGenerator()
+ result = generator.generate_from_toml(toml_content)
+
+ pretty_print(toml_content, title="Source TOML")
+ pretty_print(result, title="Generated Result")
+
+ assert 'dominant_feature: Optional[str] = Field(default=None, description="The most important or dominant feature in the image")' in result
+ assert 'simple_text: Optional[str] = Field(default=None, description="Just some text")' in result
+
+ def test_complex_types_generation(self):
+ """Test generation with complex types like lists and dicts."""
+ toml_content = """
+[schema.ComplexModel]
+definition = "A model with complex types"
+
+[schema.ComplexModel.fields]
+tags = { type = "list", item_type = "text", definition = "List of tags" }
+metadata = { type = "dict", key_type = "text", value_type = "text", definition = "Metadata dictionary" }
+scores = { type = "list", item_type = "number", definition = "List of scores", required = true }
+"""
+
+ generator = StructuredOutputGenerator()
+ result = generator.generate_from_toml(toml_content)
+
+ pretty_print(toml_content, title="Source TOML")
+ pretty_print(result, title="Generated Result")
+
+ assert 'tags: Optional[List[str]] = Field(default=None, description="List of tags")' in result
+ assert 'metadata: Optional[Dict[str, str]] = Field(default=None, description="Metadata dictionary")' in result
+ assert 'scores: List[float] = Field(..., description="List of scores")' in result
+
+ def test_multiple_schemas(self):
+ """Test generation of multiple schemas in one TOML."""
+ toml_content = """
+[schema.FirstModel]
+definition = "First model"
+
+[schema.FirstModel.fields]
+title = { type = "text", definition = "Title", required = true }
+
+[schema.SecondModel]
+definition = "Second model"
+
+[schema.SecondModel.fields]
+content = { type = "text", definition = "Content" }
+"""
+
+ generator = StructuredOutputGenerator()
+ result = generator.generate_from_toml(toml_content)
+
+ pretty_print(toml_content, title="Source TOML")
+ pretty_print(result, title="Generated Result")
+
+ assert "class FirstModel(StructuredContent):" in result
+ assert "class SecondModel(StructuredContent):" in result
+ assert '"""First model"""' in result
+ assert '"""Second model"""' in result
+
+ def test_empty_schema(self):
+ """Test generation of schema with no fields."""
+ toml_content = """
+[schema.EmptyModel]
+definition = "An empty model"
+"""
+
+ generator = StructuredOutputGenerator()
+ result = generator.generate_from_toml(toml_content)
+
+ pretty_print(toml_content, title="Source TOML")
+ pretty_print(result, title="Generated Result")
+
+ assert "class EmptyModel(StructuredContent):" in result
+ assert '"""An empty model"""' in result
+ assert "pass" in result
+
+ def test_required_field_syntax(self):
+ """Test that fields can be marked as required."""
+ toml_content = """
+[schema.RequiredFieldsModel]
+definition = "Model with required fields"
+
+[schema.RequiredFieldsModel.fields]
+title = { type = "text", definition = "Required title", required = true }
+optional_field = { type = "text", definition = "Optional field" }
+"""
+
+ generator = StructuredOutputGenerator()
+ result = generator.generate_from_toml(toml_content)
+
+ pretty_print(toml_content, title="Source TOML")
+ pretty_print(result, title="Generated Result")
+
+ assert 'title: str = Field(..., description="Required title")' in result
+ assert 'optional_field: Optional[str] = Field(default=None, description="Optional field")' in result
+
+ def test_custom_types(self):
+ """Test handling of custom/unknown types."""
+ toml_content = """
+[schema.CustomTypeModel]
+definition = "Model with custom types"
+
+[schema.CustomTypeModel.fields]
+custom_field = { type = "CustomClass", definition = "Custom class field" }
+"""
+
+ generator = StructuredOutputGenerator()
+ result = generator.generate_from_toml(toml_content)
+
+ pretty_print(toml_content, title="Source TOML")
+ pretty_print(result, title="Generated Result")
+
+ assert 'custom_field: Optional[CustomClass] = Field(default=None, description="Custom class field")' in result
+
+ def test_missing_schema_section_raises_error(self):
+ """Test that missing schema section raises appropriate error."""
+ toml_content = """
+[other_section]
+value = "test"
+"""
+
+ generator = StructuredOutputGenerator()
+
+ pretty_print(toml_content, title="Source TOML (should raise error)")
+
+ with pytest.raises(ValueError, match="TOML must contain a 'schema' section"):
+ generator.generate_from_toml(toml_content)
+
+ def test_generate_from_string_convenience_function(self):
+ """Test the convenience function for generating from string."""
+ toml_content = """
+[schema.ConvenienceTest]
+definition = "Test convenience function"
+
+[schema.ConvenienceTest.fields]
+value = { type = "text", definition = "Test value", required = true }
+"""
+
+ result = generate_structured_outputs_from_toml_string(toml_content)
+
+ pretty_print(toml_content, title="Source TOML")
+ pretty_print(result, title="Generated Result")
+
+ assert "class ConvenienceTest(StructuredContent):" in result
+ assert 'value: str = Field(..., description="Test value")' in result
+
+ def test_field_type_enum_usage(self):
+ """Test that FieldType enum values are properly handled."""
+ toml_content = """
+[schema.TypeMappingTest]
+definition = "Test all type mappings"
+
+[schema.TypeMappingTest.fields]
+text_field = { type = "text", definition = "Text field" }
+number_field = { type = "number", definition = "Number field" }
+integer_field = { type = "integer", definition = "Integer field" }
+boolean_field = { type = "boolean", definition = "Boolean field" }
+list_field = { type = "list", item_type = "text", definition = "List field" }
+dict_field = { type = "dict", key_type = "text", value_type = "integer", definition = "Dict field" }
+"""
+
+ generator = StructuredOutputGenerator()
+ result = generator.generate_from_toml(toml_content)
+
+ pretty_print(toml_content, title="Source TOML")
+ pretty_print(result, title="Generated Result")
+
+ assert "text_field: Optional[str]" in result
+ assert "number_field: Optional[float]" in result
+ assert "integer_field: Optional[int]" in result
+ assert "boolean_field: Optional[bool]" in result
+ assert "list_field: Optional[List[str]]" in result
+ assert "dict_field: Optional[Dict[str, int]]" in result
+
+ def test_mixed_field_syntax(self):
+ """Test mixing simplified syntax with full field definitions."""
+ toml_content = """
+[schema.MixedSyntaxModel]
+definition = "Model with mixed field syntax"
+
+[schema.MixedSyntaxModel.fields]
+# Simplified syntax
+simple_description = "A simple text description"
+# Full syntax
+complex_field = { type = "integer", definition = "A complex field", required = true }
+another_simple = "Another simple field"
+"""
+
+ generator = StructuredOutputGenerator()
+ result = generator.generate_from_toml(toml_content)
+
+ pretty_print(toml_content, title="Source TOML")
+ pretty_print(result, title="Generated Result")
+
+ assert 'simple_description: Optional[str] = Field(default=None, description="A simple text description")' in result
+ assert 'complex_field: int = Field(..., description="A complex field")' in result
+ assert 'another_simple: Optional[str] = Field(default=None, description="Another simple field")' in result