Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
Base feature to generate BaseModels from toml
Louis Choquel5392dcb31093
diff --git a/pipelex/create/structured_output_generator.py b/pipelex/create/structured_output_generator.pynew file mode 100644index 000000000..d275afa37--- /dev/null+++ b/pipelex/create/structured_output_generator.py@@ -0,0 +1,233 @@+"""Generate Pydantic BaseModel classes from TOML definitions for structured outputs."""++from __future__ import annotations++import textwrap+from typing import Any, ClassVar, Dict, List, Optional, Union++import tomlkit++from pipelex.core.stuff_content import StructuredContent+from pipelex.types import StrEnum+++class FieldType(StrEnum):+ """Supported field types for structured outputs."""+ TEXT = "text"+ NUMBER = "number"+ INTEGER = "integer"+ BOOLEAN = "boolean"+ LIST = "list"+ DICT = "dict"+++class StructuredOutputGenerator:+ """Generate Pydantic BaseModel classes from TOML structured output definitions."""++ # Map high-level type names to Python types+ TYPE_MAPPING: ClassVar[Dict[FieldType, str]] = {+ FieldType.TEXT: "str",+ FieldType.NUMBER: "float",+ FieldType.INTEGER: "int",+ FieldType.BOOLEAN: "bool",+ FieldType.LIST: "List",+ FieldType.DICT: "Dict",+ }++ def __init__(self):+ self.imports = {+ "from typing import Optional, List, Dict, Any",+ "from pipelex.core.stuff_content import StructuredContent",+ "from pydantic import Field",+ }++ def generate_from_toml(self, toml_content: str) -> str:+ """Generate Python module content from TOML structured output definitions.++ Args:+ toml_content: TOML content containing schema definitions++ Returns:+ Generated Python module content+ """+ data = tomlkit.parse(toml_content)++ if "schema" not in data:+ raise ValueError("TOML must contain a 'schema' section")++ schemas = data["schema"]+ classes: List[str] = []++ for class_name, schema_def in schemas.items(): # type: ignore[attr-defined,union-attr]+ class_code = self._generate_class(str(class_name), dict(schema_def)) # type: ignore[arg-type]+ classes.append(class_code)++ # Generate the complete module+ imports_section = "\n".join(sorted(self.imports))+ classes_section = "\n\n\n".join(classes)++ return f"{imports_section}\n\n\n{classes_section}\n"++ def _generate_class(self, class_name: str, schema_def: Dict[str, Any]) -> str:+ """Generate a single class definition.++ Args:+ class_name: Name of the class+ schema_def: Schema definition from TOML++ Returns:+ Generated class code+ """+ definition = schema_def.get("definition", f"Generated {class_name} class")+ fields = schema_def.get("fields", {})++ # Generate class header+ class_header = f'class {class_name}(StructuredContent):\n """{definition}"""\n'++ # Generate fields+ field_definitions: List[str] = []+ for field_name, field_def in fields.items():+ field_code = self._generate_field(str(field_name), field_def) # type: ignore[arg-type]+ field_definitions.append(field_code)++ if not field_definitions:+ # Empty class with just pass+ return class_header + "\n pass"++ fields_code = "\n".join(field_definitions)+ return class_header + "\n" + fields_code++ def _generate_field(self, field_name: str, field_def: Union[Dict[str, Any], str]) -> str:+ """Generate a single field definition.++ Args:+ field_name: Name of the field+ field_def: Field definition (dict or string for simple types)++ Returns:+ Generated field code+ """+ # Handle simple string definitions (just the definition text)+ if isinstance(field_def, str):+ field_def = {"type": FieldType.TEXT, "definition": field_def}++ field_type = field_def.get("type", FieldType.TEXT)+ definition = field_def.get("definition", f"{field_name} field")+ required = field_def.get("required", False)+ default_value = field_def.get("default")++ # Handle complex types+ python_type = self._get_python_type(field_type, field_def)++ # Make optional if not required+ if not required:+ python_type = f"Optional[{python_type}]"++ # Generate Field parameters+ field_params = [f'description="{definition}"']++ if required:+ if default_value is not None:+ field_params.insert(0, f'default={repr(default_value)}')+ else:+ field_params.insert(0, "...")+ else:+ if default_value is not None:+ field_params.insert(0, f'default={repr(default_value)}')+ else:+ field_params.insert(0, "default=None")++ field_call = f"Field({', '.join(field_params)})"++ return f" {field_name}: {python_type} = {field_call}"++ def _get_python_type(self, field_type: Any, field_def: Dict[str, Any]) -> str:+ """Convert high-level type to Python type annotation.++ Args:+ field_type: High-level type name or FieldType enum+ field_def: Complete field definition++ Returns:+ Python type annotation string+ """+ # Convert string to FieldType if needed+ if isinstance(field_type, str):+ try:+ field_type_enum = FieldType(field_type)+ except ValueError:+ # Unknown type, assume it's a custom type or class reference+ return field_type+ field_type = field_type_enum++ # Use match/case for type handling+ match field_type:+ case FieldType.TEXT:+ return "str"+ case FieldType.NUMBER:+ return "float"+ case FieldType.INTEGER:+ return "int"+ case FieldType.BOOLEAN:+ return "bool"+ case FieldType.LIST:+ item_type = field_def.get("item_type", "Any")+ # Recursively handle item types+ if isinstance(item_type, str):+ try:+ item_type_enum = FieldType(item_type)+ item_type = self._get_python_type(item_type_enum, {})+ except ValueError:+ # Keep as string if not a known FieldType+ pass+ return f"List[{item_type}]"+ case FieldType.DICT:+ key_type = field_def.get("key_type", "str")+ value_type = field_def.get("value_type", "Any")+ # Recursively handle key and value types+ if isinstance(key_type, str):+ try:+ key_type_enum = FieldType(key_type)+ key_type = self._get_python_type(key_type_enum, {})+ except ValueError:+ pass+ if isinstance(value_type, str):+ try:+ value_type_enum = FieldType(value_type)+ value_type = self._get_python_type(value_type_enum, {})+ except ValueError:+ pass+ return f"Dict[{key_type}, {value_type}]"+ case _:+ # Unknown FieldType, assume it's a custom type+ return str(field_type)+++def generate_structured_outputs_from_toml_file(toml_file_path: str, output_file_path: str) -> None:+ """Generate structured output Python module from TOML file.++ Args:+ toml_file_path: Path to input TOML file+ output_file_path: Path to output Python file+ """+ with open(toml_file_path, "r", encoding="utf-8") as f:+ toml_content = f.read()++ generator = StructuredOutputGenerator()+ python_code = generator.generate_from_toml(toml_content)++ with open(output_file_path, "w", encoding="utf-8") as f:+ f.write(python_code)+++def generate_structured_outputs_from_toml_string(toml_content: str) -> str:+ """Generate structured output Python code from TOML string.++ Args:+ toml_content: TOML content as string++ Returns:+ Generated Python module content+ """+ generator = StructuredOutputGenerator()+ return generator.generate_from_toml(toml_content)\ No newline at end of filediff --git a/tests/unit/pipelex/create/__init__.py b/tests/unit/pipelex/create/__init__.pynew file mode 100644index 000000000..7bb4a8e50--- /dev/null+++ b/tests/unit/pipelex/create/__init__.py@@ -0,0 +1 @@+# Init file for create tests\ No newline at end of filediff --git a/tests/unit/pipelex/create/test_structured_output_generator.py b/tests/unit/pipelex/create/test_structured_output_generator.pynew file mode 100644index 000000000..7512b94f4--- /dev/null+++ b/tests/unit/pipelex/create/test_structured_output_generator.py@@ -0,0 +1,248 @@+"""Tests for structured output generator."""++import pytest++from pipelex import pretty_print+from pipelex.create.structured_output_generator import (+ FieldType,+ StructuredOutputGenerator,+ generate_structured_outputs_from_toml_string,+)+++class TestStructuredOutputGenerator:+ def test_simple_schema_generation(self):+ """Test generation of a simple schema with basic fields."""+ toml_content = """+[schema.TestModel]+definition = "A test model"++[schema.TestModel.fields]+name = { type = "text", definition = "Name field", required = true }+age = { type = "integer", definition = "Age field" }+active = { type = "boolean", definition = "Active status", default = true }+"""++ generator = StructuredOutputGenerator()+ result = generator.generate_from_toml(toml_content)++ pretty_print(toml_content, title="Source TOML")+ pretty_print(result, title="Generated Result")++ # Check that basic structure is correct+ assert "from typing import Optional" in result+ assert "from pipelex.core.stuff_content import StructuredContent" in result+ assert "from pydantic import Field" in result+ assert "class TestModel(StructuredContent):" in result+ assert '"""A test model"""' in result+ assert 'name: str = Field(..., description="Name field")' in result+ assert 'age: Optional[int] = Field(default=None, description="Age field")' in result+ assert 'active: Optional[bool] = Field(default=True, description="Active status")' in result++ def test_simplified_field_syntax(self):+ """Test the simplified field syntax where a field is just a string definition."""+ toml_content = """+[schema.SimpleModel]+definition = "A model with simplified field syntax"++[schema.SimpleModel.fields]+dominant_feature = "The most important or dominant feature in the image"+simple_text = "Just some text"+"""++ generator = StructuredOutputGenerator()+ result = generator.generate_from_toml(toml_content)++ pretty_print(toml_content, title="Source TOML")+ pretty_print(result, title="Generated Result")++ assert 'dominant_feature: Optional[str] = Field(default=None, description="The most important or dominant feature in the image")' in result+ assert 'simple_text: Optional[str] = Field(default=None, description="Just some text")' in result++ def test_complex_types_generation(self):+ """Test generation with complex types like lists and dicts."""+ toml_content = """+[schema.ComplexModel]+definition = "A model with complex types"++[schema.ComplexModel.fields]+tags = { type = "list", item_type = "text", definition = "List of tags" }+metadata = { type = "dict", key_type = "text", value_type = "text", definition = "Metadata dictionary" }+scores = { type = "list", item_type = "number", definition = "List of scores", required = true }+"""++ generator = StructuredOutputGenerator()+ result = generator.generate_from_toml(toml_content)++ pretty_print(toml_content, title="Source TOML")+ pretty_print(result, title="Generated Result")++ assert 'tags: Optional[List[str]] = Field(default=None, description="List of tags")' in result+ assert 'metadata: Optional[Dict[str, str]] = Field(default=None, description="Metadata dictionary")' in result+ assert 'scores: List[float] = Field(..., description="List of scores")' in result++ def test_multiple_schemas(self):+ """Test generation of multiple schemas in one TOML."""+ toml_content = """+[schema.FirstModel]+definition = "First model"++[schema.FirstModel.fields]+title = { type = "text", definition = "Title", required = true }++[schema.SecondModel]+definition = "Second model"++[schema.SecondModel.fields]+content = { type = "text", definition = "Content" }+"""++ generator = StructuredOutputGenerator()+ result = generator.generate_from_toml(toml_content)++ pretty_print(toml_content, title="Source TOML")+ pretty_print(result, title="Generated Result")++ assert "class FirstModel(StructuredContent):" in result+ assert "class SecondModel(StructuredContent):" in result+ assert '"""First model"""' in result+ assert '"""Second model"""' in result++ def test_empty_schema(self):+ """Test generation of schema with no fields."""+ toml_content = """+[schema.EmptyModel]+definition = "An empty model"+"""++ generator = StructuredOutputGenerator()+ result = generator.generate_from_toml(toml_content)++ pretty_print(toml_content, title="Source TOML")+ pretty_print(result, title="Generated Result")++ assert "class EmptyModel(StructuredContent):" in result+ assert '"""An empty model"""' in result+ assert "pass" in result++ def test_required_field_syntax(self):+ """Test that fields can be marked as required."""+ toml_content = """+[schema.RequiredFieldsModel]+definition = "Model with required fields"++[schema.RequiredFieldsModel.fields]+title = { type = "text", definition = "Required title", required = true }+optional_field = { type = "text", definition = "Optional field" }+"""++ generator = StructuredOutputGenerator()+ result = generator.generate_from_toml(toml_content)++ pretty_print(toml_content, title="Source TOML")+ pretty_print(result, title="Generated Result")++ assert 'title: str = Field(..., description="Required title")' in result+ assert 'optional_field: Optional[str] = Field(default=None, description="Optional field")' in result++ def test_custom_types(self):+ """Test handling of custom/unknown types."""+ toml_content = """+[schema.CustomTypeModel]+definition = "Model with custom types"++[schema.CustomTypeModel.fields]+custom_field = { type = "CustomClass", definition = "Custom class field" }+"""++ generator = StructuredOutputGenerator()+ result = generator.generate_from_toml(toml_content)++ pretty_print(toml_content, title="Source TOML")+ pretty_print(result, title="Generated Result")++ assert 'custom_field: Optional[CustomClass] = Field(default=None, description="Custom class field")' in result++ def test_missing_schema_section_raises_error(self):+ """Test that missing schema section raises appropriate error."""+ toml_content = """+[other_section]+value = "test"+"""++ generator = StructuredOutputGenerator()++ pretty_print(toml_content, title="Source TOML (should raise error)")++ with pytest.raises(ValueError, match="TOML must contain a 'schema' section"):+ generator.generate_from_toml(toml_content)++ def test_generate_from_string_convenience_function(self):+ """Test the convenience function for generating from string."""+ toml_content = """+[schema.ConvenienceTest]+definition = "Test convenience function"++[schema.ConvenienceTest.fields]+value = { type = "text", definition = "Test value", required = true }+"""++ result = generate_structured_outputs_from_toml_string(toml_content)++ pretty_print(toml_content, title="Source TOML")+ pretty_print(result, title="Generated Result")++ assert "class ConvenienceTest(StructuredContent):" in result+ assert 'value: str = Field(..., description="Test value")' in result++ def test_field_type_enum_usage(self):+ """Test that FieldType enum values are properly handled."""+ toml_content = """+[schema.TypeMappingTest]+definition = "Test all type mappings"++[schema.TypeMappingTest.fields]+text_field = { type = "text", definition = "Text field" }+number_field = { type = "number", definition = "Number field" }+integer_field = { type = "integer", definition = "Integer field" }+boolean_field = { type = "boolean", definition = "Boolean field" }+list_field = { type = "list", item_type = "text", definition = "List field" }+dict_field = { type = "dict", key_type = "text", value_type = "integer", definition = "Dict field" }+"""++ generator = StructuredOutputGenerator()+ result = generator.generate_from_toml(toml_content)++ pretty_print(toml_content, title="Source TOML")+ pretty_print(result, title="Generated Result")++ assert "text_field: Optional[str]" in result+ assert "number_field: Optional[float]" in result+ assert "integer_field: Optional[int]" in result+ assert "boolean_field: Optional[bool]" in result+ assert "list_field: Optional[List[str]]" in result+ assert "dict_field: Optional[Dict[str, int]]" in result++ def test_mixed_field_syntax(self):+ """Test mixing simplified syntax with full field definitions."""+ toml_content = """+[schema.MixedSyntaxModel]+definition = "Model with mixed field syntax"++[schema.MixedSyntaxModel.fields]+# Simplified syntax+simple_description = "A simple text description"+# Full syntax+complex_field = { type = "integer", definition = "A complex field", required = true }+another_simple = "Another simple field"+"""++ generator = StructuredOutputGenerator()+ result = generator.generate_from_toml(toml_content)++ pretty_print(toml_content, title="Source TOML")+ pretty_print(result, title="Generated Result")++ assert 'simple_description: Optional[str] = Field(default=None, description="A simple text description")' in result+ assert 'complex_field: int = Field(..., description="A complex field")' in result+ assert 'another_simple: Optional[str] = Field(default=None, description="Another simple field")' in result