Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
Schema generation with enums too
Louis Choquel8898481c28d8
diff --git a/pipelex/create/structured_output_generator.py b/pipelex/create/structured_output_generator.pyindex d275afa37..cff3738af 100644--- a/pipelex/create/structured_output_generator.py+++ b/pipelex/create/structured_output_generator.py@@ -13,17 +13,19 @@ from pipelex.types import StrEnumclass FieldType(StrEnum):"""Supported field types for structured outputs."""+TEXT = "text"NUMBER = "number"INTEGER = "integer"BOOLEAN = "boolean"LIST = "list"DICT = "dict"+ ENUM = "enum" # For referencing defined enumsclass StructuredOutputGenerator:"""Generate Pydantic BaseModel classes from TOML structured output definitions."""-+# Map high-level type names to Python typesTYPE_MAPPING: ClassVar[Dict[FieldType, str]] = {FieldType.TEXT: "str",@@ -36,121 +38,192 @@ class StructuredOutputGenerator:def __init__(self):self.imports = {- "from typing import Optional, List, Dict, Any",+ "from typing import Optional, List, Dict, Any, Literal",+ "from enum import Enum","from pipelex.core.stuff_content import StructuredContent","from pydantic import Field",}+ self.enum_definitions: Dict[str, Dict[str, Any]] = {} # Store enum definitionsdef generate_from_toml(self, toml_content: str) -> str:"""Generate Python module content from TOML structured output definitions.-+Args:- toml_content: TOML content containing schema definitions-+ toml_content: TOML content containing schema and enum definitions+Returns:Generated Python module content"""data = tomlkit.parse(toml_content)-++ # Process enums first (if any)+ enums: List[str] = []+ if "enum" in data:+ enum_defs = data["enum"]+ for enum_name, enum_def in enum_defs.items(): # type: ignore[attr-defined,union-attr]+ self.enum_definitions[str(enum_name)] = dict(enum_def) # type: ignore[arg-type]+ enum_code = self._generate_enum(str(enum_name), dict(enum_def)) # type: ignore[arg-type]+ enums.append(enum_code)++ # Process schemasif "schema" not in data:raise ValueError("TOML must contain a 'schema' section")-+schemas = data["schema"]classes: List[str] = []-+for class_name, schema_def in schemas.items(): # type: ignore[attr-defined,union-attr]class_code = self._generate_class(str(class_name), dict(schema_def)) # type: ignore[arg-type]classes.append(class_code)-+# Generate the complete moduleimports_section = "\n".join(sorted(self.imports))- classes_section = "\n\n\n".join(classes)-- return f"{imports_section}\n\n\n{classes_section}\n"++ # Combine enums and classes+ all_definitions: List[str] = []+ if enums:+ all_definitions.extend(enums)+ if classes:+ all_definitions.extend(classes)++ definitions_section = "\n\n\n".join(all_definitions)++ return f"{imports_section}\n\n\n{definitions_section}\n"++ def _generate_enum(self, enum_name: str, enum_def: Dict[str, Any]) -> str:+ """Generate an enum class definition.++ Args:+ enum_name: Name of the enum+ enum_def: Enum definition from TOML++ Returns:+ Generated enum class code+ """+ definition = enum_def.get("definition", f"Generated {enum_name} enum")+ values: List[str] | Dict[str, str] = enum_def.get("values", [])++ # Generate enum header+ enum_header = f'class {enum_name}(str, Enum):\n """{definition}"""\n'++ # Generate enum values+ value_definitions: List[str] = []++ if isinstance(values, list):+ # Simple list of values+ for value in values:+ # Convert to uppercase for enum member name+ value_str = str(value)+ member_name = value_str.upper().replace(" ", "_").replace("-", "_")+ value_definitions.append(f' {member_name} = "{value_str}"')+ else:+ # Key-value pairs with descriptions+ for key, description in values.items():+ key_str = str(key)+ desc_str = str(description)+ member_name = key_str.upper().replace(" ", "_").replace("-", "_")+ value_definitions.append(f' {member_name} = "{key_str}" # {desc_str}')++ if not value_definitions:+ # Empty enum with just pass+ return enum_header + "\n pass"++ values_code = "\n".join(value_definitions)+ return enum_header + "\n" + values_codedef _generate_class(self, class_name: str, schema_def: Dict[str, Any]) -> str:"""Generate a single class definition.-+Args:class_name: Name of the classschema_def: Schema definition from TOML-+Returns:Generated class code"""definition = schema_def.get("definition", f"Generated {class_name} class")fields = schema_def.get("fields", {})-+# Generate class headerclass_header = f'class {class_name}(StructuredContent):\n """{definition}"""\n'-+# Generate fieldsfield_definitions: List[str] = []for field_name, field_def in fields.items():field_code = self._generate_field(str(field_name), field_def) # type: ignore[arg-type]field_definitions.append(field_code)-+if not field_definitions:# Empty class with just passreturn class_header + "\n pass"-+fields_code = "\n".join(field_definitions)return class_header + "\n" + fields_codedef _generate_field(self, field_name: str, field_def: Union[Dict[str, Any], str]) -> str:"""Generate a single field definition.-+Args:field_name: Name of the fieldfield_def: Field definition (dict or string for simple types)-+Returns:Generated field code"""# Handle simple string definitions (just the definition text)if isinstance(field_def, str):field_def = {"type": FieldType.TEXT, "definition": field_def}-+field_type = field_def.get("type", FieldType.TEXT)definition = field_def.get("definition", f"{field_name} field")required = field_def.get("required", False)default_value = field_def.get("default")-- # Handle complex types- python_type = self._get_python_type(field_type, field_def)-+ choices = field_def.get("choices") # For inline enum-like choices++ # Determine Python type+ if choices:+ # Inline choices - use Literal type+ python_type = f"Literal[{', '.join(repr(c) for c in choices)}]"+ else:+ # Handle complex types or enum references+ python_type = self._get_python_type(field_type, field_def)+# Make optional if not requiredif not required:python_type = f"Optional[{python_type}]"-+# Generate Field parametersfield_params = [f'description="{definition}"']-+if required:if default_value is not None:- field_params.insert(0, f'default={repr(default_value)}')+ field_params.insert(0, f"default={repr(default_value)}")else:field_params.insert(0, "...")else:if default_value is not None:- field_params.insert(0, f'default={repr(default_value)}')+ field_params.insert(0, f"default={repr(default_value)}")else:field_params.insert(0, "default=None")-+field_call = f"Field({', '.join(field_params)})"-+return f" {field_name}: {python_type} = {field_call}"def _get_python_type(self, field_type: Any, field_def: Dict[str, Any]) -> str:"""Convert high-level type to Python type annotation.-+Args:field_type: High-level type name or FieldType enumfield_def: Complete field definition-+Returns:Python type annotation string"""+ # Check if it's a reference to a defined enum+ if isinstance(field_type, str) and field_type in self.enum_definitions:+ return field_type+# Convert string to FieldType if neededif isinstance(field_type, str):try:@@ -159,7 +232,7 @@ class StructuredOutputGenerator:# Unknown type, assume it's a custom type or class referencereturn field_typefield_type = field_type_enum-+# Use match/case for type handlingmatch field_type:case FieldType.TEXT:@@ -172,6 +245,9 @@ class StructuredOutputGenerator:return "bool"case FieldType.LIST:item_type = field_def.get("item_type", "Any")+ # Check if item_type is an enum reference+ if isinstance(item_type, str) and item_type in self.enum_definitions:+ return f"List[{item_type}]"# Recursively handle item typesif isinstance(item_type, str):try:@@ -205,29 +281,29 @@ class StructuredOutputGenerator:def generate_structured_outputs_from_toml_file(toml_file_path: str, output_file_path: str) -> None:"""Generate structured output Python module from TOML file.-+Args:toml_file_path: Path to input TOML fileoutput_file_path: Path to output Python file"""with open(toml_file_path, "r", encoding="utf-8") as f:toml_content = f.read()-+generator = StructuredOutputGenerator()python_code = generator.generate_from_toml(toml_content)-+with open(output_file_path, "w", encoding="utf-8") as f:f.write(python_code)def generate_structured_outputs_from_toml_string(toml_content: str) -> str:"""Generate structured output Python code from TOML string.-+Args:toml_content: TOML content as string-+Returns:Generated Python module content"""generator = StructuredOutputGenerator()- return generator.generate_from_toml(toml_content)\ No newline at end of file+ return generator.generate_from_toml(toml_content)diff --git a/tests/unit/pipelex/create/__init__.py b/tests/unit/pipelex/create/__init__.pydeleted file mode 100644index 7bb4a8e50..000000000--- a/tests/unit/pipelex/create/__init__.py+++ /dev/null@@ -1 +0,0 @@-# Init file for create tests\ No newline at end of filediff --git a/tests/unit/pipelex/create/test_structured_output_generator.py b/tests/unit/pipelex/create/test_structured_output_generator.pyindex 7512b94f4..6cdbc7678 100644--- a/tests/unit/pipelex/create/test_structured_output_generator.py+++ b/tests/unit/pipelex/create/test_structured_output_generator.py@@ -246,3 +246,172 @@ another_simple = "Another simple field"assert 'simple_description: Optional[str] = Field(default=None, description="A simple text description")' in resultassert 'complex_field: int = Field(..., description="A complex field")' in resultassert 'another_simple: Optional[str] = Field(default=None, description="Another simple field")' in result++ def test_enum_generation_simple_list(self):+ """Test generation of enum with simple list of values."""+ toml_content = """+[enum.OrderStatus]+definition = "Possible states of an order"+values = ["pending", "processing", "shipped", "delivered", "cancelled"]++[schema.Order]+definition = "An order in the system"++[schema.Order.fields]+order_id = "Unique identifier for the order"+status = { type = "OrderStatus", definition = "Current status of the order", required = true }+customer_name = "Name of the customer"+"""++ generator = StructuredOutputGenerator()+ result = generator.generate_from_toml(toml_content)++ pretty_print(toml_content, title="Source TOML")+ pretty_print(result, title="Generated Result")++ # Check enum generation+ assert "from enum import Enum" in result+ assert "class OrderStatus(str, Enum):" in result+ assert '"""Possible states of an order"""' in result+ assert 'PENDING = "pending"' in result+ assert 'PROCESSING = "processing"' in result+ assert 'SHIPPED = "shipped"' in result+ assert 'DELIVERED = "delivered"' in result+ assert 'CANCELLED = "cancelled"' in result++ # Check schema using the enum+ assert "class Order(StructuredContent):" in result+ assert 'status: OrderStatus = Field(..., description="Current status of the order")' in result++ def test_enum_generation_with_descriptions(self):+ """Test generation of enum with key-value pairs (descriptions)."""+ toml_content = """+[enum.Priority]+definition = "Task priority levels"++[enum.Priority.values]+low = "Low Priority"+medium = "Medium Priority"+high = "High Priority"+urgent = "Urgent - Handle Immediately"++[schema.Task]+definition = "A task to be completed"++[schema.Task.fields]+title = "Task title"+priority = { type = "Priority", definition = "How urgent is this task", required = true }+assigned_to = "Person responsible for the task"+"""++ generator = StructuredOutputGenerator()+ result = generator.generate_from_toml(toml_content)++ pretty_print(toml_content, title="Source TOML")+ pretty_print(result, title="Generated Result")++ # Check enum generation with descriptions+ assert "class Priority(str, Enum):" in result+ assert '"""Task priority levels"""' in result+ assert 'LOW = "low" # Low Priority' in result+ assert 'MEDIUM = "medium" # Medium Priority' in result+ assert 'HIGH = "high" # High Priority' in result+ assert 'URGENT = "urgent" # Urgent - Handle Immediately' in result++ # Check schema using the enum+ assert 'priority: Priority = Field(..., description="How urgent is this task")' in result++ def test_inline_choices(self):+ """Test generation with inline choices (Literal type)."""+ toml_content = """+[schema.Product]+definition = "A product in our catalog"++[schema.Product.fields]+name = "Product name"+category = { choices = ["electronics", "clothing", "food", "books"], definition = "Product category", required = true }+size = { choices = ["XS", "S", "M", "L", "XL"], definition = "Size of the product" }+"""++ generator = StructuredOutputGenerator()+ result = generator.generate_from_toml(toml_content)++ pretty_print(toml_content, title="Source TOML")+ pretty_print(result, title="Generated Result")++ # Check Literal type usage+ assert "from typing import Optional, List, Dict, Any, Literal" in result+ assert "category: Literal['electronics', 'clothing', 'food', 'books'] = Field(..., description=\"Product category\")" in result+ assert "size: Optional[Literal['XS', 'S', 'M', 'L', 'XL']] = Field(default=None, description=\"Size of the product\")" in result++ def test_mixed_enums_and_inline_choices(self):+ """Test combining enum references and inline choices in the same schema."""+ toml_content = """+[enum.Color]+definition = "Available colors"+values = ["red", "blue", "green", "yellow", "black", "white"]++[enum.ShippingMethod]+definition = "How the product will be shipped"+values = ["standard", "express", "overnight", "pickup"]++[schema.ProductVariant]+definition = "A specific variant of a product"++[schema.ProductVariant.fields]+sku = "Stock keeping unit"+color = { type = "Color", definition = "Color of this variant" }+size = { choices = ["S", "M", "L"], definition = "Size options" }+shipping_method = { type = "ShippingMethod", definition = "Shipping option", required = true }+availability = { choices = ["in_stock", "out_of_stock", "pre_order"], definition = "Current availability" }+"""++ generator = StructuredOutputGenerator()+ result = generator.generate_from_toml(toml_content)++ pretty_print(toml_content, title="Source TOML")+ pretty_print(result, title="Generated Result")++ # Check enums are generated+ assert "class Color(str, Enum):" in result+ assert "class ShippingMethod(str, Enum):" in result++ # Check mixed usage in schema+ assert 'color: Optional[Color] = Field(default=None, description="Color of this variant")' in result+ assert "size: Optional[Literal['S', 'M', 'L']] = Field(default=None, description=\"Size options\")" in result+ assert 'shipping_method: ShippingMethod = Field(..., description="Shipping option")' in result+ assert (+ "availability: Optional[Literal['in_stock', 'out_of_stock', 'pre_order']] = Field(default=None, description=\"Current availability\")"+ in result+ )++ def test_enum_in_list_field(self):+ """Test using enum types in list fields."""+ toml_content = """+[enum.DietaryRestriction]+definition = "Dietary restrictions and preferences"+values = ["vegetarian", "vegan", "gluten_free", "dairy_free", "nut_free"]++[schema.MenuItem]+definition = "An item on the restaurant menu"++[schema.MenuItem.fields]+name = "Name of the dish"+dietary_info = { type = "list", item_type = "DietaryRestriction", definition = "Dietary accommodations" }+tags = { type = "list", item_type = "text", definition = "General tags" }+"""++ generator = StructuredOutputGenerator()+ result = generator.generate_from_toml(toml_content)++ pretty_print(toml_content, title="Source TOML")+ pretty_print(result, title="Generated Result")++ # Check enum is generated+ assert "class DietaryRestriction(str, Enum):" in result+ assert 'VEGETARIAN = "vegetarian"' in result+ assert 'GLUTEN_FREE = "gluten_free"' in result++ # Check list with enum type+ assert 'dietary_info: Optional[List[DietaryRestriction]] = Field(default=None, description="Dietary accommodations")' in result+ assert 'tags: Optional[List[str]] = Field(default=None, description="General tags")' in result