Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

Schema generation with enums too

Louis Choquel8898481c28d8
diff --git a/pipelex/create/structured_output_generator.py b/pipelex/create/structured_output_generator.py
index d275afa37..cff3738af 100644
--- a/pipelex/create/structured_output_generator.py
+++ b/pipelex/create/structured_output_generator.py
@@ -13,17 +13,19 @@ from pipelex.types import StrEnum
class FieldType(StrEnum):
"""Supported field types for structured outputs."""
+
TEXT = "text"
NUMBER = "number"
INTEGER = "integer"
BOOLEAN = "boolean"
LIST = "list"
DICT = "dict"
+ ENUM = "enum" # For referencing defined enums
class StructuredOutputGenerator:
"""Generate Pydantic BaseModel classes from TOML structured output definitions."""
-
+
# Map high-level type names to Python types
TYPE_MAPPING: ClassVar[Dict[FieldType, str]] = {
FieldType.TEXT: "str",
@@ -36,121 +38,192 @@ class StructuredOutputGenerator:
def __init__(self):
self.imports = {
- "from typing import Optional, List, Dict, Any",
+ "from typing import Optional, List, Dict, Any, Literal",
+ "from enum import Enum",
"from pipelex.core.stuff_content import StructuredContent",
"from pydantic import Field",
}
+ self.enum_definitions: Dict[str, Dict[str, Any]] = {} # Store enum definitions
def generate_from_toml(self, toml_content: str) -> str:
"""Generate Python module content from TOML structured output definitions.
-
+
Args:
- toml_content: TOML content containing schema definitions
-
+ toml_content: TOML content containing schema and enum definitions
+
Returns:
Generated Python module content
"""
data = tomlkit.parse(toml_content)
-
+
+ # Process enums first (if any)
+ enums: List[str] = []
+ if "enum" in data:
+ enum_defs = data["enum"]
+ for enum_name, enum_def in enum_defs.items(): # type: ignore[attr-defined,union-attr]
+ self.enum_definitions[str(enum_name)] = dict(enum_def) # type: ignore[arg-type]
+ enum_code = self._generate_enum(str(enum_name), dict(enum_def)) # type: ignore[arg-type]
+ enums.append(enum_code)
+
+ # Process schemas
if "schema" not in data:
raise ValueError("TOML must contain a 'schema' section")
-
+
schemas = data["schema"]
classes: List[str] = []
-
+
for class_name, schema_def in schemas.items(): # type: ignore[attr-defined,union-attr]
class_code = self._generate_class(str(class_name), dict(schema_def)) # type: ignore[arg-type]
classes.append(class_code)
-
+
# Generate the complete module
imports_section = "\n".join(sorted(self.imports))
- classes_section = "\n\n\n".join(classes)
-
- return f"{imports_section}\n\n\n{classes_section}\n"
+
+ # Combine enums and classes
+ all_definitions: List[str] = []
+ if enums:
+ all_definitions.extend(enums)
+ if classes:
+ all_definitions.extend(classes)
+
+ definitions_section = "\n\n\n".join(all_definitions)
+
+ return f"{imports_section}\n\n\n{definitions_section}\n"
+
+ def _generate_enum(self, enum_name: str, enum_def: Dict[str, Any]) -> str:
+ """Generate an enum class definition.
+
+ Args:
+ enum_name: Name of the enum
+ enum_def: Enum definition from TOML
+
+ Returns:
+ Generated enum class code
+ """
+ definition = enum_def.get("definition", f"Generated {enum_name} enum")
+ values: List[str] | Dict[str, str] = enum_def.get("values", [])
+
+ # Generate enum header
+ enum_header = f'class {enum_name}(str, Enum):\n """{definition}"""\n'
+
+ # Generate enum values
+ value_definitions: List[str] = []
+
+ if isinstance(values, list):
+ # Simple list of values
+ for value in values:
+ # Convert to uppercase for enum member name
+ value_str = str(value)
+ member_name = value_str.upper().replace(" ", "_").replace("-", "_")
+ value_definitions.append(f' {member_name} = "{value_str}"')
+ else:
+ # Key-value pairs with descriptions
+ for key, description in values.items():
+ key_str = str(key)
+ desc_str = str(description)
+ member_name = key_str.upper().replace(" ", "_").replace("-", "_")
+ value_definitions.append(f' {member_name} = "{key_str}" # {desc_str}')
+
+ if not value_definitions:
+ # Empty enum with just pass
+ return enum_header + "\n pass"
+
+ values_code = "\n".join(value_definitions)
+ return enum_header + "\n" + values_code
def _generate_class(self, class_name: str, schema_def: Dict[str, Any]) -> str:
"""Generate a single class definition.
-
+
Args:
class_name: Name of the class
schema_def: Schema definition from TOML
-
+
Returns:
Generated class code
"""
definition = schema_def.get("definition", f"Generated {class_name} class")
fields = schema_def.get("fields", {})
-
+
# Generate class header
class_header = f'class {class_name}(StructuredContent):\n """{definition}"""\n'
-
+
# Generate fields
field_definitions: List[str] = []
for field_name, field_def in fields.items():
field_code = self._generate_field(str(field_name), field_def) # type: ignore[arg-type]
field_definitions.append(field_code)
-
+
if not field_definitions:
# Empty class with just pass
return class_header + "\n pass"
-
+
fields_code = "\n".join(field_definitions)
return class_header + "\n" + fields_code
def _generate_field(self, field_name: str, field_def: Union[Dict[str, Any], str]) -> str:
"""Generate a single field definition.
-
+
Args:
field_name: Name of the field
field_def: Field definition (dict or string for simple types)
-
+
Returns:
Generated field code
"""
# Handle simple string definitions (just the definition text)
if isinstance(field_def, str):
field_def = {"type": FieldType.TEXT, "definition": field_def}
-
+
field_type = field_def.get("type", FieldType.TEXT)
definition = field_def.get("definition", f"{field_name} field")
required = field_def.get("required", False)
default_value = field_def.get("default")
-
- # Handle complex types
- python_type = self._get_python_type(field_type, field_def)
-
+ choices = field_def.get("choices") # For inline enum-like choices
+
+ # Determine Python type
+ if choices:
+ # Inline choices - use Literal type
+ python_type = f"Literal[{', '.join(repr(c) for c in choices)}]"
+ else:
+ # Handle complex types or enum references
+ python_type = self._get_python_type(field_type, field_def)
+
# Make optional if not required
if not required:
python_type = f"Optional[{python_type}]"
-
+
# Generate Field parameters
field_params = [f'description="{definition}"']
-
+
if required:
if default_value is not None:
- field_params.insert(0, f'default={repr(default_value)}')
+ field_params.insert(0, f"default={repr(default_value)}")
else:
field_params.insert(0, "...")
else:
if default_value is not None:
- field_params.insert(0, f'default={repr(default_value)}')
+ field_params.insert(0, f"default={repr(default_value)}")
else:
field_params.insert(0, "default=None")
-
+
field_call = f"Field({', '.join(field_params)})"
-
+
return f" {field_name}: {python_type} = {field_call}"
def _get_python_type(self, field_type: Any, field_def: Dict[str, Any]) -> str:
"""Convert high-level type to Python type annotation.
-
+
Args:
field_type: High-level type name or FieldType enum
field_def: Complete field definition
-
+
Returns:
Python type annotation string
"""
+ # Check if it's a reference to a defined enum
+ if isinstance(field_type, str) and field_type in self.enum_definitions:
+ return field_type
+
# Convert string to FieldType if needed
if isinstance(field_type, str):
try:
@@ -159,7 +232,7 @@ class StructuredOutputGenerator:
# Unknown type, assume it's a custom type or class reference
return field_type
field_type = field_type_enum
-
+
# Use match/case for type handling
match field_type:
case FieldType.TEXT:
@@ -172,6 +245,9 @@ class StructuredOutputGenerator:
return "bool"
case FieldType.LIST:
item_type = field_def.get("item_type", "Any")
+ # Check if item_type is an enum reference
+ if isinstance(item_type, str) and item_type in self.enum_definitions:
+ return f"List[{item_type}]"
# Recursively handle item types
if isinstance(item_type, str):
try:
@@ -205,29 +281,29 @@ class StructuredOutputGenerator:
def generate_structured_outputs_from_toml_file(toml_file_path: str, output_file_path: str) -> None:
"""Generate structured output Python module from TOML file.
-
+
Args:
toml_file_path: Path to input TOML file
output_file_path: Path to output Python file
"""
with open(toml_file_path, "r", encoding="utf-8") as f:
toml_content = f.read()
-
+
generator = StructuredOutputGenerator()
python_code = generator.generate_from_toml(toml_content)
-
+
with open(output_file_path, "w", encoding="utf-8") as f:
f.write(python_code)
def generate_structured_outputs_from_toml_string(toml_content: str) -> str:
"""Generate structured output Python code from TOML string.
-
+
Args:
toml_content: TOML content as string
-
+
Returns:
Generated Python module content
"""
generator = StructuredOutputGenerator()
- return generator.generate_from_toml(toml_content)
\ No newline at end of file
+ return generator.generate_from_toml(toml_content)
diff --git a/tests/unit/pipelex/create/__init__.py b/tests/unit/pipelex/create/__init__.py
deleted file mode 100644
index 7bb4a8e50..000000000
--- a/tests/unit/pipelex/create/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-# Init file for create tests
\ No newline at end of file
diff --git a/tests/unit/pipelex/create/test_structured_output_generator.py b/tests/unit/pipelex/create/test_structured_output_generator.py
index 7512b94f4..6cdbc7678 100644
--- a/tests/unit/pipelex/create/test_structured_output_generator.py
+++ b/tests/unit/pipelex/create/test_structured_output_generator.py
@@ -246,3 +246,172 @@ another_simple = "Another simple field"
assert 'simple_description: Optional[str] = Field(default=None, description="A simple text description")' in result
assert 'complex_field: int = Field(..., description="A complex field")' in result
assert 'another_simple: Optional[str] = Field(default=None, description="Another simple field")' in result
+
+ def test_enum_generation_simple_list(self):
+ """Test generation of enum with simple list of values."""
+ toml_content = """
+[enum.OrderStatus]
+definition = "Possible states of an order"
+values = ["pending", "processing", "shipped", "delivered", "cancelled"]
+
+[schema.Order]
+definition = "An order in the system"
+
+[schema.Order.fields]
+order_id = "Unique identifier for the order"
+status = { type = "OrderStatus", definition = "Current status of the order", required = true }
+customer_name = "Name of the customer"
+"""
+
+ generator = StructuredOutputGenerator()
+ result = generator.generate_from_toml(toml_content)
+
+ pretty_print(toml_content, title="Source TOML")
+ pretty_print(result, title="Generated Result")
+
+ # Check enum generation
+ assert "from enum import Enum" in result
+ assert "class OrderStatus(str, Enum):" in result
+ assert '"""Possible states of an order"""' in result
+ assert 'PENDING = "pending"' in result
+ assert 'PROCESSING = "processing"' in result
+ assert 'SHIPPED = "shipped"' in result
+ assert 'DELIVERED = "delivered"' in result
+ assert 'CANCELLED = "cancelled"' in result
+
+ # Check schema using the enum
+ assert "class Order(StructuredContent):" in result
+ assert 'status: OrderStatus = Field(..., description="Current status of the order")' in result
+
+ def test_enum_generation_with_descriptions(self):
+ """Test generation of enum with key-value pairs (descriptions)."""
+ toml_content = """
+[enum.Priority]
+definition = "Task priority levels"
+
+[enum.Priority.values]
+low = "Low Priority"
+medium = "Medium Priority"
+high = "High Priority"
+urgent = "Urgent - Handle Immediately"
+
+[schema.Task]
+definition = "A task to be completed"
+
+[schema.Task.fields]
+title = "Task title"
+priority = { type = "Priority", definition = "How urgent is this task", required = true }
+assigned_to = "Person responsible for the task"
+"""
+
+ generator = StructuredOutputGenerator()
+ result = generator.generate_from_toml(toml_content)
+
+ pretty_print(toml_content, title="Source TOML")
+ pretty_print(result, title="Generated Result")
+
+ # Check enum generation with descriptions
+ assert "class Priority(str, Enum):" in result
+ assert '"""Task priority levels"""' in result
+ assert 'LOW = "low" # Low Priority' in result
+ assert 'MEDIUM = "medium" # Medium Priority' in result
+ assert 'HIGH = "high" # High Priority' in result
+ assert 'URGENT = "urgent" # Urgent - Handle Immediately' in result
+
+ # Check schema using the enum
+ assert 'priority: Priority = Field(..., description="How urgent is this task")' in result
+
+ def test_inline_choices(self):
+ """Test generation with inline choices (Literal type)."""
+ toml_content = """
+[schema.Product]
+definition = "A product in our catalog"
+
+[schema.Product.fields]
+name = "Product name"
+category = { choices = ["electronics", "clothing", "food", "books"], definition = "Product category", required = true }
+size = { choices = ["XS", "S", "M", "L", "XL"], definition = "Size of the product" }
+"""
+
+ generator = StructuredOutputGenerator()
+ result = generator.generate_from_toml(toml_content)
+
+ pretty_print(toml_content, title="Source TOML")
+ pretty_print(result, title="Generated Result")
+
+ # Check Literal type usage
+ assert "from typing import Optional, List, Dict, Any, Literal" in result
+ assert "category: Literal['electronics', 'clothing', 'food', 'books'] = Field(..., description=\"Product category\")" in result
+ assert "size: Optional[Literal['XS', 'S', 'M', 'L', 'XL']] = Field(default=None, description=\"Size of the product\")" in result
+
+ def test_mixed_enums_and_inline_choices(self):
+ """Test combining enum references and inline choices in the same schema."""
+ toml_content = """
+[enum.Color]
+definition = "Available colors"
+values = ["red", "blue", "green", "yellow", "black", "white"]
+
+[enum.ShippingMethod]
+definition = "How the product will be shipped"
+values = ["standard", "express", "overnight", "pickup"]
+
+[schema.ProductVariant]
+definition = "A specific variant of a product"
+
+[schema.ProductVariant.fields]
+sku = "Stock keeping unit"
+color = { type = "Color", definition = "Color of this variant" }
+size = { choices = ["S", "M", "L"], definition = "Size options" }
+shipping_method = { type = "ShippingMethod", definition = "Shipping option", required = true }
+availability = { choices = ["in_stock", "out_of_stock", "pre_order"], definition = "Current availability" }
+"""
+
+ generator = StructuredOutputGenerator()
+ result = generator.generate_from_toml(toml_content)
+
+ pretty_print(toml_content, title="Source TOML")
+ pretty_print(result, title="Generated Result")
+
+ # Check enums are generated
+ assert "class Color(str, Enum):" in result
+ assert "class ShippingMethod(str, Enum):" in result
+
+ # Check mixed usage in schema
+ assert 'color: Optional[Color] = Field(default=None, description="Color of this variant")' in result
+ assert "size: Optional[Literal['S', 'M', 'L']] = Field(default=None, description=\"Size options\")" in result
+ assert 'shipping_method: ShippingMethod = Field(..., description="Shipping option")' in result
+ assert (
+ "availability: Optional[Literal['in_stock', 'out_of_stock', 'pre_order']] = Field(default=None, description=\"Current availability\")"
+ in result
+ )
+
+ def test_enum_in_list_field(self):
+ """Test using enum types in list fields."""
+ toml_content = """
+[enum.DietaryRestriction]
+definition = "Dietary restrictions and preferences"
+values = ["vegetarian", "vegan", "gluten_free", "dairy_free", "nut_free"]
+
+[schema.MenuItem]
+definition = "An item on the restaurant menu"
+
+[schema.MenuItem.fields]
+name = "Name of the dish"
+dietary_info = { type = "list", item_type = "DietaryRestriction", definition = "Dietary accommodations" }
+tags = { type = "list", item_type = "text", definition = "General tags" }
+"""
+
+ generator = StructuredOutputGenerator()
+ result = generator.generate_from_toml(toml_content)
+
+ pretty_print(toml_content, title="Source TOML")
+ pretty_print(result, title="Generated Result")
+
+ # Check enum is generated
+ assert "class DietaryRestriction(str, Enum):" in result
+ assert 'VEGETARIAN = "vegetarian"' in result
+ assert 'GLUTEN_FREE = "gluten_free"' in result
+
+ # Check list with enum type
+ assert 'dietary_info: Optional[List[DietaryRestriction]] = Field(default=None, description="Dietary accommodations")' in result
+ assert 'tags: Optional[List[str]] = Field(default=None, description="General tags")' in result