Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
fix: mcp schema generation and non-strict schema validation
jnjpngfeb232f04a0c
diff --git a/letta/functions/schema_generator.py b/letta/functions/schema_generator.pyindex 05ea6dd25..6d0507496 100644--- a/letta/functions/schema_generator.py+++ b/letta/functions/schema_generator.py@@ -608,6 +608,20 @@ def generate_tool_schema_for_mcp(# Normalise so downstream code can treat it consistently.parameters_schema.setdefault("required", [])+ # Process properties to handle anyOf types+ if "properties" in parameters_schema:+ for field_name, field_props in parameters_schema["properties"].items():+ # Handle anyOf types by flattening to type array+ if "anyOf" in field_props and "type" not in field_props:+ types = []+ for option in field_props["anyOf"]:+ if "type" in option:+ types.append(option["type"])+ if types:+ field_props["type"] = types+ # Remove the anyOf since we've flattened it+ del field_props["anyOf"]+# Add the optional heartbeat parameterif append_heartbeat:parameters_schema["properties"][REQUEST_HEARTBEAT_PARAM] = {diff --git a/letta/functions/schema_validator.py b/letta/functions/schema_validator.pyindex db7652ac8..5a78230ad 100644--- a/letta/functions/schema_validator.py+++ b/letta/functions/schema_validator.py@@ -125,6 +125,14 @@ def validate_complete_json_schema(schema: Dict[str, Any]) -> Tuple[SchemaHealth,required = []# OpenAI strict-mode extra checks:+ # Check that all properties are in required array (applies to all levels)+ if props:+ for prop_name in props.keys():+ if prop_name not in required:+ mark_non_strict(+ f"{path}: property '{prop_name}' is not in required array (OpenAI strict mode requires all properties to be required)"+ )+for req_key in required:if props and req_key not in props:mark_invalid(f"{path}: required contains '{req_key}' not found in properties")diff --git a/letta/helpers/tool_execution_helper.py b/letta/helpers/tool_execution_helper.pyindex 524593918..886e52396 100644--- a/letta/helpers/tool_execution_helper.py+++ b/letta/helpers/tool_execution_helper.py@@ -39,12 +39,10 @@ def enable_strict_mode(tool_schema: Dict[str, Any]) -> Dict[str, Any]:# Ensure parameters is a valid dictionaryparameters = schema.get("parameters", {})-if isinstance(parameters, dict) and parameters.get("type") == "object":# Set additionalProperties to Falseparameters["additionalProperties"] = Falseschema["parameters"] = parameters-# Remove the metadata fields from the schemaschema.pop(MCP_TOOL_METADATA_SCHEMA_STATUS, None)schema.pop(MCP_TOOL_METADATA_SCHEMA_WARNINGS, None)diff --git a/letta/llm_api/helpers.py b/letta/llm_api/helpers.pyindex cff7b96ad..b6f3acb86 100644--- a/letta/llm_api/helpers.py+++ b/letta/llm_api/helpers.py@@ -133,7 +133,6 @@ def convert_to_structured_output(openai_function: dict, allow_optional: bool = Fstructured_output["parameters"]["required"] = list(structured_output["parameters"]["properties"].keys())else:raise NotImplementedError("Optional parameter handling is not implemented.")-return structured_outputdiff --git a/letta/server/server.py b/letta/server/server.pyindex d8b5a45be..358f95069 100644--- a/letta/server/server.py+++ b/letta/server/server.py@@ -2068,7 +2068,6 @@ class SyncServer(Server):raise ValueError(f"No client was created for MCP server: {mcp_server_name}")tools = await self.mcp_clients[mcp_server_name].list_tools()-# Add health information to each toolfor tool in tools:if tool.inputSchema:diff --git a/letta/services/mcp_manager.py b/letta/services/mcp_manager.pyindex c645a34ce..95a9e11b0 100644--- a/letta/services/mcp_manager.py+++ b/letta/services/mcp_manager.py@@ -68,7 +68,6 @@ class MCPManager:# list toolstools = await mcp_client.list_tools()-# Add health information to each toolfor tool in tools:if tool.inputSchema:@@ -129,7 +128,6 @@ class MCPManager:raise ValueError(f"MCP server '{mcp_server_name}' not found")mcp_tools = await self.list_mcp_server_tools(mcp_server_name, actor=actor)-for mcp_tool in mcp_tools:# TODO: @jnjpng move health check to tool classif mcp_tool.name == mcp_tool_name:diff --git a/tests/mcp_tests/test_schema_validator.py b/tests/mcp_tests/test_schema_validator.pyindex 5087153f8..0d675e0bf 100644--- a/tests/mcp_tests/test_schema_validator.py+++ b/tests/mcp_tests/test_schema_validator.py@@ -22,7 +22,7 @@ class TestSchemaValidator:"additionalProperties": False,},},- "required": ["name", "age"],+ "required": ["name", "age", "address"], # All properties must be required for strict mode"additionalProperties": False,}@@ -235,22 +235,22 @@ class TestSchemaValidator:assert status == SchemaHealth.INVALIDassert any("dict" in reason for reason in reasons)- def test_schema_with_defaults_strict_compliant(self):- """Test that root-level schemas without required field are STRICT_COMPLIANT."""+ def test_schema_with_defaults_non_strict(self):+ """Test that root-level schemas without required field are NON_STRICT_ONLY."""schema = {"type": "object","properties": {"name": {"type": "string"}, "optional": {"type": "string"}},- # Missing "required" field at root level is OK+ # Missing "required" field at root level - all properties must be required for strict mode"additionalProperties": False,}status, reasons = validate_complete_json_schema(schema)- # After fix, root level without required should be STRICT_COMPLIANT- assert status == SchemaHealth.STRICT_COMPLIANT- assert reasons == []+ # Root level without all properties required should be NON_STRICT_ONLY+ assert status == SchemaHealth.NON_STRICT_ONLY+ assert any("not in required array" in reason for reason in reasons)- def test_composio_schema_with_optional_root_properties_strict_compliant(self):- """Test that Composio-like schemas with optional root properties are STRICT_COMPLIANT."""+ def test_composio_schema_with_optional_root_properties_non_strict(self):+ """Test that Composio-like schemas with optional root properties are NON_STRICT_ONLY."""schema = {"type": "object","properties": {@@ -264,25 +264,25 @@ class TestSchemaValidator:}status, reasons = validate_complete_json_schema(schema)- assert status == SchemaHealth.STRICT_COMPLIANT- assert reasons == []+ assert status == SchemaHealth.NON_STRICT_ONLY+ assert any("not in required array" in reason for reason in reasons)- def test_root_level_without_required_strict_compliant(self):- """Test that root-level objects without 'required' field are STRICT_COMPLIANT."""+ def test_root_level_without_required_non_strict(self):+ """Test that root-level objects without 'required' field are NON_STRICT_ONLY."""schema = {"type": "object","properties": {"name": {"type": "string"},"age": {"type": "integer"},},- # No "required" field at root level+ # No "required" field at root level - all properties must be required for strict mode"additionalProperties": False,}status, reasons = validate_complete_json_schema(schema)- # Root level without required should be STRICT_COMPLIANT- assert status == SchemaHealth.STRICT_COMPLIANT- assert reasons == []+ # Root level without required should be NON_STRICT_ONLY+ assert status == SchemaHealth.NON_STRICT_ONLY+ assert any("not in required array" in reason for reason in reasons)def test_nested_object_without_required_non_strict(self):"""Test that nested objects without 'required' remain NON_STRICT_ONLY."""@@ -312,3 +312,35 @@ class TestSchemaValidator:assert status == SchemaHealth.NON_STRICT_ONLY# Should have warning about nested preferences object missing 'required'assert any("required" in reason and "preferences" in reason for reason in reasons)++ def test_user_example_schema_non_strict(self):+ """Test the user's example schema with optional properties - should be NON_STRICT_ONLY."""+ schema = {+ "type": "object",+ "properties": {+ "a": {"title": "A", "type": "integer"},+ "b": {"anyOf": [{"type": "integer"}, {"type": "null"}], "default": None, "title": "B"},+ },+ "required": ["a"], # Only 'a' is required, 'b' is not+ "additionalProperties": False,+ }++ status, reasons = validate_complete_json_schema(schema)+ assert status == SchemaHealth.NON_STRICT_ONLY+ assert any("property 'b' is not in required array" in reason for reason in reasons)++ def test_all_properties_required_strict_compliant(self):+ """Test that schemas with all properties required are STRICT_COMPLIANT."""+ schema = {+ "type": "object",+ "properties": {+ "a": {"title": "A", "type": "integer"},+ "b": {"anyOf": [{"type": "integer"}, {"type": "null"}], "default": None, "title": "B"},+ },+ "required": ["a", "b"], # All properties are required+ "additionalProperties": False,+ }++ status, reasons = validate_complete_json_schema(schema)+ assert status == SchemaHealth.STRICT_COMPLIANT+ assert reasons == []diff --git a/tests/test_schema_validator.py b/tests/test_schema_validator.pynew file mode 100644index 000000000..b3ee314ed--- /dev/null+++ b/tests/test_schema_validator.py@@ -0,0 +1,237 @@+"""+Test schema validation for OpenAI strict mode compliance.+"""++from letta.functions.schema_validator import SchemaHealth, validate_complete_json_schema+++def test_user_example_schema_not_strict():+ """Test the user's provided example schema is correctly marked as NON_STRICT_ONLY."""+ schema = {+ "properties": {+ "a": {"title": "A", "type": "integer"},+ "b": {+ "anyOf": [{"type": "integer"}, {"type": "null"}],+ "default": None,+ "title": "B",+ },+ },+ "required": ["a"], # Only 'a' is required, 'b' is not+ "type": "object",+ "additionalProperties": False,+ }++ status, reasons = validate_complete_json_schema(schema)++ # Should be NON_STRICT_ONLY because not all properties are required+ assert status == SchemaHealth.NON_STRICT_ONLY+ assert len(reasons) > 0+ # Check that the reason mentions property 'b' not being required+ assert any("property 'b' is not in required array" in reason for reason in reasons)+++def test_all_properties_required_is_strict():+ """Test that schemas with all properties required are STRICT_COMPLIANT."""+ schema = {+ "type": "object",+ "properties": {+ "a": {"type": "integer"},+ "b": {"anyOf": [{"type": "integer"}, {"type": "null"}]}, # Optional via null type+ },+ "required": ["a", "b"], # All properties are required+ "additionalProperties": False,+ }++ status, reasons = validate_complete_json_schema(schema)++ # Should be STRICT_COMPLIANT since all properties are required+ assert status == SchemaHealth.STRICT_COMPLIANT+ assert reasons == []+++def test_nested_object_missing_required_not_strict():+ """Test that nested objects with missing required properties are NON_STRICT_ONLY."""+ schema = {+ "type": "object",+ "properties": {+ "config": {+ "type": "object",+ "properties": {+ "host": {"type": "string"},+ "port": {"type": "integer"},+ "optional_field": {"anyOf": [{"type": "string"}, {"type": "null"}]},+ },+ "required": ["host", "port"], # optional_field not required+ "additionalProperties": False,+ }+ },+ "required": ["config"],+ "additionalProperties": False,+ }++ status, reasons = validate_complete_json_schema(schema)++ # Should be NON_STRICT_ONLY because nested object doesn't require all properties+ assert status == SchemaHealth.NON_STRICT_ONLY+ assert len(reasons) > 0+ assert any("optional_field" in reason and "not in required array" in reason for reason in reasons)+++def test_nested_object_all_required_is_strict():+ """Test that nested objects with all properties required are STRICT_COMPLIANT."""+ schema = {+ "type": "object",+ "properties": {+ "config": {+ "type": "object",+ "properties": {+ "host": {"type": "string"},+ "port": {"type": "integer"},+ "timeout": {"anyOf": [{"type": "integer"}, {"type": "null"}]},+ },+ "required": ["host", "port", "timeout"], # All properties required+ "additionalProperties": False,+ }+ },+ "required": ["config"],+ "additionalProperties": False,+ }++ status, reasons = validate_complete_json_schema(schema)++ # Should be STRICT_COMPLIANT since all properties at all levels are required+ assert status == SchemaHealth.STRICT_COMPLIANT+ assert reasons == []+++def test_empty_object_no_properties_is_strict():+ """Test that objects with no properties are STRICT_COMPLIANT."""+ schema = {+ "type": "object",+ "properties": {},+ "required": [],+ "additionalProperties": False,+ }++ status, reasons = validate_complete_json_schema(schema)++ # Empty objects with no properties should be STRICT_COMPLIANT+ assert status == SchemaHealth.STRICT_COMPLIANT+ assert reasons == []+++def test_missing_additionalproperties_not_strict():+ """Test that missing additionalProperties makes schema NON_STRICT_ONLY."""+ schema = {+ "type": "object",+ "properties": {+ "field": {"type": "string"},+ },+ "required": ["field"],+ # Missing additionalProperties+ }++ status, reasons = validate_complete_json_schema(schema)++ # Should be NON_STRICT_ONLY due to missing additionalProperties+ assert status == SchemaHealth.NON_STRICT_ONLY+ assert any("additionalProperties" in reason and "not explicitly set" in reason for reason in reasons)+++def test_additionalproperties_true_not_strict():+ """Test that additionalProperties: true makes schema NON_STRICT_ONLY."""+ schema = {+ "type": "object",+ "properties": {+ "field": {"type": "string"},+ },+ "required": ["field"],+ "additionalProperties": True, # Allows additional properties+ }++ status, reasons = validate_complete_json_schema(schema)++ # Should be NON_STRICT_ONLY due to additionalProperties not being false+ assert status == SchemaHealth.NON_STRICT_ONLY+ assert any("additionalProperties" in reason and "not false" in reason for reason in reasons)+++def test_complex_schema_with_arrays():+ """Test a complex schema with arrays and nested objects."""+ schema = {+ "type": "object",+ "properties": {+ "items": {+ "type": "array",+ "items": {+ "type": "object",+ "properties": {+ "id": {"type": "integer"},+ "name": {"type": "string"},+ "tags": {+ "type": "array",+ "items": {"type": "string"},+ },+ },+ "required": ["id", "name", "tags"], # All properties required+ "additionalProperties": False,+ },+ },+ "total": {"type": "integer"},+ },+ "required": ["items", "total"], # All properties required+ "additionalProperties": False,+ }++ status, reasons = validate_complete_json_schema(schema)++ # Should be STRICT_COMPLIANT since all properties at all levels are required+ assert status == SchemaHealth.STRICT_COMPLIANT+ assert reasons == []+++def test_fastmcp_tool_schema_not_strict():+ """Test that a schema from FastMCP with optional field 'b' is marked as NON_STRICT_ONLY."""+ # This is the exact schema format provided by the user+ schema = {+ "properties": {+ "a": {"title": "A", "type": "integer"},+ "b": {"anyOf": [{"type": "integer"}, {"type": "null"}], "default": None, "title": "B"},+ },+ "required": ["a"], # Only 'a' is required, violates strict mode requirement+ "type": "object",+ "additionalProperties": False,+ }++ status, reasons = validate_complete_json_schema(schema)++ # Should be NON_STRICT_ONLY because 'b' is not in the required array+ # Even though 'b' accepts null (making it optional), OpenAI strict mode+ # requires ALL properties to be in the required array+ assert status == SchemaHealth.NON_STRICT_ONLY+ assert len(reasons) > 0+ assert any("property 'b' is not in required array" in reason for reason in reasons)+++def test_union_types_with_anyof():+ """Test that anyOf unions are handled correctly."""+ schema = {+ "type": "object",+ "properties": {+ "value": {+ "anyOf": [+ {"type": "string"},+ {"type": "number"},+ {"type": "null"},+ ]+ }+ },+ "required": ["value"],+ "additionalProperties": False,+ }++ status, reasons = validate_complete_json_schema(schema)++ # Should be STRICT_COMPLIANT - anyOf is allowed and all properties are required+ assert status == SchemaHealth.STRICT_COMPLIANT+ assert reasons == []