Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
TypeScript agentic-aiagentsaiai-agentsaiagentsdeveloper-toolsfunction-callinggpt-4javascriptjsllmllmopsmcppythonremote-mcp-serverssetypescript
fix(python): preserve and police dynamic object content in schema conversion
jkomynobbeacac65363
diff --git a/python/composio/utils/schema_converter.py b/python/composio/utils/schema_converter.pyindex 8ff04adbd..2f249f855 100644--- a/python/composio/utils/schema_converter.py+++ b/python/composio/utils/schema_converter.py@@ -9,6 +9,7 @@ boolean schemas (true/false values that are valid in JSON Schema draft-06+).We handle those by pre-filtering them before passing to the library."""+import reimport typing as tfrom functools import reduce@@ -17,13 +18,22 @@ from json_schema_to_pydantic import (SchemaError,create_model as create_model_from_schema,)-from pydantic import Field, create_model as create_pydantic_model+from pydantic import (+ BaseModel,+ ConfigDict,+ Field,+ TypeAdapter,+ create_model as create_pydantic_model,+ model_validator,+)from pydantic_core import core_schemafrom composio.utils.logging import get as get_loggerlogger = get_logger(__name__)+_MISSING = object()+class _UnsatisfiableSchema:"""Pydantic type for JSON schemas that reject every value."""@@ -221,6 +231,208 @@ def _contains_unsatisfiable_schema(schema: t.Any) -> bool:return False+_PRIMITIVE_TYPES = frozenset({"string", "number", "integer", "boolean", "null"})++# JSON Schema keyword -> Pydantic `Field` constraint, by instance type. Simple+# primitives lose their constraints in `PYDANTIC_TYPE_TO_PYTHON_TYPE`, which is+# fine for declared fields the library models but not for dynamic keys, where+# this module is the only validator.+_CONSTRAINTS: t.Dict[str, t.Dict[str, str]] = {+ "string": {+ "minLength": "min_length",+ "maxLength": "max_length",+ "pattern": "pattern",+ },+ "array": {"minItems": "min_length", "maxItems": "max_length"},+}+_NUMERIC_CONSTRAINTS = {+ "minimum": "ge",+ "maximum": "le",+ "exclusiveMinimum": "gt",+ "exclusiveMaximum": "lt",+ "multipleOf": "multiple_of",+}+_CONSTRAINTS["number"] = _NUMERIC_CONSTRAINTS+_CONSTRAINTS["integer"] = _NUMERIC_CONSTRAINTS+++class _DynamicKeyValidator(t.NamedTuple):+ """Validator for one dynamic-key schema, plus how strictly to apply it."""++ adapter: TypeAdapter+ strict: bool++ def coerce(self, value: t.Any) -> t.Any:+ return self.adapter.dump_python(+ self.adapter.validate_python(value, strict=self.strict)+ )+++class _ObjectPolicy(t.NamedTuple):+ """Compiled dynamic-key policy for one JSON Schema object node.++ This is the single place the covered object contract lives, so+ ``json_schema_to_model`` and ``json_schema_to_pydantic_type`` cannot drift+ apart on which keys are accepted, validated, or preserved.+ """++ declared: t.FrozenSet[str]+ patterns: t.Tuple[t.Tuple[t.Pattern[str], t.Optional[_DynamicKeyValidator]], ...]+ additional: t.Optional[_DynamicKeyValidator]+ rejects_unmatched: bool++ @property+ def is_trivial(self) -> bool:+ """No dynamic keys to police and nothing to reject."""+ return (+ not self.patterns and not self.rejects_unmatched and self.additional is None+ )+++def object_is_open(schema: t.Dict[str, t.Any]) -> bool:+ """Whether an object schema accepts arbitrary keys it says nothing about.++ An object is open when it declares no named properties and either omits+ ``additionalProperties`` or sets it to ``true``. Objects that *do* name+ properties stay strict on omission, matching the TypeScript converters.+ """+ additional = schema.get("additionalProperties", _MISSING)+ return not (schema.get("properties") or {}) and additional in (_MISSING, True)+++def _dynamic_key_validator(schema: t.Any) -> t.Optional[_DynamicKeyValidator]:+ """Build the validator for one dynamic-key schema, or ``None`` if unmodellable."""+ try:+ annotation: t.Any = json_schema_to_pydantic_type(schema)+ strict = False++ if isinstance(schema, dict):+ schema_type = schema.get("type") or ""+ constraints = {+ field: schema[keyword]+ for keyword, field in _CONSTRAINTS.get(schema_type, {}).items()+ if keyword in schema+ }+ if constraints:+ annotation = t.Annotated[annotation, Field(**constraints)]+ # Primitives validate strictly so Pydantic's lax coercion cannot+ # admit a value the TypeScript converters reject (`true` as a+ # number, `1` as a boolean).+ strict = schema_type in _PRIMITIVE_TYPES++ return _DynamicKeyValidator(TypeAdapter(annotation), strict)+ except Exception as exc: # pragma: no cover - defensive+ logger.debug(f"Could not build a dynamic-key validator for {schema!r}: {exc}")+ return None+++def _compile_object_policy(schema: t.Dict[str, t.Any]) -> _ObjectPolicy:+ declared = frozenset(schema.get("properties") or {})+ additional = schema.get("additionalProperties", _MISSING)++ patterns = []+ for pattern, pattern_schema in (schema.get("patternProperties") or {}).items():+ try:+ patterns.append(+ (re.compile(pattern), _dynamic_key_validator(pattern_schema))+ )+ except re.error:+ logger.debug(f"Skipping uncompilable patternProperties regex: {pattern!r}")++ return _ObjectPolicy(+ declared=declared,+ patterns=tuple(patterns),+ additional=(+ _dynamic_key_validator(additional) if isinstance(additional, dict) else None+ ),+ rejects_unmatched=additional is False+ or (additional is _MISSING and bool(declared)),+ )+++class _DynamicObjectModel(BaseModel):+ """Base for object models that must police and preserve dynamic keys.++ ``extra="allow"`` is what makes preservation possible at all: Pydantic's+ default silently discards accepted arguments, so a provider that re-reads+ them (``LangchainProvider`` uses ``getattr``) would lose valid payloads.+ Coerced values are written back into ``__pydantic_extra__`` so they show up+ in ``model_dump()`` and remain readable as attributes.+ """++ model_config = ConfigDict(extra="allow")++ @model_validator(mode="after")+ def _apply_object_policy(self):+ policy: t.Optional[_ObjectPolicy] = getattr(+ type(self), "__composio_object_policy__", None+ )+ if policy is None or policy.is_trivial:+ return self++ # A key declared in `properties` is still checked against every matching+ # pattern; its declared value is authoritative and is not overwritten.+ for name, field in type(self).model_fields.items():+ if name not in self.model_fields_set:+ continue+ key = field.alias or name+ for regex, validator in policy.patterns:+ if validator is not None and regex.search(key):+ validator.coerce(getattr(self, name))++ extra = self.__pydantic_extra__+ if not extra:+ return self++ unrecognized = []+ for key, value in list(extra.items()):+ matched = False+ for regex, validator in policy.patterns:+ if regex.search(key):+ matched = True+ if validator is not None:+ extra[key] = validator.coerce(value)++ if matched or key in policy.declared:+ continue++ if policy.rejects_unmatched:+ unrecognized.append(key)+ elif policy.additional is not None:+ extra[key] = policy.additional.coerce(value)++ if unrecognized:+ raise ValueError(+ f"Unrecognized key(s) in object: {', '.join(repr(key) for key in unrecognized)}"+ )++ return self+++def apply_object_policy(+ schema: t.Dict[str, t.Any],+ base_model: t.Type[BaseModel],+ model_name: t.Optional[str] = None,+) -> t.Type[BaseModel]:+ """Wrap ``base_model`` with the covered dynamic-key policy for ``schema``."""+ policy = _compile_object_policy(schema)+ name = model_name or base_model.__name__++ return t.cast(+ t.Type[BaseModel],+ type(+ name,+ (_DynamicObjectModel, base_model),+ {+ "__module__": __name__,+ "__qualname__": name,+ "model_config": ConfigDict(extra="allow"),+ "__composio_object_policy__": policy,+ },+ ),+ )++def json_schema_to_pydantic_type(json_schema: t.Union[t.Dict[str, t.Any], bool],) -> t.Union[t.Type, t.Optional[t.Any]]:@@ -354,12 +566,20 @@ def _convert_with_library(if schema.get("type") == "object":if _contains_unsatisfiable_schema(schema.get("properties", {})):return _convert_object_with_unsatisfiable_properties(schema)+ # A property-less object with no dynamic-key constraints accepts and+ # preserves arbitrary content. Modelling it as an empty Pydantic+ # model instead silently discarded every key (issue #4064).+ if object_is_open(schema) and not schema.get("patternProperties"):+ return t.cast(t.Type, t.Dict[str, t.Any])if "title" not in schema:schema = {**schema, "title": "GeneratedModel"}- return create_model_from_schema(+ return apply_object_policy(schema,- allow_undefined_array_items=True,- allow_undefined_type=True,+ create_model_from_schema(+ schema,+ allow_undefined_array_items=True,+ allow_undefined_type=True,+ ),)# For array schemasdiff --git a/python/composio/utils/shared.py b/python/composio/utils/shared.pyindex 9ac99b841..d43114335 100644--- a/python/composio/utils/shared.py+++ b/python/composio/utils/shared.py@@ -22,6 +22,7 @@ from composio.utils.schema_converter import (CONTAINER_TYPE,FALLBACK_VALUES,PYDANTIC_TYPE_TO_PYTHON_TYPE,+ apply_object_policy,json_schema_to_pydantic_type,)@@ -457,7 +458,13 @@ def json_schema_to_model(skip_default=skip_default,)field_definitions[updated_name] = (pydantic_type, pydantic_field)- return create_model(model_name, **field_definitions) # type: ignore+ # The dynamic-key policy is shared with `json_schema_to_pydantic_type` so the+ # two entry points cannot disagree about which arguments survive conversion.+ return apply_object_policy(+ json_schema,+ create_model(model_name, **field_definitions), # type: ignore+ model_name=model_name,+ )def pydantic_model_from_param_schema(param_schema: t.Dict) -> t.Type:diff --git a/python/tests/fixtures/json_schema_conversion_corpus.py b/python/tests/fixtures/json_schema_conversion_corpus.pynew file mode 100644index 000000000..1331079d7--- /dev/null+++ b/python/tests/fixtures/json_schema_conversion_corpus.py@@ -0,0 +1,127 @@+"""Decoder for the shared cross-SDK JSON Schema conversion corpus.++The corpus is one logical fixture with a byte-identical copy per language. This+module resolves the Python copy module-relatively and decodes it into typed+models, so a malformed corpus fails at the test boundary rather than surfacing+as a confusing assertion error deep inside a converter test.+"""++import json+import typing as t+from pathlib import Path++from pydantic import BaseModel, ConfigDict, Field++CORPUS_PATH = Path(__file__).parent / "json-schema-conversion" / "object-cases.json"++Language = t.Literal["zod", "effect", "python"]+LANGUAGES: t.Tuple[Language, ...] = ("zod", "effect", "python")++_UNSET = object()+++class LanguageExpectation(BaseModel):+ """Per-language expectation. `output` is absent when the case does not pin one."""++ model_config = ConfigDict(extra="forbid")++ output: t.Any = _UNSET+ accepted: t.Optional[bool] = None++ @property+ def has_output(self) -> bool:+ return self.output is not _UNSET+++class Divergence(BaseModel):+ model_config = ConfigDict(extra="forbid")++ reason: str = Field(min_length=1)+++class CorpusInstance(BaseModel):+ model_config = ConfigDict(extra="forbid")++ input: t.Dict[str, t.Any]+ accepted: bool+ zod: t.Optional[LanguageExpectation] = None+ effect: t.Optional[LanguageExpectation] = None+ python: t.Optional[LanguageExpectation] = None+ divergence: t.Optional[Divergence] = None++ def accepted_for(self, language: Language) -> bool:+ """Acceptance for one language: the shared flag unless a declared divergence overrides it."""+ expectation: t.Optional[LanguageExpectation] = getattr(self, language)+ if expectation is not None and expectation.accepted is not None:+ return expectation.accepted+ return self.accepted+++class Ingress(BaseModel):+ model_config = ConfigDict(extra="forbid")++ accepted: bool+ preserved: t.Dict[str, t.Any]+++class CorpusCase(BaseModel):+ model_config = ConfigDict(extra="forbid")++ id: str = Field(min_length=1)+ description: t.Optional[str] = None+ divergesFromJsonSchema: t.Optional[str] = None # noqa: N815+ schema_: t.Dict[str, t.Any] = Field(alias="schema")+ ingress: t.Optional[Ingress] = None+ instances: t.List[CorpusInstance] = Field(min_length=1)+++class Corpus(BaseModel):+ model_config = ConfigDict(extra="forbid")++ cases: t.List[CorpusCase] = Field(min_length=1)+++def assert_corpus_invariants(cases: t.Sequence[CorpusCase]) -> None:+ """Ids are unique, and a language may only disagree about acceptance when it says why."""+ seen: t.Set[str] = set()+ for case in cases:+ if case.id in seen:+ raise ValueError(f"Duplicate corpus case id: {case.id}")+ seen.add(case.id)++ for index, instance in enumerate(case.instances):+ for language in LANGUAGES:+ expectation: t.Optional[LanguageExpectation] = getattr(+ instance, language+ )+ override = expectation.accepted if expectation else None+ if (+ override is not None+ and override != instance.accepted+ and instance.divergence is None+ ):+ raise ValueError(+ f"Corpus case {case.id} instance {index} overrides {language} "+ "acceptance without a divergence reason"+ )+++_cached: t.Optional[t.List[CorpusCase]] = None+++def load_object_cases() -> t.List[CorpusCase]:+ """Decode the shared corpus and enforce its cross-language invariants."""+ global _cached+ if _cached is None:+ cases = Corpus.model_validate(json.loads(CORPUS_PATH.read_text())).cases+ assert_corpus_invariants(cases)+ _cached = cases+ return _cached+++def find_case(case_id: str) -> CorpusCase:+ """Look up one case by id, failing loudly when the corpus no longer carries it."""+ for case in load_object_cases():+ if case.id == case_id:+ return case+ raise LookupError(f"Corpus is missing the {case_id} case")diff --git a/python/tests/test_provider.py b/python/tests/test_provider.pyindex 7f2c81659..b3c5dc16e 100644--- a/python/tests/test_provider.py+++ b/python/tests/test_provider.py@@ -15,7 +15,10 @@ from composio.client.types import Tool, tool_list_responsefrom composio.core.models.base import allow_trackingfrom composio.core.models.tools import Toolsfrom composio.core.provider import AgenticProvider, NonAgenticProvider+from pydantic import ValidationError+from tests.conftest import mock_http_client+from tests.fixtures.json_schema_conversion_corpus import find_case@pytest.fixture(autouse=True)@@ -902,6 +905,82 @@ class TestLangchainReservedKeywords:assert captured == {"from": "2024-01-01"}+class TestLangchainFreeFormObjectArguments:+ """Regression for issue #4064 at the Python provider boundary.++ `LangchainProvider.wrap_tool` builds its `args_schema` from+ `json_schema_to_model` and re-reads each argument with `getattr`, so a+ property-less object that validates but drops its content never reaches+ execution.+ """++ def _make_tool(self, properties: dict):+ return Tool(+ name="Metabase Create Card",+ slug="METABASE_POST_API_CARD",+ description="A tool with a free-form object argument",+ input_parameters={+ "type": "object",+ "title": "MetabaseCreateCardRequest",+ "properties": properties,+ "required": list(properties),+ },+ output_parameters={},+ available_versions=["12012025_00"],+ version="12012025_00",+ scopes=[],+ toolkit=tool_list_response.ItemToolkit(+ name="Metabase", slug="metabase", logo=""+ ),+ deprecated=tool_list_response.ItemDeprecated(+ available_versions=["12012025_00"],+ displayName="Metabase Create Card",+ version="12012025_00",+ toolkit=tool_list_response.ItemDeprecatedToolkit(logo=""),+ is_deprecated=False,+ ),+ is_deprecated=False,+ no_auth=False,+ tags=[],+ )++ def test_free_form_object_reaches_execution_intact(self):+ from composio_langchain import LangchainProvider++ case = find_case("nested-free-form-object")+ payload = case.instances[0].input++ captured = {}++ def mock_execute(slug, arguments):+ captured.update(arguments)+ return {"data": {}, "error": None, "successful": True}++ wrapped = LangchainProvider().wrap_tool(+ self._make_tool(case.schema_["properties"]), mock_execute+ )+ wrapped.run(payload)++ assert captured == payload++ def test_unknown_keys_are_still_rejected_before_execution(self):+ from composio_langchain import LangchainProvider++ executed = []++ def mock_execute(slug, arguments):+ executed.append(arguments)+ return {"data": {}, "error": None, "successful": True}++ wrapped = LangchainProvider().wrap_tool(+ self._make_tool({"name": {"type": "string"}}), mock_execute+ )++ with pytest.raises(ValidationError):+ wrapped.args_schema.model_validate({"name": "a", "typo": 1})+ assert executed == []++class TestProviderEdgeCases:"""Test edge cases and error handling for providers."""diff --git a/python/tests/test_schema_converter.py b/python/tests/test_schema_converter.pyindex 1d6b27531..cba165526 100644--- a/python/tests/test_schema_converter.py+++ b/python/tests/test_schema_converter.py@@ -8,6 +8,7 @@ from composio.utils.schema_converter import (json_schema_to_pydantic_type,)from composio.utils.shared import json_schema_to_model+from tests.fixtures.json_schema_conversion_corpus import load_object_casesdef test_true_becomes_empty_schema():@@ -194,3 +195,39 @@ def test_schema_converter_required_unsatisfiable_property_has_no_valid_value():for impossible_value in (None, "some string", 123):with pytest.raises(ValidationError):model(impossible=impossible_value)+++ "case,index",+ [+ (case, index)+ for case in load_object_cases()+ for index in range(len(case.instances))+ ],+ ids=[+ f"{case.id}[{index}]"+ for case in load_object_cases()+ for index in range(len(case.instances))+ ],+)+def test_shared_object_corpus_through_json_schema_to_pydantic_type(case, index):+ """`json_schema_to_pydantic_type` must agree with `json_schema_to_model`.++ The two entry points return different things — a Pydantic type versus a+ `BaseModel` subclass — so the type is exercised through a `TypeAdapter` and+ compared as a JSON-shaped value.+ """+ instance = case.instances[index]+ adapter = TypeAdapter(json_schema_to_pydantic_type(case.schema_))++ if not instance.accepted_for("python"):+ with pytest.raises(ValidationError):+ adapter.validate_python(instance.input)+ return++ result = adapter.validate_python(instance.input)+ if instance.python is not None and instance.python.has_output:+ assert (+ adapter.dump_python(result, mode="json", by_alias=True)+ == instance.python.output+ )diff --git a/python/tests/test_schema_parser.py b/python/tests/test_schema_parser.pyindex 32185ea84..d786088f9 100644--- a/python/tests/test_schema_parser.py+++ b/python/tests/test_schema_parser.py@@ -11,6 +11,8 @@ import pytestfrom pydantic import BaseModel, TypeAdapter, ValidationErrorfrom pydantic.fields import PydanticUndefined+from tests.fixtures.json_schema_conversion_corpus import find_case, load_object_cases+from composio.utils.shared import (get_signature_format_from_schema_params,json_schema_to_fields_dict,@@ -1715,3 +1717,50 @@ class TestGetSignatureFormatFromSchemaParams:if __name__ == "__main__":pytest.main([__file__, "-v"])+++class TestSharedObjectCorpusThroughJsonSchemaToModel:+ """`json_schema_to_model` must satisfy the shared cross-SDK object contract.++ This is the provider-facing entry point (`args_schema`), so a model that+ accepts an argument but discards it is as much a defect as one that rejects+ a valid payload.+ """++ @pytest.mark.parametrize(+ "case,index",+ [+ (case, index)+ for case in load_object_cases()+ for index in range(len(case.instances))+ ],+ ids=[+ f"{case.id}[{index}]"+ for case in load_object_cases()+ for index in range(len(case.instances))+ ],+ )+ def test_case(self, case, index: int) -> None:+ instance = case.instances[index]+ model = json_schema_to_model(case.schema_)++ if not instance.accepted_for("python"):+ with pytest.raises(ValidationError):+ model.model_validate(instance.input)+ return++ result = model.model_validate(instance.input)+ if instance.python is not None and instance.python.has_output:+ assert (+ result.model_dump(mode="json", by_alias=True) == instance.python.output+ )++ def test_free_form_content_is_readable_as_attributes(self) -> None:+ """Preserved dynamic keys must survive `getattr`, not just `model_dump`."""+ case = find_case("root-free-form-absent-properties")+ model = json_schema_to_model(case.schema_)++ result = model.model_validate(case.instances[0].input)++ assert getattr(result, "anything") == {"a": 1}+ assert getattr(result, "other") == "x"