Enterprise agent platform for building, deploying, and operating AI teammates.
"""The schema copy shown to the model: every node but the root may be null."""
from __future__ import annotations
import copy
import json
import time
from typing import Any
import pytest
from jsonschema import Draft202012Validator
from xagent.core.agent import structured_schema
from xagent.core.agent.structured_schema import (
allowed_values_suffix,
canonical_json_size,
check_schema_structure,
measure_expansion,
)
from xagent.core.model.chat.basic import claude, gemini
MARKER = "__NOT_EXPRESSIBLE__"
STRING: dict[str, Any] = {"type": "string"}
NULLABLE_STRING: dict[str, Any] = {"type": ["string", "null"]}
# This bound only catches a slowdown by an order of magnitude; the cost of
# stopping early is pinned by how much text is serialized.
BACKSTOP_SECONDS = 10.0
def _obj(properties: dict[str, Any], **extra: Any) -> dict[str, Any]:
node: dict[str, Any] = {
"type": "object",
"properties": properties,
"required": list(properties),
"additionalProperties": False,
}
node.update(extra)
return node
def _ref(name: str) -> dict[str, Any]:
return {"$ref": f"#/$defs/{name}"}
def _union(node: dict[str, Any], **extra: Any) -> dict[str, Any]:
return {"anyOf": [node, {"type": "null"}], **extra}
def _suffix(*values: Any) -> str:
return " Allowed values: " + json.dumps(list(values), separators=(",", ":"))
def _copy_of(schema: dict[str, Any]) -> dict[str, Any]:
assert check_schema_structure(schema) is None
return structured_schema.model_facing_schema(
structured_schema.compile_schema(schema)
)
# --- the nine shapes, each with its copy written out in full -------------------------
COPY_CASES: dict[str, tuple[dict[str, Any], dict[str, Any]]] = {
"plain-property": (
_obj({"a": STRING}),
_obj({"a": NULLABLE_STRING}),
),
"array-element": (
_obj({"xs": {"type": "array", "items": {"type": "integer"}}}),
_obj(
{
"xs": {
"type": ["array", "null"],
"items": {"type": ["integer", "null"]},
}
}
),
),
"reference": (
_obj(
{"a": _ref("T"), "b": _ref("T")},
**{"$defs": {"T": {"type": "string", "description": "d"}}},
),
_obj(
{
"a": {"type": ["string", "null"], "description": "d"},
"b": {"type": ["string", "null"], "description": "d"},
}
),
),
"union-at-a-property": (
_obj(
{
"a": _union(
{"type": "string", "description": "inner"}, description="outer"
)
}
),
_obj({"a": {"type": ["string", "null"], "description": "outer inner"}}),
),
"union-at-an-array-element": (
_obj(
{
"xs": {
"type": "array",
"items": _union(
{"type": "string", "description": "inner", "title": "X"},
description="outer",
),
}
}
),
_obj(
{
"xs": {
"type": ["array", "null"],
"items": {
"type": ["string", "null"],
"title": "X",
"description": "outer inner",
},
}
}
),
),
"nullable-type-with-an-enum": (
_obj({"a": {"type": ["string", "null"], "enum": ["a", None]}}),
_obj(
{
"a": {
"type": ["string", "null"],
"enum": ["a", None, MARKER],
"description": _suffix("a", None, MARKER),
}
}
),
),
"const": (
_obj({"a": {"type": "string", "const": "x"}}),
_obj(
{
"a": {
"type": ["string", "null"],
"enum": ["x", None, MARKER],
"description": _suffix("x", None, MARKER),
}
}
),
),
"root": (
_obj({"a": STRING}, description="root text", title="R"),
_obj({"a": NULLABLE_STRING}, description="root text", title="R"),
),
"enum-that-is-not-text": (
_obj({"a": {"type": "integer", "enum": [1, 2]}}),
_obj(
{
"a": {
"type": ["integer", "null"],
"enum": [1, 2, None],
"description": _suffix(1, 2, None),
}
}
),
),
"union-over-an-alias-chain": (
_obj(
{"a": _union(_ref("A"), description="o")},
**{
"$defs": {
"A": _ref("B"),
"B": {"type": "string", "description": "i", "title": "Inner"},
}
},
),
_obj(
{"a": {"type": ["string", "null"], "description": "o i", "title": "Inner"}}
),
),
"union-title-conflict-takes-the-value-title": (
_obj({"a": _union({"type": "string", "title": "Inner"}, title="Outer")}),
_obj({"a": {"type": ["string", "null"], "title": "Inner"}}),
),
"union-title-only-on-the-union": (
_obj({"a": _union(STRING, title="Outer")}),
_obj({"a": {"type": ["string", "null"], "title": "Outer"}}),
),
"union-description-only-on-the-union": (
_obj({"a": _union(STRING, description="outer")}),
_obj({"a": {"type": ["string", "null"], "description": "outer"}}),
),
"union-description-only-on-the-value": (
_obj({"a": _union({"type": "string", "description": "inner"})}),
_obj({"a": {"type": ["string", "null"], "description": "inner"}}),
),
"union-over-an-enum-adds-one-null": (
_obj({"a": _union({"type": "string", "enum": ["a", "b"]})}),
_obj(
{
"a": {
"type": ["string", "null"],
"enum": ["a", "b", None, MARKER],
"description": _suffix("a", "b", None, MARKER),
}
}
),
),
"target-that-is-a-nullable-union": (
_obj(
{"a": _ref("N"), "xs": {"type": "array", "items": _ref("N")}},
**{
"$defs": {
"N": _union(
{"type": "string", "description": "x"},
description="y",
title="T",
)
}
},
),
_obj(
{
"a": {"type": ["string", "null"], "title": "T", "description": "y x"},
"xs": {
"type": ["array", "null"],
"items": {
"type": ["string", "null"],
"title": "T",
"description": "y x",
},
},
}
),
),
"null-listed-first-keeps-its-order": (
_obj({"a": {"type": ["null", "string"]}}),
_obj({"a": {"type": ["null", "string"]}}),
),
}
@pytest.mark.parametrize(
("schema", "expected"), COPY_CASES.values(), ids=COPY_CASES.keys()
)
def test_the_copy_of_each_shape_is_the_hand_written_schema(
schema: dict[str, Any], expected: dict[str, Any]
) -> None:
assert _copy_of(schema) == expected
def test_the_copy_lists_properties_in_the_callers_order() -> None:
schema = _obj(
{
"z": {"type": "array", "items": _obj({"k": STRING, "d": STRING})},
"a": _ref("A"),
"m": _union(_ref("M")),
},
**{
"$defs": {
"A": _obj({"y": STRING, "b": STRING}),
"M": _obj({"q": STRING, "c": STRING}),
}
},
)
copy_ = _copy_of(schema)
assert list(copy_["properties"]) == ["z", "a", "m"]
assert list(copy_["properties"]["z"]["items"]["properties"]) == ["k", "d"]
assert list(copy_["properties"]["a"]["properties"]) == ["y", "b"]
assert list(copy_["properties"]["m"]["properties"]) == ["q", "c"]
def test_make_nullable_leaves_a_nullable_node_as_it_is() -> None:
node = {"type": ["null", "string"], "enum": [None, "x"], "description": "d"}
assert structured_schema.make_nullable(node) == node
assert structured_schema.make_nullable({"type": ["string", "null"]}) == {
"type": ["string", "null"]
}
def test_the_schema_is_not_changed_and_the_copy_shares_nothing_with_it() -> None:
schema = _obj(
{
"a": _ref("T"),
"b": {"type": "array", "items": _ref("T")},
"c": {"type": "string", "enum": ["x", "y"]},
"d": _union(_ref("T")),
},
**{"$defs": {"T": _obj({"k": {"type": "integer", "enum": [1]}})}},
)
before = copy.deepcopy(schema)
result = _copy_of(schema)
assert schema == before
owned: set[int] = set()
def collect(value: Any, into: set[int]) -> None:
if isinstance(value, dict | list):
assert id(value) not in into, "the same container appears twice"
into.add(id(value))
for child in value.values() if isinstance(value, dict) else value:
collect(child, into)
collect(result, owned)
original: set[int] = set()
collect(schema, original)
assert owned.isdisjoint(original)
# --- a null anywhere but the root ------------------------------------------------------
RICH = _obj(
{
"name": STRING,
"age": {"type": "integer"},
"tags": {"type": "array", "items": STRING},
"addr": _ref("Addr"),
"nick": _union(STRING),
"alts": {"type": "array", "items": _union(STRING)},
"status": {"type": "string", "enum": ["open", "closed"]},
"kind": {"type": "string", "const": "person"},
"level": {"type": "integer", "enum": [1, 2]},
"opt": {"type": ["string", "null"], "enum": ["x", None]},
"flag": {"type": "boolean"},
"ratio": {"type": "number"},
},
**{
"$defs": {
"Addr": _obj({"street": STRING, "geo": _obj({"lat": {"type": "number"}})})
}
},
)
RICH_FULL = {
"name": "Ann",
"age": 3,
"tags": ["t", "u"],
"addr": {"street": "s", "geo": {"lat": 1.5}},
"nick": "n",
"alts": ["p", "q"],
"status": "open",
"kind": "person",
"level": 2,
"opt": "x",
"flag": True,
"ratio": 0.5,
}
def _paths(value: Any, prefix: tuple[Any, ...] = ()) -> list[tuple[Any, ...]]:
found = [prefix]
if isinstance(value, dict):
for key, child in value.items():
found.extend(_paths(child, prefix + (key,)))
elif isinstance(value, list):
for index, child in enumerate(value):
found.extend(_paths(child, prefix + (index,)))
return found
def _with_null_at(instance: Any, path: tuple[Any, ...]) -> Any:
changed = copy.deepcopy(instance)
target = changed
for step in path[:-1]:
target = target[step]
target[path[-1]] = None
return changed
def test_the_copy_accepts_a_null_at_every_position_but_the_root() -> None:
result = _copy_of(RICH)
validator = Draft202012Validator(result)
assert Draft202012Validator(RICH).is_valid(RICH_FULL)
assert validator.is_valid(RICH_FULL)
positions = [p for p in _paths(RICH_FULL) if p]
assert len(positions) > 15
for path in positions:
assert validator.is_valid(_with_null_at(RICH_FULL, path)), path
assert not validator.is_valid(None)
# --- the size of the copy ------------------------------------------------------
def _walk_nodes(copy_node: dict[str, Any]) -> list[dict[str, Any]]:
found = [copy_node]
for child in copy_node.get("properties", {}).values():
found.extend(_walk_nodes(child))
if "items" in copy_node:
found.extend(_walk_nodes(copy_node["items"]))
return found
def _keys_anywhere(value: Any) -> set[str]:
keys: set[str] = set()
if isinstance(value, dict):
for key, child in value.items():
keys.add(key)
keys |= _keys_anywhere(child)
elif isinstance(value, list):
for child in value:
keys |= _keys_anywhere(child)
return keys
def test_the_copy_has_no_reference_no_definitions_and_stays_inside_the_limits() -> None:
schema = _obj(
{
"a": _ref("A"),
"b": {"type": "array", "items": _ref("A")},
"c": _union(_ref("A")),
},
**{
"$defs": {
"A": _obj({"x": {"type": "string", "enum": ["p", "q"]}, "y": STRING})
}
},
)
result = _copy_of(schema)
assert not _keys_anywhere(result) & {"$ref", "$defs", "anyOf"}
assert len(_walk_nodes(result)) <= measure_expansion(schema).nodes
assert canonical_json_size(result) <= 65536
def test_a_schema_of_3_kb_whose_copy_is_290_kb_is_refused_at_request_time() -> None:
shared = _obj({f"k{i:02d}": {"type": "string", "const": "v"} for i in range(43)})
schema = _obj(
{f"r{j:02d}": _ref("D") for j in range(45)}, **{"$defs": {"D": shared}}
)
found = check_schema_structure(schema)
assert len(json.dumps(schema, separators=(",", ":"))) == 3454
assert measure_expansion(schema).rewritten_bytes == 292_709
assert found is not None and found.rule == "S11"
def test_a_copy_of_exactly_64_kib_of_chinese_text_is_accepted() -> None:
props: dict[str, Any] = {f"p{i:02d}": _ref("A") for i in range(21)}
props["x"] = {"type": "integer"}
schema = _obj(
props,
**{"$defs": {"A": {"type": "string", "description": "中" * 1000}}},
description="文" * 411,
)
assert check_schema_structure(schema) is None
result = _copy_of(schema)
assert canonical_json_size(result) == 65536
ascii_size = len(json.dumps(result, separators=(",", ":")).encode("utf-8"))
assert ascii_size > 65536 * 1.5
def test_an_integer_constant_written_as_1e308_stays_a_float_in_the_copy() -> None:
result = _copy_of(_obj({"n": {"type": "integer", "const": 1e308}}))
node = result["properties"]["n"]
assert node["enum"] == [1e308, None]
assert type(node["enum"][0]) is float
assert json.dumps(node["enum"]) == "[1e+308, null]"
assert "1e+308" in node["description"]
# --- the constant and the list of allowed values ----------------------------------
def test_a_constant_becomes_an_enum_and_its_values_end_the_description() -> None:
schema = _obj({"a": {"type": "string", "const": "x", "description": "Pick it."}})
node = _copy_of(schema)["properties"]["a"]
assert "const" not in node
assert node["enum"] == ["x", None, MARKER]
assert node["description"] == "Pick it." + _suffix("x", None, MARKER)
def test_a_cut_list_of_allowed_values_matches_the_function_that_writes_it() -> None:
values = [f"value-{i:03d}" for i in range(60)]
node = _copy_of(_obj({"a": {"type": "string", "enum": values}}))["properties"]["a"]
assert node["enum"] == [*values, None, MARKER]
assert node["description"] == allowed_values_suffix(node["enum"])
assert node["description"].endswith("…")
listing = node["description"].removeprefix(" Allowed values: ")
assert len(listing) == 501
# --- through the real provider rewriters -------------------------------------------
ADAPTER_SCHEMA = _obj(
{
"status": {"type": "string", "enum": ["open", "closed"]},
"statuses": {
"type": "array",
"items": {"type": "string", "enum": ["open", "closed"]},
},
"level": {"type": "integer", "enum": [1, 2]},
"maybe": _union({"type": "string", "description": "d"}),
"alts": {"type": "array", "items": _union(STRING)},
"kind": {"type": "string", "const": "person"},
}
)
def test_claude_leaves_the_copy_as_it_is() -> None:
result = _copy_of(ADAPTER_SCHEMA)
assert claude._fix_pydantic_schema_for_claude(result) == result
def _through_gemini(schema: dict[str, Any]) -> dict[str, Any]:
parameters = schema
if gemini._contains_refs_or_defs(parameters):
parameters = gemini._flatten_json_schema(parameters)
sanitized: dict[str, Any] = gemini._sanitize_schema_for_gemini(parameters)
return sanitized
def test_gemini_keeps_the_marker_and_the_null_of_a_text_enum() -> None:
result = _through_gemini(_copy_of(ADAPTER_SCHEMA))
status = result["properties"]["status"]
element = result["properties"]["statuses"]["items"]
for node in (status, element, result["properties"]["kind"]):
assert MARKER in node["enum"]
assert node["nullable"] is True
assert node["type"] == "string"
assert result["properties"]["alts"]["items"]["nullable"] is True
assert result["properties"]["maybe"]["nullable"] is True
def test_gemini_drops_an_enum_that_is_not_text_but_the_description_keeps_it() -> None:
result = _through_gemini(_copy_of(ADAPTER_SCHEMA))
level = result["properties"]["level"]
assert "enum" not in level
assert level["type"] == "integer"
assert "Allowed values: [1,2,null]" in level["description"]
# --- the budget on the copy -------------------------------------------------------------------
def _layered_schema(layers: int, leaf: dict[str, Any]) -> dict[str, Any]:
defs: dict[str, Any] = {"L0": leaf}
for layer in range(1, layers):
defs[f"L{layer}"] = _obj(
{"x": _ref(f"L{layer - 1}"), "y": _ref(f"L{layer - 1}")}
)
return _obj(
{"x": _ref(f"L{layers - 1}"), "y": _ref(f"L{layers - 1}")}, **{"$defs": defs}
)
def test_a_copy_over_the_byte_limit_stops_with_a_value_error() -> None:
schema = _obj({"a": {"type": "string", "description": "d" * 200}})
compiled = structured_schema.compile_schema(schema)
with pytest.MonkeyPatch.context() as patch:
patch.setattr(structured_schema, "MAX_EXPANDED_BYTES", 100)
with pytest.raises(ValueError) as caught:
structured_schema.model_facing_schema(compiled)
assert "d" * 20 not in str(caught.value)
assert structured_schema.model_facing_schema(compiled)
def test_a_copy_that_would_be_megabytes_is_cut_off_before_it_is_built(
monkeypatch: pytest.MonkeyPatch,
) -> None:
values = [f"{i:04d}-" + "v" * 14 for i in range(500)]
schema = _layered_schema(9, {"type": "string", "enum": values})
compiled = structured_schema.compile_schema(schema)
real_dumps = json.dumps
written: list[int] = []
def counting_dumps(value: Any, *args: Any, **kwargs: Any) -> str:
text = real_dumps(value, *args, **kwargs)
written.append(len(text.encode("utf-8", "surrogatepass")))
return text
monkeypatch.setattr(json, "dumps", counting_dumps)
start = time.perf_counter()
with pytest.raises(ValueError):
structured_schema.model_facing_schema(compiled)
elapsed = time.perf_counter() - start
monkeypatch.undo()
# Building stops at the first node that takes the copy past
# MAX_EXPANDED_BYTES, and each node is serialized at most twice (itself and
# its enum for the allowed-values text), so less than four times the limit
# is written; the whole copy would be about 6 MB.
assert sum(written) <= 4 * structured_schema.MAX_EXPANDED_BYTES
assert elapsed < BACKSTOP_SECONDS