Build and run agents you can see, understand and trust.
Python agentchatbotlarge-language-modelsllmllm-agentmcpmulti-agentmulti-modalreact-agentrealtime-agentvoice-agentvoice-assistant
fix(tool): apply schema-guided repair to tool arguments (#2496)
DiegoCluv74f52788b7cb5
diff --git a/pyproject.toml b/pyproject.tomlindex 909bf3b85..0ab175f51 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -25,7 +25,7 @@ dependencies = ["docstring_parser","filetype","json5",- "json_repair",+ "json_repair[schema]>=0.63.4","mcp<2.0.0","httpx","numpy",diff --git a/src/agentscope/_utils/_common.py b/src/agentscope/_utils/_common.pyindex a6207f54d..ec206ad9f 100644--- a/src/agentscope/_utils/_common.py+++ b/src/agentscope/_utils/_common.py@@ -87,7 +87,8 @@ def _json_loads_with_repair(json_str: str,schema: dict | None = None,) -> dict:- """The given json_str maybe incomplete, e.g. '{"key', so we need to+ """The given json_str maybe incomplete, e.g. '{"key', or carry arguments+ whose types don't match the schema, e.g. '{"n": "42"}', so we need torepair and load it into a Python object... note::@@ -99,23 +100,32 @@ def _json_loads_with_repair(json_str (`str`):The JSON string to parse, which may be incomplete or malformed.schema (`dict`, optional):- An optional JSON schema to guide the repair process.+ An optional JSON schema to guide the repair process. The repair+ is best-effort: arguments that it cannot fix are returned+ unchanged, so that the caller's validation reports them.Returns:`dict`:A dictionary parsed from the JSON string after repair attempts.- Returns an empty dict if all repair attempts fail.++ Raises:+ `ToolJSONDecodeError`:+ If the JSON string cannot be loaded into a dict."""+ parsed = None+ error_message = "Error: Failed to parse your tool arguments."try:- # Loads directly- res = json.loads(json_str)- if isinstance(res, dict):- return res+ # Loads directly. A valid dict still goes through the repair below+ # when a schema is given, because its argument types may be wrong.+ parsed = json.loads(json_str)+ if not isinstance(parsed, dict):+ error_message = (+ f"Error: Your argument string is decoded into a "+ f"{type(parsed)} object, but a dict object is expected!"+ )- error_message = (- f"Error: Your argument string is decoded into a {type(res)} "- f"object, but a dict object is expected!"- )+ elif schema is None:+ return parsedexcept json.JSONDecodeError as e:error_message = (f"Error: When decoding your tool arguments from JSON format "@@ -127,10 +137,35 @@ def _json_loads_with_repair(# Try to repair with json_repairfrom json_repair import repair_json- repaired = repair_json(json_str, stream_stable=True, schema=schema)- res = json.loads(repaired)+ try:+ res = repair_json(+ json_str,+ stream_stable=True,+ schema=schema,+ return_objects=True,+ )+ except ValueError:+ # The repair is best-effort. Leave arguments that it cannot fix+ # to the caller's schema validation, whose error message is more+ # helpful for the agent.+ res = parsed+if isinstance(res, dict):- return res+ if isinstance(parsed, dict) and parsed.keys() - res.keys():+ # Dropping arguments, e.g. under `additionalProperties:+ # false`, is a rewrite rather than a type repair.+ res = parsed++ try:+ # NaN and Infinity are accepted as numbers by jsonschema, but+ # silently bypass the minimum/maximum constraints.+ json.dumps(res, allow_nan=False)+ except ValueError:+ error_message = (+ "Error: NaN and Infinity are not valid JSON numbers."+ )+ else:+ return resexcept Exception:# Whatever the error is, we throw the original error message to thediff --git a/src/agentscope/tool/_toolkit.py b/src/agentscope/tool/_toolkit.pyindex 759388da5..2cd949e9f 100644--- a/src/agentscope/tool/_toolkit.py+++ b/src/agentscope/tool/_toolkit.py@@ -295,8 +295,12 @@ class Toolkit:# Async functiontry:- # Prepare keyword arguments- kwargs = _json_loads_with_repair(tool_call.input)+ # Prepare keyword arguments, repairing the argument types+ # against the tool schema when the model got them wrong.+ kwargs = _json_loads_with_repair(+ tool_call.input,+ tool_func.input_schema,+ )# State injectionif (diff --git a/tests/tool_argument_repair_test.py b/tests/tool_argument_repair_test.pynew file mode 100644index 000000000..e23566d3a--- /dev/null+++ b/tests/tool_argument_repair_test.py@@ -0,0 +1,264 @@+# -*- coding: utf-8 -*-+"""Unittests for the schema-guided repair of the tool call arguments."""+import unittest+from typing import Any+from unittest.async_case import IsolatedAsyncioTestCase++from utils import AnyString, MockModel++from agentscope._utils._common import _json_loads_with_repair+from agentscope.agent import Agent, InjectionConfig+from agentscope.exception import ToolJSONDecodeError+from agentscope.message import TextBlock, ToolCallBlock, UserMsg+from agentscope.model import ChatResponse+from agentscope.permission import (+ PermissionBehavior,+ PermissionContext,+ PermissionDecision,+)+from agentscope.tool import ToolBase, ToolChunk, Toolkit+++class _RecordTool(ToolBase):+ """A tool that records the arguments reaching the permission checking+ and the execution. It overrides ``__call__`` on purpose, so that the+ arguments are only repaired by ``Toolkit.call_tool``."""++ name = "record"+ description = "Record the given value."+ is_concurrency_safe = True+ is_read_only = False++ def __init__(self, value_schema: dict[str, Any]) -> None:+ """Initialize the tool with the schema of the ``value`` argument."""+ super().__init__()+ self.input_schema = {+ "type": "object",+ "properties": {"value": value_schema},+ "required": ["value"],+ }+ self.permission_inputs: list[dict[str, Any]] = []+ self.executed: list[Any] = []++ async def check_permissions(+ self,+ tool_input: dict[str, Any],+ context: PermissionContext,+ ) -> PermissionDecision:+ """Record and allow the tool input."""+ self.permission_inputs.append(tool_input)+ return PermissionDecision(+ behavior=PermissionBehavior.ALLOW,+ message="Allowed",+ )++ async def __call__(self, value: Any) -> ToolChunk:+ """Record the executed value."""+ self.executed.append(value)+ return ToolChunk(content=[TextBlock(text="Recorded")])+++class JsonLoadsWithRepairTest(unittest.TestCase):+ """Unittest for the `_json_loads_with_repair` function."""++ def test_repair_argument_types(self) -> None:+ """Test repairing the argument types with the given schema."""+ schema = {+ "type": "object",+ "properties": {+ "count": {"type": "integer"},+ "verbose": {"type": "boolean"},+ "tags": {"type": "array", "items": {"type": "string"}},+ },+ }+ self.assertDictEqual(+ _json_loads_with_repair(+ '{"count": "42", "verbose": "true", "tags": "a"}',+ schema,+ ),+ {"count": 42, "verbose": True, "tags": ["a"]},+ )+ # A malformed JSON string is repaired in both syntax and types+ self.assertDictEqual(+ _json_loads_with_repair("{count: '42', verbose: 'true',", schema),+ {"count": 42, "verbose": True},+ )+ # Without a schema, the arguments are kept as they are+ self.assertDictEqual(+ _json_loads_with_repair('{"count": "42"}'),+ {"count": "42"},+ )++ def test_keep_unrepairable_arguments(self) -> None:+ """Test that unrepairable arguments are left to the validation."""+ self.assertDictEqual(+ _json_loads_with_repair(+ '{"count": "many"}',+ {+ "type": "object",+ "properties": {"count": {"type": "integer"}},+ },+ ),+ {"count": "many"},+ )+ # Dropping an argument is a rewrite rather than a type repair+ self.assertDictEqual(+ _json_loads_with_repair(+ '{"count": "42", "verbse": true}',+ {+ "type": "object",+ "properties": {"count": {"type": "integer"}},+ "additionalProperties": False,+ },+ ),+ {"count": "42", "verbse": True},+ )++ def test_reject_invalid_arguments(self) -> None:+ """Test the arguments that cannot be loaded into a valid dict."""+ for json_str in ['"hello"', "[1, 2]"]:+ with self.subTest(json_str=json_str):+ with self.assertRaises(ToolJSONDecodeError):+ _json_loads_with_repair(json_str)++ # NaN and Infinity pass jsonschema as numbers, but silently bypass+ # the minimum/maximum constraints+ for json_str in ['{"ratio": NaN}', '{"ratio": "1e400"}']:+ with self.subTest(json_str=json_str):+ with self.assertRaises(ToolJSONDecodeError):+ _json_loads_with_repair(+ json_str,+ {+ "type": "object",+ "properties": {"ratio": {"type": "number"}},+ },+ )+++class ToolCallArgumentRepairTest(IsolatedAsyncioTestCase):+ """Unittest for repairing the arguments within an agent reply."""++ async def test_repaired_arguments_reach_permission_and_execution(+ self,+ ) -> None:+ """Test that the repaired arguments are used consistently, while the+ tool call input keeps what the model generated."""+ tool = _RecordTool({"type": "integer"})+ tool_call = ToolCallBlock(+ id="record_call_0",+ name="record",+ input='{"value": "42"}',+ )+ model = MockModel()+ model.set_responses(+ [+ [ChatResponse(content=[tool_call], is_last=True)],+ [+ ChatResponse(+ content=[TextBlock(text="Done")],+ is_last=True,+ ),+ ],+ ],+ )+ agent = Agent(+ name="Friday",+ system_prompt="You're a helpful assistant.",+ model=model,+ toolkit=Toolkit(tools=[tool]),+ injection_config=InjectionConfig(inject_runtime_state=False),+ )++ await agent.reply(UserMsg(name="user", content="Record 42"))++ self.assertListEqual(tool.permission_inputs, [{"value": 42}])+ self.assertListEqual(tool.executed, [42])+ self.assertEqual(tool_call.input, '{"value": "42"}')+ self.assertListEqual(+ [+ block.model_dump()+ for block in agent.state.context[-1].get_content_blocks(+ "tool_result",+ )+ ],+ [+ {+ "type": "tool_result",+ "id": "record_call_0",+ "name": "record",+ "output": [+ {+ "type": "text",+ "text": "Recorded",+ "id": AnyString(),+ "created_at": AnyString(),+ "finished_at": None,+ },+ ],+ "state": "success",+ "metadata": {},+ "created_at": AnyString(),+ "finished_at": None,+ },+ ],+ )++ async def test_unrepairable_arguments_are_rejected(self) -> None:+ """Test that unrepairable arguments are reported by the validation,+ without reaching the permission checking or the execution."""+ tool = _RecordTool({"type": "integer"})+ tool_call = ToolCallBlock(+ id="record_call_0",+ name="record",+ input='{"value": "many"}',+ )+ model = MockModel()+ model.set_responses(+ [+ [ChatResponse(content=[tool_call], is_last=True)],+ [+ ChatResponse(+ content=[TextBlock(text="Done")],+ is_last=True,+ ),+ ],+ ],+ )+ agent = Agent(+ name="Friday",+ system_prompt="You're a helpful assistant.",+ model=model,+ toolkit=Toolkit(tools=[tool]),+ injection_config=InjectionConfig(inject_runtime_state=False),+ )++ await agent.reply(UserMsg(name="user", content="Record many"))++ self.assertListEqual(tool.permission_inputs, [])+ self.assertListEqual(tool.executed, [])+ self.assertEqual(tool_call.input, '{"value": "many"}')+ self.assertListEqual(+ [+ block.model_dump()+ for block in agent.state.context[-1].get_content_blocks(+ "tool_result",+ )+ ],+ [+ {+ "type": "tool_result",+ "id": "record_call_0",+ "name": "record",+ "output": "Input validation failed for tool 'record': "+ "'many' is not of type 'integer'",+ "state": "error",+ "metadata": {},+ "created_at": AnyString(),+ "finished_at": None,+ },+ ],+ )+++if __name__ == "__main__":+ unittest.main()