Personal AI, On Personal Devices
Python
fix(agents): harden structured tool input normalization
Elliot Slusky0d32784ed63d
diff --git a/src/openjarvis/agents/orchestrator.py b/src/openjarvis/agents/orchestrator.pyindex d97cceb1..352aadbd 100644--- a/src/openjarvis/agents/orchestrator.py+++ b/src/openjarvis/agents/orchestrator.py@@ -151,7 +151,13 @@ class OrchestratorAgent(ToolUsingAgent):tool_result = self._executor.execute(tool_call)all_tool_results.append(tool_result)- observation = f"Observation: {tool_result.content}"+ if tool_result.success:+ observation = f"Observation: {tool_result.content}"+ else:+ observation = (+ f"Observation: Tool '{tool_result.tool_name}' failed: "+ f"{tool_result.content}"+ )messages.append(Message(role=Role.USER, content=observation))continue@@ -171,32 +177,39 @@ class OrchestratorAgent(ToolUsingAgent):tool_name: str,raw_input: str,) -> str:- """Map an unambiguous string input to the tool's JSON parameter."""+ """Map unambiguous structured text input to a string parameter."""if not raw_input:return "{}"try:parsed_input = json.loads(raw_input)except json.JSONDecodeError:- parsed_input = raw_input-- if isinstance(parsed_input, dict):- return raw_input- if not isinstance(parsed_input, str):+ invalid_json = True+ string_value = raw_input+ else:+ invalid_json = False+ if isinstance(parsed_input, dict):+ return raw_input+ # INPUT is a text protocol. A non-object JSON value such as 42,+ # true, null, or [1, 2] may still be the intended text for a tool's+ # string parameter. Quoted JSON strings are decoded to remove only+ # their surrounding quotes; other values retain their source text.+ string_value = parsed_input if isinstance(parsed_input, str) else raw_input++ tool_spec = None+ for candidate in reversed(self._tools):+ candidate_spec = candidate.spec+ if candidate_spec.name == tool_name:+ tool_spec = candidate_spec+ break+ if tool_spec is None:return raw_input- tool = next(- (- candidate- for candidate in self._tools- if candidate.spec.name == tool_name- ),- None,- )- if tool is None:+ parameters = tool_spec.parameters+ parameter_container_type = parameters.get("type")+ if parameter_container_type not in (None, "object"):return raw_input- parameters = tool.spec.parametersproperties = parameters.get("properties", {})required = parameters.get("required", [])if not isinstance(properties, dict) or not isinstance(required, list):@@ -210,13 +223,23 @@ class OrchestratorAgent(ToolUsingAgent):return raw_inputparameter_schema = properties[parameter_name]- if isinstance(parameter_schema, dict) and parameter_schema.get("type") not in (- None,- "string",- ):+ if not isinstance(parameter_schema, dict):+ return raw_input+ parameter_type = parameter_schema.get("type")+ accepts_string = parameter_type == "string" or (+ isinstance(parameter_type, list) and "string" in parameter_type+ )+ if not accepts_string:+ return raw_input++ allow_object_text = (+ tool_spec.metadata.get("structured_allow_object_text") is True+ )+ starts_like_object = raw_input.lstrip("\ufeff \t\r\n").startswith("{")+ if invalid_json and starts_like_object and not allow_object_text:return raw_input- return json.dumps({parameter_name: parsed_input})+ return json.dumps({parameter_name: string_value})@staticmethoddef _parse_structured_response(text: str) -> dict:diff --git a/src/openjarvis/tools/_stubs.py b/src/openjarvis/tools/_stubs.pyindex 4eb2bea6..b355bb26 100644--- a/src/openjarvis/tools/_stubs.py+++ b/src/openjarvis/tools/_stubs.py@@ -136,6 +136,15 @@ class ToolExecutor:content=f"Invalid arguments JSON: {exc}",success=False,)+ if not isinstance(params, dict):+ return ToolResult(+ tool_name=tool_call.name,+ content=(+ "Invalid arguments: expected a JSON object, "+ f"got {type(params).__name__}."+ ),+ success=False,+ )# Boundary guard: scan external tool argumentsif self._boundary_guard is not None and not getattr(tool, "is_local", True):@@ -143,6 +152,15 @@ class ToolExecutor:tool_call = self._boundary_guard.check_outbound(tool_call)# Re-parse arguments after potential redactionparams = json.loads(tool_call.arguments) if tool_call.arguments else {}+ if not isinstance(params, dict):+ return ToolResult(+ tool_name=tool_call.name,+ content=(+ "Invalid arguments: expected a JSON object, "+ f"got {type(params).__name__}."+ ),+ success=False,+ )except Exception as exc:return ToolResult(tool_name=tool_call.name,diff --git a/src/openjarvis/tools/code_interpreter.py b/src/openjarvis/tools/code_interpreter.pyindex a5d0e3cc..4bc52347 100644--- a/src/openjarvis/tools/code_interpreter.py+++ b/src/openjarvis/tools/code_interpreter.py@@ -56,6 +56,7 @@ class CodeInterpreterTool(BaseTool):"required": ["code"],},category="code",+ metadata={"structured_allow_object_text": True},)def execute(self, **params: Any) -> ToolResult:diff --git a/src/openjarvis/tools/code_interpreter_docker.py b/src/openjarvis/tools/code_interpreter_docker.pyindex c2147b65..2e53d644 100644--- a/src/openjarvis/tools/code_interpreter_docker.py+++ b/src/openjarvis/tools/code_interpreter_docker.py@@ -55,6 +55,7 @@ class DockerCodeInterpreterTool(BaseTool):},category="code",timeout_seconds=60.0,+ metadata={"structured_allow_object_text": True},)def execute(self, **params: Any) -> ToolResult:diff --git a/src/openjarvis/tools/repl.py b/src/openjarvis/tools/repl.pyindex 1394c046..1e83a669 100644--- a/src/openjarvis/tools/repl.py+++ b/src/openjarvis/tools/repl.py@@ -191,6 +191,7 @@ class ReplTool(BaseTool):"required": ["code"],},category="code",+ metadata={"structured_allow_object_text": True},)def execute(self, **params: Any) -> ToolResult:diff --git a/tests/test_orchestrator_learning/test_orchestrator_structured.py b/tests/test_orchestrator_learning/test_orchestrator_structured.pyindex e6578496..b438c70a 100644--- a/tests/test_orchestrator_learning/test_orchestrator_structured.py+++ b/tests/test_orchestrator_learning/test_orchestrator_structured.py@@ -20,8 +20,10 @@ class _MockEngine(InferenceEngine):def __init__(self, responses: list[str]) -> None:self._responses = list(responses)self._call_idx = 0+ self.calls = []def generate(self, messages, **kwargs) -> dict:+ self.calls.append(list(messages))if self._call_idx < len(self._responses):content = self._responses[self._call_idx]self._call_idx += 1@@ -112,6 +114,57 @@ class _AmbiguousTool(BaseTool):return ToolResult(tool_name="copy", content="copied", success=True)+class _CodeLikeTool(BaseTool):+ """Test double that explicitly accepts object-prefixed source text."""++ tool_id = "code"++ @property+ def spec(self) -> ToolSpec:+ return ToolSpec(+ name="code",+ description="Execute source code",+ parameters={+ "type": "object",+ "properties": {"code": {"type": "string"}},+ "required": ["code"],+ },+ metadata={"structured_allow_object_text": True},+ )++ def execute(self, **params) -> ToolResult:+ return ToolResult(+ tool_name="code",+ content=params.get("code", ""),+ success=True,+ )+++class _UnionStringTool(BaseTool):+ """Test double with a JSON Schema union that accepts strings."""++ tool_id = "union_file_read"++ @property+ def spec(self) -> ToolSpec:+ return ToolSpec(+ name="union_file_read",+ description="Read a nullable path",+ parameters={+ "type": "object",+ "properties": {"path": {"type": ["string", "null"]}},+ "required": ["path"],+ },+ )++ def execute(self, **params) -> ToolResult:+ return ToolResult(+ tool_name="union_file_read",+ content=params.get("path", ""),+ success=True,+ )++# -- Tests -------------------------------------------------------------------@@ -137,16 +190,63 @@ class TestStructuredMode:assert result.tool_results[0].success is Trueassert result.tool_results[0].content == "2+2"+ @pytest.mark.parametrize(+ ("raw_input", "expected"),+ [+ ("notes/today.md", "notes/today.md"),+ ('"notes/today.md"', "notes/today.md"),+ ('{"path": "notes/today.md"}', "notes/today.md"),+ ("42", "42"),+ ("true", "true"),+ ("false", "false"),+ ("null", "null"),+ ("1e3", "1e3"),+ ('["notes/today.md"]', '["notes/today.md"]'),+ ('["notes/today.md",]', '["notes/today.md",]'),+ ("[draft] notes.md", "[draft] notes.md"),+ (+ "[x * 2 for x in range(3)]",+ "[x * 2 for x in range(3)]",+ ),+ ('"notes/today.md', '"notes/today.md'),+ ('""', ""),+ ('"{draft} notes.md"', "{draft} notes.md"),+ (r'"C:\\Users\\me\\notes.txt"', r"C:\Users\me\notes.txt"),+ ],+ )+ def test_single_string_parameter_accepts_text_input(self, raw_input, expected):+ """Structured text maps to the unambiguous string parameter."""+ engine = _MockEngine(+ [+ f"TOOL: file_read\nINPUT: {raw_input}",+ "FINAL_ANSWER: done",+ ]+ )+ agent = OrchestratorAgent(+ engine=engine,+ model="test",+ tools=[_FileReadLikeTool()],+ mode="structured",+ )++ result = agent.run("Read my notes")++ assert result.tool_results[0].success is True+ assert result.tool_results[0].content == expected+@pytest.mark.parametrize("raw_input",[- "notes/today.md",- '"notes/today.md"',- '{"path": "notes/today.md"}',+ '{"path": "notes/today.md",}',+ '{path: "notes/today.md"}',+ "{'path': 'notes/today.md'}",+ '{"unknown": 1, "path": "notes/today.md",}',+ r'{"pa\u0074h": "notes/today.md",}',+ '\ufeff{"path": "notes/today.md",}',],)- def test_single_required_parameter_accepts_string_input(self, raw_input):- """Structured input maps unambiguous strings to the required field."""+ def test_malformed_json_like_input_remains_an_argument_error(self, raw_input):+ """Malformed JSON-looking text is not reclassified as a tool value."""engine = _MockEngine([f"TOOL: file_read\nINPUT: {raw_input}",@@ -162,14 +262,118 @@ class TestStructuredMode:result = agent.run("Read my notes")+ assert result.tool_results[0].success is False+ assert "Invalid arguments JSON" in result.tool_results[0].content+ assert "Tool 'file_read' failed: Invalid arguments JSON" in (+ engine.calls[1][-1].content+ )++ @pytest.mark.parametrize(+ "raw_input",+ [+ "{'code': value}",+ '{"code": object()}',+ "{'nested': {'value': 1}}",+ ],+ )+ def test_opted_in_tool_accepts_object_prefixed_text(self, raw_input):+ """Explicit raw-text metadata disambiguates dict-shaped source code."""+ engine = _MockEngine(+ [+ f"TOOL: code\nINPUT: {raw_input}",+ "FINAL_ANSWER: done",+ ]+ )+ agent = OrchestratorAgent(+ engine=engine,+ model="test",+ tools=[_CodeLikeTool()],+ mode="structured",+ )++ result = agent.run("Execute code")++ assert result.tool_results[0].success is True+ assert result.tool_results[0].content == raw_input++ def test_valid_object_remains_arguments_for_opted_in_tool(self):+ """Raw-text metadata does not override valid JSON argument objects."""+ engine = _MockEngine(+ [+ 'TOOL: code\nINPUT: {"code": "print(1)"}',+ "FINAL_ANSWER: done",+ ]+ )+ agent = OrchestratorAgent(+ engine=engine,+ model="test",+ tools=[_CodeLikeTool()],+ mode="structured",+ )++ result = agent.run("Execute code")++ assert result.tool_results[0].success is True+ assert result.tool_results[0].content == "print(1)"++ def test_union_string_schema_accepts_json_scalar_text(self):+ """String unions normalize text that also parses as a JSON scalar."""+ engine = _MockEngine(+ [+ "TOOL: union_file_read\nINPUT: null",+ "FINAL_ANSWER: done",+ ]+ )+ agent = OrchestratorAgent(+ engine=engine,+ model="test",+ tools=[_UnionStringTool()],+ mode="structured",+ )++ result = agent.run("Read the path named null")+assert result.tool_results[0].success is True- assert result.tool_results[0].content == "notes/today.md"+ assert result.tool_results[0].content == "null"++ def test_ambiguous_json_scalar_gets_stable_object_error(self):+ """Ambiguous valid JSON is rejected before tool dispatch."""+ engine = _MockEngine(+ [+ "TOOL: copy\nINPUT: 42",+ "FINAL_ANSWER: done",+ ]+ )+ agent = OrchestratorAgent(+ engine=engine,+ model="test",+ tools=[_AmbiguousTool()],+ mode="structured",+ )++ result = agent.run("Copy my notes")++ assert result.tool_results[0].success is False+ assert result.tool_results[0].content == (+ "Invalid arguments: expected a JSON object, got int."+ )+ assert "Tool 'copy' failed: Invalid arguments" in (engine.calls[1][-1].content)- def test_multiple_required_parameters_do_not_guess_string_mapping(self):+ @pytest.mark.parametrize(+ "raw_input",+ [+ "notes/today.md",+ '{"source": "notes/today.md",}',+ ],+ )+ def test_multiple_required_parameters_do_not_guess_string_mapping(+ self,+ raw_input,+ ):"""Ambiguous bare input remains invalid instead of choosing a field."""engine = _MockEngine([- "TOOL: copy\nINPUT: notes/today.md",+ f"TOOL: copy\nINPUT: {raw_input}","FINAL_ANSWER: done",])diff --git a/tests/tools/test_code_interpreter.py b/tests/tools/test_code_interpreter.pyindex 2f5a3acc..be2c4eac 100644--- a/tests/tools/test_code_interpreter.py+++ b/tests/tools/test_code_interpreter.py@@ -127,6 +127,10 @@ class TestCodeInterpreterTool:tool = CodeInterpreterTool()assert tool.tool_id == "code_interpreter"+ def test_structured_object_text_opt_in(self):+ tool = CodeInterpreterTool()+ assert tool.spec.metadata["structured_allow_object_text"] is True+def test_registry_registration(self):ToolRegistry.register_value("code_interpreter", CodeInterpreterTool)assert ToolRegistry.contains("code_interpreter")diff --git a/tests/tools/test_code_interpreter_docker.py b/tests/tools/test_code_interpreter_docker.pyindex a4dee212..f0aede79 100644--- a/tests/tools/test_code_interpreter_docker.py+++ b/tests/tools/test_code_interpreter_docker.py@@ -25,6 +25,7 @@ class TestDockerCodeInterpreterTool:assert spec.name == "code_interpreter_docker"assert "code" in spec.parameters["properties"]assert spec.category == "code"+ assert spec.metadata["structured_allow_object_text"] is Truedef test_empty_code(self):from openjarvis.tools.code_interpreter_docker import (diff --git a/tests/tools/test_repl.py b/tests/tools/test_repl.pyindex 3029b287..7ca80982 100644--- a/tests/tools/test_repl.py+++ b/tests/tools/test_repl.py@@ -17,6 +17,10 @@ class TestReplSpec:tool = ReplTool()assert tool.spec.category == "code"+ def test_structured_object_text_opt_in(self):+ tool = ReplTool()+ assert tool.spec.metadata["structured_allow_object_text"] is True+def test_spec_parameters(self):tool = ReplTool()params = tool.spec.parametersdiff --git a/tests/tools/test_stubs.py b/tests/tools/test_stubs.pyindex 528cb77e..377d1566 100644--- a/tests/tools/test_stubs.py+++ b/tests/tools/test_stubs.py@@ -2,6 +2,10 @@from __future__ import annotations+import json++import pytest+from openjarvis.core.events import EventBus, EventTypefrom openjarvis.core.types import ToolCall, ToolResultfrom openjarvis.tools._stubs import BaseTool, ToolExecutor, ToolSpec@@ -50,6 +54,17 @@ class _ErrorTool(BaseTool):raise RuntimeError("boom")+class _ScalarBoundaryGuard:+ """Test guard that rewrites outbound arguments to a JSON scalar."""++ def check_outbound(self, tool_call: ToolCall) -> ToolCall:+ return ToolCall(+ id=tool_call.id,+ name=tool_call.name,+ arguments=json.dumps("redacted"),+ )++# ---------------------------------------------------------------------------# ToolSpec tests# ---------------------------------------------------------------------------@@ -134,6 +149,38 @@ class TestToolExecutor:assert result.success is Falseassert "Invalid arguments JSON" in result.content+ @pytest.mark.parametrize(+ ("arguments", "decoded_type"),+ [+ ("42", "int"),+ ("true", "bool"),+ ("null", "NoneType"),+ ("[]", "list"),+ ('"text"', "str"),+ ],+ )+ def test_execute_rejects_non_object_json(self, arguments, decoded_type):+ executor = ToolExecutor([_EchoTool()])+ call = ToolCall(id="1", name="echo", arguments=arguments)++ result = executor.execute(call)++ assert result.success is False+ assert result.content == (+ f"Invalid arguments: expected a JSON object, got {decoded_type}."+ )++ def test_execute_revalidates_boundary_guard_arguments(self):+ tool = _EchoTool()+ tool.is_local = False+ executor = ToolExecutor([tool], boundary_guard=_ScalarBoundaryGuard())+ call = ToolCall(id="1", name="echo", arguments='{"text":"safe"}')++ result = executor.execute(call)++ assert result.success is False+ assert result.content == ("Invalid arguments: expected a JSON object, got str.")+def test_execute_empty_arguments(self):executor = ToolExecutor([_EchoTool()])call = ToolCall(id="1", name="echo", arguments="")