Skip to content

Personal AI, On Personal Devices

Python

fix(agents): harden structured tool input normalization

Elliot Slusky0d32784ed63d
diff --git a/src/openjarvis/agents/orchestrator.py b/src/openjarvis/agents/orchestrator.py
index d97cceb1..352aadbd 100644
--- a/src/openjarvis/agents/orchestrator.py
+++ b/src/openjarvis/agents/orchestrator.py
@@ -151,7 +151,13 @@ class OrchestratorAgent(ToolUsingAgent):
tool_result = self._executor.execute(tool_call)
all_tool_results.append(tool_result)
- observation = f"Observation: {tool_result.content}"
+ if tool_result.success:
+ observation = f"Observation: {tool_result.content}"
+ else:
+ observation = (
+ f"Observation: Tool '{tool_result.tool_name}' failed: "
+ f"{tool_result.content}"
+ )
messages.append(Message(role=Role.USER, content=observation))
continue
@@ -171,32 +177,39 @@ class OrchestratorAgent(ToolUsingAgent):
tool_name: str,
raw_input: str,
) -> str:
- """Map an unambiguous string input to the tool's JSON parameter."""
+ """Map unambiguous structured text input to a string parameter."""
if not raw_input:
return "{}"
try:
parsed_input = json.loads(raw_input)
except json.JSONDecodeError:
- parsed_input = raw_input
-
- if isinstance(parsed_input, dict):
- return raw_input
- if not isinstance(parsed_input, str):
+ invalid_json = True
+ string_value = raw_input
+ else:
+ invalid_json = False
+ if isinstance(parsed_input, dict):
+ return raw_input
+ # INPUT is a text protocol. A non-object JSON value such as 42,
+ # true, null, or [1, 2] may still be the intended text for a tool's
+ # string parameter. Quoted JSON strings are decoded to remove only
+ # their surrounding quotes; other values retain their source text.
+ string_value = parsed_input if isinstance(parsed_input, str) else raw_input
+
+ tool_spec = None
+ for candidate in reversed(self._tools):
+ candidate_spec = candidate.spec
+ if candidate_spec.name == tool_name:
+ tool_spec = candidate_spec
+ break
+ if tool_spec is None:
return raw_input
- tool = next(
- (
- candidate
- for candidate in self._tools
- if candidate.spec.name == tool_name
- ),
- None,
- )
- if tool is None:
+ parameters = tool_spec.parameters
+ parameter_container_type = parameters.get("type")
+ if parameter_container_type not in (None, "object"):
return raw_input
- parameters = tool.spec.parameters
properties = parameters.get("properties", {})
required = parameters.get("required", [])
if not isinstance(properties, dict) or not isinstance(required, list):
@@ -210,13 +223,23 @@ class OrchestratorAgent(ToolUsingAgent):
return raw_input
parameter_schema = properties[parameter_name]
- if isinstance(parameter_schema, dict) and parameter_schema.get("type") not in (
- None,
- "string",
- ):
+ if not isinstance(parameter_schema, dict):
+ return raw_input
+ parameter_type = parameter_schema.get("type")
+ accepts_string = parameter_type == "string" or (
+ isinstance(parameter_type, list) and "string" in parameter_type
+ )
+ if not accepts_string:
+ return raw_input
+
+ allow_object_text = (
+ tool_spec.metadata.get("structured_allow_object_text") is True
+ )
+ starts_like_object = raw_input.lstrip("\ufeff \t\r\n").startswith("{")
+ if invalid_json and starts_like_object and not allow_object_text:
return raw_input
- return json.dumps({parameter_name: parsed_input})
+ return json.dumps({parameter_name: string_value})
@staticmethod
def _parse_structured_response(text: str) -> dict:
diff --git a/src/openjarvis/tools/_stubs.py b/src/openjarvis/tools/_stubs.py
index 4eb2bea6..b355bb26 100644
--- a/src/openjarvis/tools/_stubs.py
+++ b/src/openjarvis/tools/_stubs.py
@@ -136,6 +136,15 @@ class ToolExecutor:
content=f"Invalid arguments JSON: {exc}",
success=False,
)
+ if not isinstance(params, dict):
+ return ToolResult(
+ tool_name=tool_call.name,
+ content=(
+ "Invalid arguments: expected a JSON object, "
+ f"got {type(params).__name__}."
+ ),
+ success=False,
+ )
# Boundary guard: scan external tool arguments
if self._boundary_guard is not None and not getattr(tool, "is_local", True):
@@ -143,6 +152,15 @@ class ToolExecutor:
tool_call = self._boundary_guard.check_outbound(tool_call)
# Re-parse arguments after potential redaction
params = json.loads(tool_call.arguments) if tool_call.arguments else {}
+ if not isinstance(params, dict):
+ return ToolResult(
+ tool_name=tool_call.name,
+ content=(
+ "Invalid arguments: expected a JSON object, "
+ f"got {type(params).__name__}."
+ ),
+ success=False,
+ )
except Exception as exc:
return ToolResult(
tool_name=tool_call.name,
diff --git a/src/openjarvis/tools/code_interpreter.py b/src/openjarvis/tools/code_interpreter.py
index a5d0e3cc..4bc52347 100644
--- a/src/openjarvis/tools/code_interpreter.py
+++ b/src/openjarvis/tools/code_interpreter.py
@@ -56,6 +56,7 @@ class CodeInterpreterTool(BaseTool):
"required": ["code"],
},
category="code",
+ metadata={"structured_allow_object_text": True},
)
def execute(self, **params: Any) -> ToolResult:
diff --git a/src/openjarvis/tools/code_interpreter_docker.py b/src/openjarvis/tools/code_interpreter_docker.py
index c2147b65..2e53d644 100644
--- a/src/openjarvis/tools/code_interpreter_docker.py
+++ b/src/openjarvis/tools/code_interpreter_docker.py
@@ -55,6 +55,7 @@ class DockerCodeInterpreterTool(BaseTool):
},
category="code",
timeout_seconds=60.0,
+ metadata={"structured_allow_object_text": True},
)
def execute(self, **params: Any) -> ToolResult:
diff --git a/src/openjarvis/tools/repl.py b/src/openjarvis/tools/repl.py
index 1394c046..1e83a669 100644
--- a/src/openjarvis/tools/repl.py
+++ b/src/openjarvis/tools/repl.py
@@ -191,6 +191,7 @@ class ReplTool(BaseTool):
"required": ["code"],
},
category="code",
+ metadata={"structured_allow_object_text": True},
)
def execute(self, **params: Any) -> ToolResult:
diff --git a/tests/test_orchestrator_learning/test_orchestrator_structured.py b/tests/test_orchestrator_learning/test_orchestrator_structured.py
index e6578496..b438c70a 100644
--- a/tests/test_orchestrator_learning/test_orchestrator_structured.py
+++ b/tests/test_orchestrator_learning/test_orchestrator_structured.py
@@ -20,8 +20,10 @@ class _MockEngine(InferenceEngine):
def __init__(self, responses: list[str]) -> None:
self._responses = list(responses)
self._call_idx = 0
+ self.calls = []
def generate(self, messages, **kwargs) -> dict:
+ self.calls.append(list(messages))
if self._call_idx < len(self._responses):
content = self._responses[self._call_idx]
self._call_idx += 1
@@ -112,6 +114,57 @@ class _AmbiguousTool(BaseTool):
return ToolResult(tool_name="copy", content="copied", success=True)
+class _CodeLikeTool(BaseTool):
+ """Test double that explicitly accepts object-prefixed source text."""
+
+ tool_id = "code"
+
+ @property
+ def spec(self) -> ToolSpec:
+ return ToolSpec(
+ name="code",
+ description="Execute source code",
+ parameters={
+ "type": "object",
+ "properties": {"code": {"type": "string"}},
+ "required": ["code"],
+ },
+ metadata={"structured_allow_object_text": True},
+ )
+
+ def execute(self, **params) -> ToolResult:
+ return ToolResult(
+ tool_name="code",
+ content=params.get("code", ""),
+ success=True,
+ )
+
+
+class _UnionStringTool(BaseTool):
+ """Test double with a JSON Schema union that accepts strings."""
+
+ tool_id = "union_file_read"
+
+ @property
+ def spec(self) -> ToolSpec:
+ return ToolSpec(
+ name="union_file_read",
+ description="Read a nullable path",
+ parameters={
+ "type": "object",
+ "properties": {"path": {"type": ["string", "null"]}},
+ "required": ["path"],
+ },
+ )
+
+ def execute(self, **params) -> ToolResult:
+ return ToolResult(
+ tool_name="union_file_read",
+ content=params.get("path", ""),
+ success=True,
+ )
+
+
# -- Tests -------------------------------------------------------------------
@@ -137,16 +190,63 @@ class TestStructuredMode:
assert result.tool_results[0].success is True
assert result.tool_results[0].content == "2+2"
+ @pytest.mark.parametrize(
+ ("raw_input", "expected"),
+ [
+ ("notes/today.md", "notes/today.md"),
+ ('"notes/today.md"', "notes/today.md"),
+ ('{"path": "notes/today.md"}', "notes/today.md"),
+ ("42", "42"),
+ ("true", "true"),
+ ("false", "false"),
+ ("null", "null"),
+ ("1e3", "1e3"),
+ ('["notes/today.md"]', '["notes/today.md"]'),
+ ('["notes/today.md",]', '["notes/today.md",]'),
+ ("[draft] notes.md", "[draft] notes.md"),
+ (
+ "[x * 2 for x in range(3)]",
+ "[x * 2 for x in range(3)]",
+ ),
+ ('"notes/today.md', '"notes/today.md'),
+ ('""', ""),
+ ('"{draft} notes.md"', "{draft} notes.md"),
+ (r'"C:\\Users\\me\\notes.txt"', r"C:\Users\me\notes.txt"),
+ ],
+ )
+ def test_single_string_parameter_accepts_text_input(self, raw_input, expected):
+ """Structured text maps to the unambiguous string parameter."""
+ engine = _MockEngine(
+ [
+ f"TOOL: file_read\nINPUT: {raw_input}",
+ "FINAL_ANSWER: done",
+ ]
+ )
+ agent = OrchestratorAgent(
+ engine=engine,
+ model="test",
+ tools=[_FileReadLikeTool()],
+ mode="structured",
+ )
+
+ result = agent.run("Read my notes")
+
+ assert result.tool_results[0].success is True
+ assert result.tool_results[0].content == expected
+
@pytest.mark.parametrize(
"raw_input",
[
- "notes/today.md",
- '"notes/today.md"',
- '{"path": "notes/today.md"}',
+ '{"path": "notes/today.md",}',
+ '{path: "notes/today.md"}',
+ "{'path': 'notes/today.md'}",
+ '{"unknown": 1, "path": "notes/today.md",}',
+ r'{"pa\u0074h": "notes/today.md",}',
+ '\ufeff{"path": "notes/today.md",}',
],
)
- def test_single_required_parameter_accepts_string_input(self, raw_input):
- """Structured input maps unambiguous strings to the required field."""
+ def test_malformed_json_like_input_remains_an_argument_error(self, raw_input):
+ """Malformed JSON-looking text is not reclassified as a tool value."""
engine = _MockEngine(
[
f"TOOL: file_read\nINPUT: {raw_input}",
@@ -162,14 +262,118 @@ class TestStructuredMode:
result = agent.run("Read my notes")
+ assert result.tool_results[0].success is False
+ assert "Invalid arguments JSON" in result.tool_results[0].content
+ assert "Tool 'file_read' failed: Invalid arguments JSON" in (
+ engine.calls[1][-1].content
+ )
+
+ @pytest.mark.parametrize(
+ "raw_input",
+ [
+ "{'code': value}",
+ '{"code": object()}',
+ "{'nested': {'value': 1}}",
+ ],
+ )
+ def test_opted_in_tool_accepts_object_prefixed_text(self, raw_input):
+ """Explicit raw-text metadata disambiguates dict-shaped source code."""
+ engine = _MockEngine(
+ [
+ f"TOOL: code\nINPUT: {raw_input}",
+ "FINAL_ANSWER: done",
+ ]
+ )
+ agent = OrchestratorAgent(
+ engine=engine,
+ model="test",
+ tools=[_CodeLikeTool()],
+ mode="structured",
+ )
+
+ result = agent.run("Execute code")
+
+ assert result.tool_results[0].success is True
+ assert result.tool_results[0].content == raw_input
+
+ def test_valid_object_remains_arguments_for_opted_in_tool(self):
+ """Raw-text metadata does not override valid JSON argument objects."""
+ engine = _MockEngine(
+ [
+ 'TOOL: code\nINPUT: {"code": "print(1)"}',
+ "FINAL_ANSWER: done",
+ ]
+ )
+ agent = OrchestratorAgent(
+ engine=engine,
+ model="test",
+ tools=[_CodeLikeTool()],
+ mode="structured",
+ )
+
+ result = agent.run("Execute code")
+
+ assert result.tool_results[0].success is True
+ assert result.tool_results[0].content == "print(1)"
+
+ def test_union_string_schema_accepts_json_scalar_text(self):
+ """String unions normalize text that also parses as a JSON scalar."""
+ engine = _MockEngine(
+ [
+ "TOOL: union_file_read\nINPUT: null",
+ "FINAL_ANSWER: done",
+ ]
+ )
+ agent = OrchestratorAgent(
+ engine=engine,
+ model="test",
+ tools=[_UnionStringTool()],
+ mode="structured",
+ )
+
+ result = agent.run("Read the path named null")
+
assert result.tool_results[0].success is True
- assert result.tool_results[0].content == "notes/today.md"
+ assert result.tool_results[0].content == "null"
+
+ def test_ambiguous_json_scalar_gets_stable_object_error(self):
+ """Ambiguous valid JSON is rejected before tool dispatch."""
+ engine = _MockEngine(
+ [
+ "TOOL: copy\nINPUT: 42",
+ "FINAL_ANSWER: done",
+ ]
+ )
+ agent = OrchestratorAgent(
+ engine=engine,
+ model="test",
+ tools=[_AmbiguousTool()],
+ mode="structured",
+ )
+
+ result = agent.run("Copy my notes")
+
+ assert result.tool_results[0].success is False
+ assert result.tool_results[0].content == (
+ "Invalid arguments: expected a JSON object, got int."
+ )
+ assert "Tool 'copy' failed: Invalid arguments" in (engine.calls[1][-1].content)
- def test_multiple_required_parameters_do_not_guess_string_mapping(self):
+ @pytest.mark.parametrize(
+ "raw_input",
+ [
+ "notes/today.md",
+ '{"source": "notes/today.md",}',
+ ],
+ )
+ def test_multiple_required_parameters_do_not_guess_string_mapping(
+ self,
+ raw_input,
+ ):
"""Ambiguous bare input remains invalid instead of choosing a field."""
engine = _MockEngine(
[
- "TOOL: copy\nINPUT: notes/today.md",
+ f"TOOL: copy\nINPUT: {raw_input}",
"FINAL_ANSWER: done",
]
)
diff --git a/tests/tools/test_code_interpreter.py b/tests/tools/test_code_interpreter.py
index 2f5a3acc..be2c4eac 100644
--- a/tests/tools/test_code_interpreter.py
+++ b/tests/tools/test_code_interpreter.py
@@ -127,6 +127,10 @@ class TestCodeInterpreterTool:
tool = CodeInterpreterTool()
assert tool.tool_id == "code_interpreter"
+ def test_structured_object_text_opt_in(self):
+ tool = CodeInterpreterTool()
+ assert tool.spec.metadata["structured_allow_object_text"] is True
+
def test_registry_registration(self):
ToolRegistry.register_value("code_interpreter", CodeInterpreterTool)
assert ToolRegistry.contains("code_interpreter")
diff --git a/tests/tools/test_code_interpreter_docker.py b/tests/tools/test_code_interpreter_docker.py
index a4dee212..f0aede79 100644
--- a/tests/tools/test_code_interpreter_docker.py
+++ b/tests/tools/test_code_interpreter_docker.py
@@ -25,6 +25,7 @@ class TestDockerCodeInterpreterTool:
assert spec.name == "code_interpreter_docker"
assert "code" in spec.parameters["properties"]
assert spec.category == "code"
+ assert spec.metadata["structured_allow_object_text"] is True
def test_empty_code(self):
from openjarvis.tools.code_interpreter_docker import (
diff --git a/tests/tools/test_repl.py b/tests/tools/test_repl.py
index 3029b287..7ca80982 100644
--- a/tests/tools/test_repl.py
+++ b/tests/tools/test_repl.py
@@ -17,6 +17,10 @@ class TestReplSpec:
tool = ReplTool()
assert tool.spec.category == "code"
+ def test_structured_object_text_opt_in(self):
+ tool = ReplTool()
+ assert tool.spec.metadata["structured_allow_object_text"] is True
+
def test_spec_parameters(self):
tool = ReplTool()
params = tool.spec.parameters
diff --git a/tests/tools/test_stubs.py b/tests/tools/test_stubs.py
index 528cb77e..377d1566 100644
--- a/tests/tools/test_stubs.py
+++ b/tests/tools/test_stubs.py
@@ -2,6 +2,10 @@
from __future__ import annotations
+import json
+
+import pytest
+
from openjarvis.core.events import EventBus, EventType
from openjarvis.core.types import ToolCall, ToolResult
from openjarvis.tools._stubs import BaseTool, ToolExecutor, ToolSpec
@@ -50,6 +54,17 @@ class _ErrorTool(BaseTool):
raise RuntimeError("boom")
+class _ScalarBoundaryGuard:
+ """Test guard that rewrites outbound arguments to a JSON scalar."""
+
+ def check_outbound(self, tool_call: ToolCall) -> ToolCall:
+ return ToolCall(
+ id=tool_call.id,
+ name=tool_call.name,
+ arguments=json.dumps("redacted"),
+ )
+
+
# ---------------------------------------------------------------------------
# ToolSpec tests
# ---------------------------------------------------------------------------
@@ -134,6 +149,38 @@ class TestToolExecutor:
assert result.success is False
assert "Invalid arguments JSON" in result.content
+ @pytest.mark.parametrize(
+ ("arguments", "decoded_type"),
+ [
+ ("42", "int"),
+ ("true", "bool"),
+ ("null", "NoneType"),
+ ("[]", "list"),
+ ('"text"', "str"),
+ ],
+ )
+ def test_execute_rejects_non_object_json(self, arguments, decoded_type):
+ executor = ToolExecutor([_EchoTool()])
+ call = ToolCall(id="1", name="echo", arguments=arguments)
+
+ result = executor.execute(call)
+
+ assert result.success is False
+ assert result.content == (
+ f"Invalid arguments: expected a JSON object, got {decoded_type}."
+ )
+
+ def test_execute_revalidates_boundary_guard_arguments(self):
+ tool = _EchoTool()
+ tool.is_local = False
+ executor = ToolExecutor([tool], boundary_guard=_ScalarBoundaryGuard())
+ call = ToolCall(id="1", name="echo", arguments='{"text":"safe"}')
+
+ result = executor.execute(call)
+
+ assert result.success is False
+ assert result.content == ("Invalid arguments: expected a JSON object, got str.")
+
def test_execute_empty_arguments(self):
executor = ToolExecutor([_EchoTool()])
call = ToolCall(id="1", name="echo", arguments="")