Skip to content

Personal AI, On Personal Devices

Python

feat(evals): add tool_calls field to TurnTrace for rich tool data

Jon Saad-Falconc5cb6a9e491d
diff --git a/src/openjarvis/evals/core/trace.py b/src/openjarvis/evals/core/trace.py
index 92b1f5ea..356f593b 100644
--- a/src/openjarvis/evals/core/trace.py
+++ b/src/openjarvis/evals/core/trace.py
@@ -28,6 +28,8 @@ class TurnTrace:
cost_usd: Optional[float] = None
# Per-action energy breakdown (lm_inference vs tool_call granularity)
action_energy_breakdown: Optional[List[Dict[str, Any]]] = None
+ # Detailed tool call data: [{"name": str, "arguments": dict, "result": str}]
+ tool_calls: List[Dict[str, Any]] = field(default_factory=list)
def to_dict(self) -> Dict[str, Any]:
return {
@@ -45,6 +47,7 @@ class TurnTrace:
"cpu_power_avg_watts": self.cpu_power_avg_watts,
"cost_usd": self.cost_usd,
"action_energy_breakdown": self.action_energy_breakdown,
+ "tool_calls": [dict(tc) for tc in self.tool_calls],
}
@classmethod
@@ -64,6 +67,7 @@ class TurnTrace:
cpu_power_avg_watts=d.get("cpu_power_avg_watts"),
cost_usd=d.get("cost_usd"),
action_energy_breakdown=d.get("action_energy_breakdown"),
+ tool_calls=d.get("tool_calls", []),
)
diff --git a/tests/evals/test_pinchbench_transcript.py b/tests/evals/test_pinchbench_transcript.py
new file mode 100644
index 00000000..7218216b
--- /dev/null
+++ b/tests/evals/test_pinchbench_transcript.py
@@ -0,0 +1,32 @@
+"""Tests for TurnTrace tool_calls field and PinchBench transcript translation."""
+
+from openjarvis.evals.core.trace import TurnTrace
+
+
+def test_turn_trace_tool_calls_default_empty():
+ """New tool_calls field defaults to empty list."""
+ turn = TurnTrace(turn_index=0)
+ assert turn.tool_calls == []
+
+
+def test_turn_trace_tool_calls_to_dict():
+ """tool_calls round-trips through to_dict/from_dict."""
+ calls = [{"name": "file_read", "arguments": {"path": "a.txt"}, "result": "hello"}]
+ turn = TurnTrace(turn_index=0, tool_calls=calls)
+ d = turn.to_dict()
+ assert d["tool_calls"] == calls
+
+
+def test_turn_trace_tool_calls_from_dict():
+ """from_dict restores tool_calls."""
+ calls = [{"name": "web_search", "arguments": {"q": "test"}, "result": "results"}]
+ d = {"turn_index": 0, "tool_calls": calls}
+ turn = TurnTrace.from_dict(d)
+ assert turn.tool_calls == calls
+
+
+def test_turn_trace_tool_calls_from_dict_missing():
+ """from_dict with missing tool_calls defaults to empty list."""
+ d = {"turn_index": 0}
+ turn = TurnTrace.from_dict(d)
+ assert turn.tool_calls == []