Skip to content

Personal AI, On Personal Devices

Python

fix(evals): ToolCall-15 JSON parsing + traces thread safety (#172)

Avanika Narayan908e28d42e5d
diff --git a/src/openjarvis/evals/backends/jarvis_agent.py b/src/openjarvis/evals/backends/jarvis_agent.py
index 61c70654..f0facd46 100644
--- a/src/openjarvis/evals/backends/jarvis_agent.py
+++ b/src/openjarvis/evals/backends/jarvis_agent.py
@@ -45,7 +45,7 @@ class JarvisAgentBackend(InferenceBackend):
# creates a GpuMonitor when building the InstrumentedEngine.
if gpu_metrics:
builder._config.telemetry.gpu_metrics = True
- self._system = builder.telemetry(telemetry).traces(True).build()
+ self._system = builder.telemetry(telemetry).traces(telemetry).build()
def generate(
self,
diff --git a/src/openjarvis/evals/datasets/toolcall15.py b/src/openjarvis/evals/datasets/toolcall15.py
index 4b2225c1..fbd6c567 100644
--- a/src/openjarvis/evals/datasets/toolcall15.py
+++ b/src/openjarvis/evals/datasets/toolcall15.py
@@ -639,24 +639,45 @@ class ToolCall15Dataset(DatasetProvider):
if max_samples is not None:
scenarios = scenarios[:max_samples]
- self._records = [
- EvalRecord(
- record_id=s["id"],
- problem=s["user_message"],
- reference="", # scoring uses metadata, not reference text
- category=s["category"],
- subject=s["name"],
- metadata={
- "system_prompt": SYSTEM_PROMPT,
- "tools": TOOLS,
- "mock_tool_outputs": s["mock_tool_outputs"],
- "reference_date": s.get("reference_date"),
- "scenario_id": s["id"],
- "scenario_name": s["name"],
- },
+ self._records = []
+ for s in scenarios:
+ # Build a self-contained prompt that includes the system
+ # instructions, available tools, and user message so the
+ # model can respond with tool calls via any backend.
+ tool_descriptions = "\n".join(
+ f"- {t['function']['name']}: "
+ f"{t['function']['description']}"
+ for t in TOOLS
+ )
+ prompt = (
+ f"{SYSTEM_PROMPT}\n\n"
+ f"## Available Tools\n"
+ f"{tool_descriptions}\n\n"
+ f"When you need to use a tool, respond with ONLY "
+ f"a JSON object in this format:\n"
+ f'{{"tool": "<tool_name>", "arguments": {{...}}}}\n\n'
+ f"If the task requires multiple tools, call them "
+ f"one at a time. If no tool is needed, respond "
+ f"directly with your answer.\n\n"
+ f"## User Request\n{s['user_message']}"
+ )
+ self._records.append(
+ EvalRecord(
+ record_id=s["id"],
+ problem=prompt,
+ reference="",
+ category=s["category"],
+ subject=s["name"],
+ metadata={
+ "system_prompt": SYSTEM_PROMPT,
+ "tools": TOOLS,
+ "mock_tool_outputs": s["mock_tool_outputs"],
+ "reference_date": s.get("reference_date"),
+ "scenario_id": s["id"],
+ "scenario_name": s["name"],
+ },
+ )
)
- for s in scenarios
- ]
LOGGER.info("ToolCall-15: loaded %d scenarios", len(self._records))
diff --git a/src/openjarvis/evals/scorers/toolcall15.py b/src/openjarvis/evals/scorers/toolcall15.py
index 40f72288..b5f06ce6 100644
--- a/src/openjarvis/evals/scorers/toolcall15.py
+++ b/src/openjarvis/evals/scorers/toolcall15.py
@@ -21,11 +21,17 @@ from openjarvis.evals.core.types import EvalRecord
LOGGER = logging.getLogger(__name__)
-def _extract_tool_calls(record: EvalRecord) -> List[Dict[str, Any]]:
- """Extract tool calls from the record's query trace or metadata.
+def _extract_tool_calls(
+ record: EvalRecord,
+ model_answer: str = "",
+) -> List[Dict[str, Any]]:
+ """Extract tool calls from traces, metadata, or model text output.
Returns a list of dicts with keys: name, arguments.
"""
+ import json
+ import re
+
tool_calls: List[Dict[str, Any]] = []
# Try query trace first (from EvalRunner)
@@ -39,7 +45,8 @@ def _extract_tool_calls(record: EvalRecord) -> List[Dict[str, Any]]:
"name": tc.get("name", ""),
"arguments": tc.get("arguments") or {},
})
- return tool_calls
+ if tool_calls:
+ return tool_calls
# Try tool_results list (from JarvisAgentBackend)
tool_results = record.metadata.get("tool_results", [])
@@ -48,6 +55,48 @@ def _extract_tool_calls(record: EvalRecord) -> List[Dict[str, Any]]:
"name": tr.get("tool_name", ""),
"arguments": tr.get("arguments") or {},
})
+ if tool_calls:
+ return tool_calls
+
+ # Parse tool calls from model text output (JSON format)
+ if model_answer:
+ # Extract balanced JSON objects from text
+ json_blocks: list[str] = []
+ # Try ```json blocks first
+ for m in re.finditer(
+ r"```(?:json)?\s*(\{.+?\})\s*```", model_answer, re.DOTALL
+ ):
+ json_blocks.append(m.group(1))
+ # Also extract bare balanced-brace JSON objects
+ depth = 0
+ start = -1
+ for i, ch in enumerate(model_answer):
+ if ch == "{":
+ if depth == 0:
+ start = i
+ depth += 1
+ elif ch == "}":
+ depth -= 1
+ if depth == 0 and start >= 0:
+ candidate = model_answer[start : i + 1]
+ if '"tool"' in candidate or '"name"' in candidate:
+ json_blocks.append(candidate)
+ start = -1
+
+ for block in json_blocks:
+ try:
+ parsed = json.loads(block)
+ name = parsed.get("tool") or parsed.get("name", "")
+ args = parsed.get("arguments") or parsed.get("params") or {}
+ if isinstance(args, str):
+ try:
+ args = json.loads(args)
+ except (json.JSONDecodeError, TypeError):
+ args = {}
+ if name:
+ tool_calls.append({"name": name, "arguments": args})
+ except json.JSONDecodeError:
+ continue
return tool_calls
@@ -500,7 +549,7 @@ class ToolCall15Scorer(LLMJudgeScorer):
"scenario_id": scenario_id,
}
- tool_calls = _extract_tool_calls(record)
+ tool_calls = _extract_tool_calls(record, model_answer)
points, reason = scorer_fn(tool_calls, model_answer)