Personal AI, On Personal Devices
Python
fix(evals): ToolCall-15 JSON parsing + traces thread safety (#172)
Avanika Narayan908e28d42e5d
diff --git a/src/openjarvis/evals/backends/jarvis_agent.py b/src/openjarvis/evals/backends/jarvis_agent.pyindex 61c70654..f0facd46 100644--- a/src/openjarvis/evals/backends/jarvis_agent.py+++ b/src/openjarvis/evals/backends/jarvis_agent.py@@ -45,7 +45,7 @@ class JarvisAgentBackend(InferenceBackend):# creates a GpuMonitor when building the InstrumentedEngine.if gpu_metrics:builder._config.telemetry.gpu_metrics = True- self._system = builder.telemetry(telemetry).traces(True).build()+ self._system = builder.telemetry(telemetry).traces(telemetry).build()def generate(self,diff --git a/src/openjarvis/evals/datasets/toolcall15.py b/src/openjarvis/evals/datasets/toolcall15.pyindex 4b2225c1..fbd6c567 100644--- a/src/openjarvis/evals/datasets/toolcall15.py+++ b/src/openjarvis/evals/datasets/toolcall15.py@@ -639,24 +639,45 @@ class ToolCall15Dataset(DatasetProvider):if max_samples is not None:scenarios = scenarios[:max_samples]- self._records = [- EvalRecord(- record_id=s["id"],- problem=s["user_message"],- reference="", # scoring uses metadata, not reference text- category=s["category"],- subject=s["name"],- metadata={- "system_prompt": SYSTEM_PROMPT,- "tools": TOOLS,- "mock_tool_outputs": s["mock_tool_outputs"],- "reference_date": s.get("reference_date"),- "scenario_id": s["id"],- "scenario_name": s["name"],- },+ self._records = []+ for s in scenarios:+ # Build a self-contained prompt that includes the system+ # instructions, available tools, and user message so the+ # model can respond with tool calls via any backend.+ tool_descriptions = "\n".join(+ f"- {t['function']['name']}: "+ f"{t['function']['description']}"+ for t in TOOLS+ )+ prompt = (+ f"{SYSTEM_PROMPT}\n\n"+ f"## Available Tools\n"+ f"{tool_descriptions}\n\n"+ f"When you need to use a tool, respond with ONLY "+ f"a JSON object in this format:\n"+ f'{{"tool": "<tool_name>", "arguments": {{...}}}}\n\n'+ f"If the task requires multiple tools, call them "+ f"one at a time. If no tool is needed, respond "+ f"directly with your answer.\n\n"+ f"## User Request\n{s['user_message']}"+ )+ self._records.append(+ EvalRecord(+ record_id=s["id"],+ problem=prompt,+ reference="",+ category=s["category"],+ subject=s["name"],+ metadata={+ "system_prompt": SYSTEM_PROMPT,+ "tools": TOOLS,+ "mock_tool_outputs": s["mock_tool_outputs"],+ "reference_date": s.get("reference_date"),+ "scenario_id": s["id"],+ "scenario_name": s["name"],+ },+ ))- for s in scenarios- ]LOGGER.info("ToolCall-15: loaded %d scenarios", len(self._records))diff --git a/src/openjarvis/evals/scorers/toolcall15.py b/src/openjarvis/evals/scorers/toolcall15.pyindex 40f72288..b5f06ce6 100644--- a/src/openjarvis/evals/scorers/toolcall15.py+++ b/src/openjarvis/evals/scorers/toolcall15.py@@ -21,11 +21,17 @@ from openjarvis.evals.core.types import EvalRecordLOGGER = logging.getLogger(__name__)-def _extract_tool_calls(record: EvalRecord) -> List[Dict[str, Any]]:- """Extract tool calls from the record's query trace or metadata.+def _extract_tool_calls(+ record: EvalRecord,+ model_answer: str = "",+) -> List[Dict[str, Any]]:+ """Extract tool calls from traces, metadata, or model text output.Returns a list of dicts with keys: name, arguments."""+ import json+ import re+tool_calls: List[Dict[str, Any]] = []# Try query trace first (from EvalRunner)@@ -39,7 +45,8 @@ def _extract_tool_calls(record: EvalRecord) -> List[Dict[str, Any]]:"name": tc.get("name", ""),"arguments": tc.get("arguments") or {},})- return tool_calls+ if tool_calls:+ return tool_calls# Try tool_results list (from JarvisAgentBackend)tool_results = record.metadata.get("tool_results", [])@@ -48,6 +55,48 @@ def _extract_tool_calls(record: EvalRecord) -> List[Dict[str, Any]]:"name": tr.get("tool_name", ""),"arguments": tr.get("arguments") or {},})+ if tool_calls:+ return tool_calls++ # Parse tool calls from model text output (JSON format)+ if model_answer:+ # Extract balanced JSON objects from text+ json_blocks: list[str] = []+ # Try ```json blocks first+ for m in re.finditer(+ r"```(?:json)?\s*(\{.+?\})\s*```", model_answer, re.DOTALL+ ):+ json_blocks.append(m.group(1))+ # Also extract bare balanced-brace JSON objects+ depth = 0+ start = -1+ for i, ch in enumerate(model_answer):+ if ch == "{":+ if depth == 0:+ start = i+ depth += 1+ elif ch == "}":+ depth -= 1+ if depth == 0 and start >= 0:+ candidate = model_answer[start : i + 1]+ if '"tool"' in candidate or '"name"' in candidate:+ json_blocks.append(candidate)+ start = -1++ for block in json_blocks:+ try:+ parsed = json.loads(block)+ name = parsed.get("tool") or parsed.get("name", "")+ args = parsed.get("arguments") or parsed.get("params") or {}+ if isinstance(args, str):+ try:+ args = json.loads(args)+ except (json.JSONDecodeError, TypeError):+ args = {}+ if name:+ tool_calls.append({"name": name, "arguments": args})+ except json.JSONDecodeError:+ continuereturn tool_calls@@ -500,7 +549,7 @@ class ToolCall15Scorer(LLMJudgeScorer):"scenario_id": scenario_id,}- tool_calls = _extract_tool_calls(record)+ tool_calls = _extract_tool_calls(record, model_answer)points, reason = scorer_fn(tool_calls, model_answer)