Personal AI, On Personal Devices
Python
fix(evals): PinchBench harness fixes — scores from 26% to 84% (#124)
Jon Saad-Falcon1d24c64f117b
diff --git a/src/openjarvis/agents/native_openhands.py b/src/openjarvis/agents/native_openhands.pyindex cf5579cd..9b518809 100644--- a/src/openjarvis/agents/native_openhands.py+++ b/src/openjarvis/agents/native_openhands.py@@ -283,14 +283,23 @@ class NativeOpenHandsAgent(ToolUsingAgent):last_content = ""total_usage = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}+ # Build OpenAI-format tool schemas for native function calling+ openai_tools = (+ self._executor.get_openai_tools() if self._tools else []+ )+for _turn in range(self._max_turns):turns += 1# Truncate before every generate call -- tool results may have# expanded the context beyond what the model supports.messages = self._truncate_if_needed(messages)+ gen_kwargs: dict[str, Any] = {}+ if openai_tools:+ gen_kwargs["tools"] = openai_tools+try:- result = self._generate(messages)+ result = self._generate(messages, **gen_kwargs)except Exception as exc:error_str = str(exc)if "400" in error_str:@@ -318,6 +327,42 @@ class NativeOpenHandsAgent(ToolUsingAgent):content = self._strip_think_tags(content)last_content = content+ # --- Native function-calling path (OpenAI, Anthropic, etc.) ---+ raw_tool_calls = result.get("tool_calls", [])+ if raw_tool_calls:+ native_calls = [+ ToolCall(+ id=tc.get("id", f"call_{turns}_{i}"),+ name=tc.get("name", ""),+ arguments=tc.get("arguments", "{}"),+ )+ for i, tc in enumerate(raw_tool_calls)+ ]+ messages.append(+ Message(+ role=Role.ASSISTANT,+ content=content,+ tool_calls=native_calls,+ )+ )+ for tc in native_calls:+ tool_result = self._executor.execute(tc)+ all_tool_results.append(tool_result)+ obs_text = tool_result.content+ if len(obs_text) > 4000:+ obs_text = obs_text[:4000] + "\n\n[Output truncated]"+ messages.append(+ Message(+ role=Role.TOOL,+ content=obs_text,+ tool_call_id=tc.id,+ name=tc.name,+ )+ )+ continue++ # --- Text-based fallback (CodeAct / Action-Input format) ---+# Try to extract codecode = self._extract_code(content)if code:diff --git a/src/openjarvis/evals/configs/pinchbench-gpt54.toml b/src/openjarvis/evals/configs/pinchbench-gpt54.tomlnew file mode 100644index 00000000..e1feee6a--- /dev/null+++ b/src/openjarvis/evals/configs/pinchbench-gpt54.toml@@ -0,0 +1,34 @@+# PinchBench eval: gpt-5.4 (cloud)+# Agent: native_openhands — all PinchBench-required tools enabled++[meta]+name = "pinchbench-gpt54"+description = "PinchBench on gpt-5.4-2026-03-05"++[defaults]+temperature = 0.6+max_tokens = 8192++[judge]+model = "claude-opus-4-5"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/pinchbench-gpt54/"+seed = 42++[[models]]+name = "gpt-5.4-2026-03-05"+engine = "cloud"++[[benchmarks]]+name = "pinchbench"+backend = "jarvis-agent"+agent = "native_openhands"+tools = [+ "think", "file_read", "file_write", "web_search", "shell_exec",+ "code_interpreter", "browser_navigate", "image_generate",+ "calculator", "http_request", "pdf_extract",+]diff --git a/src/openjarvis/evals/configs/pinchbench-qwen397b.toml b/src/openjarvis/evals/configs/pinchbench-qwen397b.tomlnew file mode 100644index 00000000..b6b1493b--- /dev/null+++ b/src/openjarvis/evals/configs/pinchbench-qwen397b.toml@@ -0,0 +1,35 @@+# PinchBench eval: Qwen3.5-397B-A17B-FP8 (vLLM, TP=8)+# Agent: native_openhands — all PinchBench-required tools enabled++[meta]+name = "pinchbench-qwen397b"+description = "PinchBench on Qwen/Qwen3.5-397B-A17B-FP8 (vLLM, TP=8)"++[defaults]+temperature = 0.6+max_tokens = 8192++[judge]+model = "claude-opus-4-5"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/pinchbench-qwen397b/"+seed = 42++[[models]]+name = "Qwen/Qwen3.5-397B-A17B-FP8"+engine = "vllm"+num_gpus = 8++[[benchmarks]]+name = "pinchbench"+backend = "jarvis-agent"+agent = "native_openhands"+tools = [+ "think", "file_read", "file_write", "web_search", "shell_exec",+ "code_interpreter", "browser_navigate", "image_generate",+ "calculator", "http_request", "pdf_extract",+]diff --git a/src/openjarvis/evals/core/config.py b/src/openjarvis/evals/core/config.pyindex ac62d191..ab864c71 100644--- a/src/openjarvis/evals/core/config.py+++ b/src/openjarvis/evals/core/config.py@@ -43,6 +43,7 @@ KNOWN_BENCHMARKS = {"knowledge_base", "coding_task","coding_assistant", "security_scanner", "daily_digest","doc_qa", "browser_assistant",+ "pinchbench",}diff --git a/src/openjarvis/evals/core/runner.py b/src/openjarvis/evals/core/runner.pyindex e3d8716e..9effbad2 100644--- a/src/openjarvis/evals/core/runner.py+++ b/src/openjarvis/evals/core/runner.py@@ -96,11 +96,19 @@ class EvalRunner:seed=cfg.seed,)- # Auto-enable episode_mode when the dataset has iter_episodes()- # (i.e. it is a lifelong/sequential benchmark like LifelongAgentBench).- # This is enforced at the runner level so it applies regardless of- # how the runner is invoked (CLI, SDK, tests, etc.).- if not cfg.episode_mode and hasattr(self._dataset, "iter_episodes"):+ # Auto-enable episode_mode when the dataset *overrides*+ # iter_episodes() (i.e. it is a lifelong/sequential benchmark like+ # LifelongAgentBench). The base DatasetProvider always defines a+ # default iter_episodes() that wraps each record in its own episode,+ # so hasattr() is always True — we must check for a real override.+ from openjarvis.evals.core.dataset import DatasetProvider as _DP+ try:+ _overrides_episodes = (+ type(self._dataset).iter_episodes is not _DP.iter_episodes+ )+ except AttributeError:+ _overrides_episodes = False+ if not cfg.episode_mode and _overrides_episodes:LOGGER.info("%s requires sequential episode processing — ""auto-enabling episode_mode.",@@ -109,6 +117,15 @@ class EvalRunner:cfg = dataclasses.replace(cfg, episode_mode=True)self._config = cfg+ # Detect if dataset provides task environments (e.g. PinchBench)+ try:+ self._has_task_env = (+ type(self._dataset).create_task_env+ is not _DP.create_task_env+ )+ except AttributeError:+ self._has_task_env = False+records = list(self._dataset.iter_records())LOGGER.info("Running %s: %d samples, backend=%s, model=%s, workers=%d, "@@ -145,6 +162,15 @@ class EvalRunner:try:if cfg.episode_mode:self._run_episode_mode(records, progress_callback, total)+ elif self._has_task_env:+ # Task environments (PinchBench etc.) change CWD —+ # must process sequentially for thread safety.+ for record in records:+ result = self._process_one(record)+ self._results.append(result)+ self._flush_result(result)+ if progress_callback is not None:+ progress_callback(len(self._results), total)else:with ThreadPoolExecutor(max_workers=cfg.max_workers) as pool:futures = {@@ -224,17 +250,41 @@ class EvalRunner:)if cfg.system_prompt:gen_kwargs["system"] = cfg.system_prompt- full = self._backend.generate_full(- record.problem,- **gen_kwargs,- )- content = full.get("content", "")++ if getattr(self, "_has_task_env", False):+ from contextlib import nullcontext+ task_env = self._dataset.create_task_env(record)+ ctx = task_env if task_env is not None else nullcontext()+ with ctx:+ full = self._backend.generate_full(+ record.problem,+ **gen_kwargs,+ )+ full = full or {}+ # Store tool results for the scorer to build transcripts+ record.metadata["tool_results"] = full.get(+ "tool_results", [],+ )+ # Score INSIDE context so workspace files still exist+ content = full.get("content", "")+ is_correct, scoring_meta = self._scorer.score(+ record, content,+ )+ else:+ full = self._backend.generate_full(+ record.problem,+ **gen_kwargs,+ )+ full = full or {}+ content = full.get("content", "")+ is_correct, scoring_meta = self._scorer.score(+ record, content,+ )+usage = full.get("usage", {})latency = full.get("latency_seconds", 0.0)cost = full.get("cost_usd", 0.0)- is_correct, scoring_meta = self._scorer.score(record, content)-energy_j = full.get("energy_joules", 0.0)power_w = full.get("power_watts", 0.0)throughput = full.get("throughput_tok_per_sec", 0.0)diff --git a/src/openjarvis/evals/scorers/pinchbench.py b/src/openjarvis/evals/scorers/pinchbench.pyindex 68269ec0..693f3d12 100644--- a/src/openjarvis/evals/scorers/pinchbench.py+++ b/src/openjarvis/evals/scorers/pinchbench.py@@ -52,12 +52,12 @@ def events_to_transcript(events: List[Any]) -> List[Dict[str, Any]]:if etype == EventType.TOOL_CALL_START or etype == EventType.TOOL_CALL_START.value:tool_name = event.metadata.get("tool", "unknown")mapped = _TOOL_NAME_MAP.get(tool_name, tool_name)- arguments = event.metadata.get("arguments", {})+ arguments = event.metadata.get("arguments") or {}transcript.append({"type": "message","message": {"role": "assistant",- "content": [{"type": "toolCall", "name": mapped, "arguments": arguments}],+ "content": [{"type": "toolCall", "name": mapped, "params": arguments}],},})elif etype == EventType.TOOL_CALL_END or etype == EventType.TOOL_CALL_END.value:@@ -78,12 +78,14 @@ def _trace_to_transcript(trace: Any) -> List[Dict[str, Any]]:transcript: List[Dict[str, Any]] = []for turn in trace.turns:for tc in getattr(turn, "tool_calls", []):+ if tc is None:+ continuemapped = _TOOL_NAME_MAP.get(tc["name"], tc["name"])transcript.append({"type": "message","message": {"role": "assistant",- "content": [{"type": "toolCall", "name": mapped, "arguments": tc.get("arguments", {})}],+ "content": [{"type": "toolCall", "name": mapped, "params": tc.get("arguments") or {}}],},})transcript.append({@@ -93,6 +95,42 @@ def _trace_to_transcript(trace: Any) -> List[Dict[str, Any]]:"content": [{"text": tc.get("result", "")}],},})++ # Capture final assistant text response (for tasks graded on text output)+ response_text = getattr(trace, "response_text", "")+ if response_text:+ transcript.append({+ "type": "message",+ "message": {+ "role": "assistant",+ "content": [{"type": "text", "text": response_text}],+ },+ })+ return transcript+++def _tool_results_to_transcript(+ tool_results: List[Dict[str, Any]],+) -> List[Dict[str, Any]]:+ """Build transcript from JarvisAgentBackend tool_results list."""+ transcript: List[Dict[str, Any]] = []+ for tr in tool_results:+ tool_name = tr.get("tool_name", "unknown")+ mapped = _TOOL_NAME_MAP.get(tool_name, tool_name)+ transcript.append({+ "type": "message",+ "message": {+ "role": "assistant",+ "content": [{"type": "toolCall", "name": mapped, "params": {}}],+ },+ })+ transcript.append({+ "type": "message",+ "message": {+ "role": "toolResult",+ "content": [{"text": tr.get("content", "")}],+ },+ })return transcript@@ -118,8 +156,10 @@ def _summarize_transcript(transcript: List[Dict[str, Any]]) -> str:for item in msg.get("content", []):if item.get("type") == "toolCall":parts.append(- f"Tool: {item.get('name')}({json.dumps(item.get('arguments', {}))})"+ f"Tool: {item.get('name')}({json.dumps(item.get('params', {}))})")+ elif item.get("type") == "text":+ parts.append(f"Assistant: {item.get('text', '')}")elif role == "toolResult":content = msg.get("content", [])if content:@@ -372,7 +412,7 @@ def _grade_hybrid(judge_model: str,) -> Dict[str, Any]:"""Run both automated and LLM judge grading, combine with weights."""- weights = record.metadata.get("grading_weights", {"automated": 0.5, "llm_judge": 0.5})+ weights = record.metadata.get("grading_weights") or {"automated": 0.5, "llm_judge": 0.5}auto = _grade_automated(record, transcript, workspace_path)llm = _grade_llm_judge(record, transcript, workspace_path, judge_backend, judge_model)@@ -426,7 +466,24 @@ class PinchBenchScorer(LLMJudgeScorer):self, record: EvalRecord, model_answer: str,) -> Tuple[Optional[bool], Dict[str, Any]]:trace = record.metadata.get("query_trace")- transcript = _trace_to_transcript(trace) if trace else []+ if trace:+ transcript = _trace_to_transcript(trace)+ else:+ # No trace — build transcript from tool_results if available+ tool_results = record.metadata.get("tool_results", [])+ transcript = _tool_results_to_transcript(tool_results)++ # Always append final model answer as assistant text message+ # so grading functions that check for text responses can find it+ if model_answer:+ transcript.append({+ "type": "message",+ "message": {+ "role": "assistant",+ "content": [{"type": "text", "text": model_answer}],+ },+ })+result = grade_pinchbench_task(record=record,transcript=transcript,@@ -442,4 +499,5 @@ __all__ = ["PinchBenchScorer","events_to_transcript","grade_pinchbench_task",+ "_tool_results_to_transcript",]diff --git a/src/openjarvis/security/file_policy.py b/src/openjarvis/security/file_policy.pyindex fc4006ae..36482440 100644--- a/src/openjarvis/security/file_policy.py+++ b/src/openjarvis/security/file_policy.py@@ -30,11 +30,27 @@ def is_sensitive_file(path: Union[str, Path]) -> bool:Checks both the filename and the full name against``DEFAULT_SENSITIVE_PATTERNS`` using :func:`fnmatch.fnmatch`.+ Uses the Rust implementation when available, falls back to Python."""- from openjarvis._rust_bridge import get_rust_module+ try:+ from openjarvis._rust_bridge import get_rust_module- _rust = get_rust_module()- return _rust.is_sensitive_file(str(path))+ _rust = get_rust_module()+ return _rust.is_sensitive_file(str(path))+ except ImportError:+ return _is_sensitive_file_py(str(path))+++def _is_sensitive_file_py(path_str: str) -> bool:+ """Pure-Python fallback for sensitive file detection."""+ import fnmatch++ p = Path(path_str)+ name = p.name+ for pattern in DEFAULT_SENSITIVE_PATTERNS:+ if fnmatch.fnmatch(name, pattern) or fnmatch.fnmatch(str(p), pattern):+ return True+ return Falsedef filter_sensitive_paths(paths: Iterable[Union[str, Path]]) -> List[Path]:diff --git a/src/openjarvis/tools/calculator.py b/src/openjarvis/tools/calculator.pyindex 14ca4bc4..2ec101eb 100644--- a/src/openjarvis/tools/calculator.py+++ b/src/openjarvis/tools/calculator.py@@ -89,10 +89,15 @@ def _safe_eval_node(node: ast.AST) -> Any:def safe_eval(expression: str) -> float:- """Evaluate a math expression safely — always via Rust backend."""- from openjarvis._rust_bridge import get_rust_module- _rust = get_rust_module()- return float(_rust.CalculatorTool().execute(expression))+ """Evaluate a math expression safely — Rust backend with Python fallback."""+ try:+ from openjarvis._rust_bridge import get_rust_module+ _rust = get_rust_module()+ return float(_rust.CalculatorTool().execute(expression))+ except ImportError:+ import ast as _ast+ tree = _ast.parse(expression, mode="eval")+ return float(_safe_eval_node(tree.body))@ToolRegistry.register("calculator")diff --git a/src/openjarvis/tools/file_read.py b/src/openjarvis/tools/file_read.pyindex 36fb258e..6e5b7e38 100644--- a/src/openjarvis/tools/file_read.py+++ b/src/openjarvis/tools/file_read.py@@ -114,10 +114,15 @@ class FileReadTool(BaseTool):content=f"File too large: {size} bytes (max {_MAX_SIZE_BYTES}).",success=False,)- from openjarvis._rust_bridge import get_rust_module- _rust = get_rust_module()try:+ from openjarvis._rust_bridge import get_rust_module+ _rust = get_rust_module()text = _rust.FileReadTool().execute(str(path))+ except ImportError:+ try:+ text = path.read_text(encoding="utf-8")+ except UnicodeDecodeError:+ text = path.read_text(encoding="utf-8", errors="replace")except Exception as exc:return ToolResult(tool_name="file_read",diff --git a/src/openjarvis/tools/file_write.py b/src/openjarvis/tools/file_write.pyindex d006280c..71be96df 100644--- a/src/openjarvis/tools/file_write.py+++ b/src/openjarvis/tools/file_write.py@@ -155,26 +155,26 @@ class FileWriteTool(BaseTool):success=False,)- from openjarvis._rust_bridge import get_rust_module- _rust = get_rust_module()if mode == "write":try:+ from openjarvis._rust_bridge import get_rust_module+ _rust = get_rust_module()_rust.FileWriteTool().execute(str(path), content)+ except ImportError:+ try:+ path.write_text(content, encoding="utf-8")+ except OSError as exc:+ return ToolResult(+ tool_name="file_write",+ content=f"Write error: {exc}",+ success=False,+ )except Exception as exc:return ToolResult(tool_name="file_write",content=f"Write error: {exc}",success=False,)- elif False: # dead code — all write modes go through Rust- try:- path.write_text(content, encoding="utf-8")- except OSError as exc:- return ToolResult(- tool_name="file_write",- content=f"Write error: {exc}",- success=False,- )else:# append mode — always Pythontry:diff --git a/src/openjarvis/tools/http_request.py b/src/openjarvis/tools/http_request.pyindex 4cde4d60..d519815a 100644--- a/src/openjarvis/tools/http_request.py+++ b/src/openjarvis/tools/http_request.py@@ -103,9 +103,13 @@ class HttpRequestTool(BaseTool):body = params.get("body")timeout = params.get("timeout", 30)- from openjarvis._rust_bridge import get_rust_module- _rust = get_rust_module()- if not headers:+ _rust = None+ try:+ from openjarvis._rust_bridge import get_rust_module+ _rust = get_rust_module()+ except ImportError:+ pass+ if _rust is not None and not headers:try:content = _rust.HttpRequestTool().execute(url, method, body)return ToolResult(diff --git a/src/openjarvis/tools/shell_exec.py b/src/openjarvis/tools/shell_exec.pyindex 3952d26d..6611df7a 100644--- a/src/openjarvis/tools/shell_exec.py+++ b/src/openjarvis/tools/shell_exec.py@@ -125,32 +125,33 @@ class ShellExecTool(BaseTool):if val is not None:env[key] = val- from openjarvis._rust_bridge import get_rust_module- _rust = get_rust_module()- if True:- try:- output = _rust.ShellExecTool().execute(command, working_dir)- return ToolResult(- tool_name="shell_exec",- content=output or "(no output)",- success=True,- metadata={- "returncode": 0,- "timeout_used": timeout,- "working_dir": working_dir,- },- )- except Exception as exc:- return ToolResult(- tool_name="shell_exec",- content=str(exc),- success=False,- metadata={- "returncode": -1,- "timeout_used": timeout,- "working_dir": working_dir,- },- )+ try:+ from openjarvis._rust_bridge import get_rust_module+ _rust = get_rust_module()+ output = _rust.ShellExecTool().execute(command, working_dir)+ return ToolResult(+ tool_name="shell_exec",+ content=output or "(no output)",+ success=True,+ metadata={+ "returncode": 0,+ "timeout_used": timeout,+ "working_dir": working_dir,+ },+ )+ except ImportError:+ pass # Fall through to subprocess below+ except Exception as exc:+ return ToolResult(+ tool_name="shell_exec",+ content=str(exc),+ success=False,+ metadata={+ "returncode": -1,+ "timeout_used": timeout,+ "working_dir": working_dir,+ },+ )try:result = subprocess.run(command,diff --git a/src/openjarvis/tools/think.py b/src/openjarvis/tools/think.pyindex 225c888e..699bbe03 100644--- a/src/openjarvis/tools/think.py+++ b/src/openjarvis/tools/think.py@@ -40,9 +40,12 @@ class ThinkTool(BaseTool):def execute(self, **params: Any) -> ToolResult:thought = params.get("thought", "")- from openjarvis._rust_bridge import get_rust_module- _rust = get_rust_module()- content = _rust.ThinkTool().execute(thought)+ try:+ from openjarvis._rust_bridge import get_rust_module+ _rust = get_rust_module()+ content = _rust.ThinkTool().execute(thought)+ except ImportError:+ content = thoughtreturn ToolResult(tool_name="think",content=content,diff --git a/tests/evals/test_benchmark_datasets.py b/tests/evals/test_benchmark_datasets.pyindex ee5e73a3..e569fa48 100644--- a/tests/evals/test_benchmark_datasets.py+++ b/tests/evals/test_benchmark_datasets.py@@ -313,7 +313,7 @@ class TestConfigBenchmarks:def test_benchmarks_count(self) -> None:from openjarvis.evals.core.config import KNOWN_BENCHMARKS- assert len(KNOWN_BENCHMARKS) == 25+ assert len(KNOWN_BENCHMARKS) == 26# ---------------------------------------------------------------------------diff --git a/tests/evals/test_pinchbench_grading.py b/tests/evals/test_pinchbench_grading.pyindex aa107506..d2396bfc 100644--- a/tests/evals/test_pinchbench_grading.py+++ b/tests/evals/test_pinchbench_grading.py@@ -74,7 +74,7 @@ def grade(transcript, workspace_path):{"type": "toolCall","name": "read_file",- "arguments": {"path": "a.txt"},+ "params": {"path": "a.txt"},}],},@@ -124,7 +124,7 @@ class TestSummarizeTranscript:{"type": "toolCall","name": "read_file",- "arguments": {"path": "a.txt"},+ "params": {"path": "a.txt"},}],},