Skip to content

Personal AI, On Personal Devices

Python

fix(evals): PinchBench harness fixes — scores from 26% to 84% (#124)

Jon Saad-Falcon1d24c64f117b
diff --git a/src/openjarvis/agents/native_openhands.py b/src/openjarvis/agents/native_openhands.py
index cf5579cd..9b518809 100644
--- a/src/openjarvis/agents/native_openhands.py
+++ b/src/openjarvis/agents/native_openhands.py
@@ -283,14 +283,23 @@ class NativeOpenHandsAgent(ToolUsingAgent):
last_content = ""
total_usage = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}
+ # Build OpenAI-format tool schemas for native function calling
+ openai_tools = (
+ self._executor.get_openai_tools() if self._tools else []
+ )
+
for _turn in range(self._max_turns):
turns += 1
# Truncate before every generate call -- tool results may have
# expanded the context beyond what the model supports.
messages = self._truncate_if_needed(messages)
+ gen_kwargs: dict[str, Any] = {}
+ if openai_tools:
+ gen_kwargs["tools"] = openai_tools
+
try:
- result = self._generate(messages)
+ result = self._generate(messages, **gen_kwargs)
except Exception as exc:
error_str = str(exc)
if "400" in error_str:
@@ -318,6 +327,42 @@ class NativeOpenHandsAgent(ToolUsingAgent):
content = self._strip_think_tags(content)
last_content = content
+ # --- Native function-calling path (OpenAI, Anthropic, etc.) ---
+ raw_tool_calls = result.get("tool_calls", [])
+ if raw_tool_calls:
+ native_calls = [
+ ToolCall(
+ id=tc.get("id", f"call_{turns}_{i}"),
+ name=tc.get("name", ""),
+ arguments=tc.get("arguments", "{}"),
+ )
+ for i, tc in enumerate(raw_tool_calls)
+ ]
+ messages.append(
+ Message(
+ role=Role.ASSISTANT,
+ content=content,
+ tool_calls=native_calls,
+ )
+ )
+ for tc in native_calls:
+ tool_result = self._executor.execute(tc)
+ all_tool_results.append(tool_result)
+ obs_text = tool_result.content
+ if len(obs_text) > 4000:
+ obs_text = obs_text[:4000] + "\n\n[Output truncated]"
+ messages.append(
+ Message(
+ role=Role.TOOL,
+ content=obs_text,
+ tool_call_id=tc.id,
+ name=tc.name,
+ )
+ )
+ continue
+
+ # --- Text-based fallback (CodeAct / Action-Input format) ---
+
# Try to extract code
code = self._extract_code(content)
if code:
diff --git a/src/openjarvis/evals/configs/pinchbench-gpt54.toml b/src/openjarvis/evals/configs/pinchbench-gpt54.toml
new file mode 100644
index 00000000..e1feee6a
--- /dev/null
+++ b/src/openjarvis/evals/configs/pinchbench-gpt54.toml
@@ -0,0 +1,34 @@
+# PinchBench eval: gpt-5.4 (cloud)
+# Agent: native_openhands — all PinchBench-required tools enabled
+
+[meta]
+name = "pinchbench-gpt54"
+description = "PinchBench on gpt-5.4-2026-03-05"
+
+[defaults]
+temperature = 0.6
+max_tokens = 8192
+
+[judge]
+model = "claude-opus-4-5"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 1
+output_dir = "results/pinchbench-gpt54/"
+seed = 42
+
+[[models]]
+name = "gpt-5.4-2026-03-05"
+engine = "cloud"
+
+[[benchmarks]]
+name = "pinchbench"
+backend = "jarvis-agent"
+agent = "native_openhands"
+tools = [
+ "think", "file_read", "file_write", "web_search", "shell_exec",
+ "code_interpreter", "browser_navigate", "image_generate",
+ "calculator", "http_request", "pdf_extract",
+]
diff --git a/src/openjarvis/evals/configs/pinchbench-qwen397b.toml b/src/openjarvis/evals/configs/pinchbench-qwen397b.toml
new file mode 100644
index 00000000..b6b1493b
--- /dev/null
+++ b/src/openjarvis/evals/configs/pinchbench-qwen397b.toml
@@ -0,0 +1,35 @@
+# PinchBench eval: Qwen3.5-397B-A17B-FP8 (vLLM, TP=8)
+# Agent: native_openhands — all PinchBench-required tools enabled
+
+[meta]
+name = "pinchbench-qwen397b"
+description = "PinchBench on Qwen/Qwen3.5-397B-A17B-FP8 (vLLM, TP=8)"
+
+[defaults]
+temperature = 0.6
+max_tokens = 8192
+
+[judge]
+model = "claude-opus-4-5"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 1
+output_dir = "results/pinchbench-qwen397b/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-397B-A17B-FP8"
+engine = "vllm"
+num_gpus = 8
+
+[[benchmarks]]
+name = "pinchbench"
+backend = "jarvis-agent"
+agent = "native_openhands"
+tools = [
+ "think", "file_read", "file_write", "web_search", "shell_exec",
+ "code_interpreter", "browser_navigate", "image_generate",
+ "calculator", "http_request", "pdf_extract",
+]
diff --git a/src/openjarvis/evals/core/config.py b/src/openjarvis/evals/core/config.py
index ac62d191..ab864c71 100644
--- a/src/openjarvis/evals/core/config.py
+++ b/src/openjarvis/evals/core/config.py
@@ -43,6 +43,7 @@ KNOWN_BENCHMARKS = {
"knowledge_base", "coding_task",
"coding_assistant", "security_scanner", "daily_digest",
"doc_qa", "browser_assistant",
+ "pinchbench",
}
diff --git a/src/openjarvis/evals/core/runner.py b/src/openjarvis/evals/core/runner.py
index e3d8716e..9effbad2 100644
--- a/src/openjarvis/evals/core/runner.py
+++ b/src/openjarvis/evals/core/runner.py
@@ -96,11 +96,19 @@ class EvalRunner:
seed=cfg.seed,
)
- # Auto-enable episode_mode when the dataset has iter_episodes()
- # (i.e. it is a lifelong/sequential benchmark like LifelongAgentBench).
- # This is enforced at the runner level so it applies regardless of
- # how the runner is invoked (CLI, SDK, tests, etc.).
- if not cfg.episode_mode and hasattr(self._dataset, "iter_episodes"):
+ # Auto-enable episode_mode when the dataset *overrides*
+ # iter_episodes() (i.e. it is a lifelong/sequential benchmark like
+ # LifelongAgentBench). The base DatasetProvider always defines a
+ # default iter_episodes() that wraps each record in its own episode,
+ # so hasattr() is always True — we must check for a real override.
+ from openjarvis.evals.core.dataset import DatasetProvider as _DP
+ try:
+ _overrides_episodes = (
+ type(self._dataset).iter_episodes is not _DP.iter_episodes
+ )
+ except AttributeError:
+ _overrides_episodes = False
+ if not cfg.episode_mode and _overrides_episodes:
LOGGER.info(
"%s requires sequential episode processing — "
"auto-enabling episode_mode.",
@@ -109,6 +117,15 @@ class EvalRunner:
cfg = dataclasses.replace(cfg, episode_mode=True)
self._config = cfg
+ # Detect if dataset provides task environments (e.g. PinchBench)
+ try:
+ self._has_task_env = (
+ type(self._dataset).create_task_env
+ is not _DP.create_task_env
+ )
+ except AttributeError:
+ self._has_task_env = False
+
records = list(self._dataset.iter_records())
LOGGER.info(
"Running %s: %d samples, backend=%s, model=%s, workers=%d, "
@@ -145,6 +162,15 @@ class EvalRunner:
try:
if cfg.episode_mode:
self._run_episode_mode(records, progress_callback, total)
+ elif self._has_task_env:
+ # Task environments (PinchBench etc.) change CWD —
+ # must process sequentially for thread safety.
+ for record in records:
+ result = self._process_one(record)
+ self._results.append(result)
+ self._flush_result(result)
+ if progress_callback is not None:
+ progress_callback(len(self._results), total)
else:
with ThreadPoolExecutor(max_workers=cfg.max_workers) as pool:
futures = {
@@ -224,17 +250,41 @@ class EvalRunner:
)
if cfg.system_prompt:
gen_kwargs["system"] = cfg.system_prompt
- full = self._backend.generate_full(
- record.problem,
- **gen_kwargs,
- )
- content = full.get("content", "")
+
+ if getattr(self, "_has_task_env", False):
+ from contextlib import nullcontext
+ task_env = self._dataset.create_task_env(record)
+ ctx = task_env if task_env is not None else nullcontext()
+ with ctx:
+ full = self._backend.generate_full(
+ record.problem,
+ **gen_kwargs,
+ )
+ full = full or {}
+ # Store tool results for the scorer to build transcripts
+ record.metadata["tool_results"] = full.get(
+ "tool_results", [],
+ )
+ # Score INSIDE context so workspace files still exist
+ content = full.get("content", "")
+ is_correct, scoring_meta = self._scorer.score(
+ record, content,
+ )
+ else:
+ full = self._backend.generate_full(
+ record.problem,
+ **gen_kwargs,
+ )
+ full = full or {}
+ content = full.get("content", "")
+ is_correct, scoring_meta = self._scorer.score(
+ record, content,
+ )
+
usage = full.get("usage", {})
latency = full.get("latency_seconds", 0.0)
cost = full.get("cost_usd", 0.0)
- is_correct, scoring_meta = self._scorer.score(record, content)
-
energy_j = full.get("energy_joules", 0.0)
power_w = full.get("power_watts", 0.0)
throughput = full.get("throughput_tok_per_sec", 0.0)
diff --git a/src/openjarvis/evals/scorers/pinchbench.py b/src/openjarvis/evals/scorers/pinchbench.py
index 68269ec0..693f3d12 100644
--- a/src/openjarvis/evals/scorers/pinchbench.py
+++ b/src/openjarvis/evals/scorers/pinchbench.py
@@ -52,12 +52,12 @@ def events_to_transcript(events: List[Any]) -> List[Dict[str, Any]]:
if etype == EventType.TOOL_CALL_START or etype == EventType.TOOL_CALL_START.value:
tool_name = event.metadata.get("tool", "unknown")
mapped = _TOOL_NAME_MAP.get(tool_name, tool_name)
- arguments = event.metadata.get("arguments", {})
+ arguments = event.metadata.get("arguments") or {}
transcript.append({
"type": "message",
"message": {
"role": "assistant",
- "content": [{"type": "toolCall", "name": mapped, "arguments": arguments}],
+ "content": [{"type": "toolCall", "name": mapped, "params": arguments}],
},
})
elif etype == EventType.TOOL_CALL_END or etype == EventType.TOOL_CALL_END.value:
@@ -78,12 +78,14 @@ def _trace_to_transcript(trace: Any) -> List[Dict[str, Any]]:
transcript: List[Dict[str, Any]] = []
for turn in trace.turns:
for tc in getattr(turn, "tool_calls", []):
+ if tc is None:
+ continue
mapped = _TOOL_NAME_MAP.get(tc["name"], tc["name"])
transcript.append({
"type": "message",
"message": {
"role": "assistant",
- "content": [{"type": "toolCall", "name": mapped, "arguments": tc.get("arguments", {})}],
+ "content": [{"type": "toolCall", "name": mapped, "params": tc.get("arguments") or {}}],
},
})
transcript.append({
@@ -93,6 +95,42 @@ def _trace_to_transcript(trace: Any) -> List[Dict[str, Any]]:
"content": [{"text": tc.get("result", "")}],
},
})
+
+ # Capture final assistant text response (for tasks graded on text output)
+ response_text = getattr(trace, "response_text", "")
+ if response_text:
+ transcript.append({
+ "type": "message",
+ "message": {
+ "role": "assistant",
+ "content": [{"type": "text", "text": response_text}],
+ },
+ })
+ return transcript
+
+
+def _tool_results_to_transcript(
+ tool_results: List[Dict[str, Any]],
+) -> List[Dict[str, Any]]:
+ """Build transcript from JarvisAgentBackend tool_results list."""
+ transcript: List[Dict[str, Any]] = []
+ for tr in tool_results:
+ tool_name = tr.get("tool_name", "unknown")
+ mapped = _TOOL_NAME_MAP.get(tool_name, tool_name)
+ transcript.append({
+ "type": "message",
+ "message": {
+ "role": "assistant",
+ "content": [{"type": "toolCall", "name": mapped, "params": {}}],
+ },
+ })
+ transcript.append({
+ "type": "message",
+ "message": {
+ "role": "toolResult",
+ "content": [{"text": tr.get("content", "")}],
+ },
+ })
return transcript
@@ -118,8 +156,10 @@ def _summarize_transcript(transcript: List[Dict[str, Any]]) -> str:
for item in msg.get("content", []):
if item.get("type") == "toolCall":
parts.append(
- f"Tool: {item.get('name')}({json.dumps(item.get('arguments', {}))})"
+ f"Tool: {item.get('name')}({json.dumps(item.get('params', {}))})"
)
+ elif item.get("type") == "text":
+ parts.append(f"Assistant: {item.get('text', '')}")
elif role == "toolResult":
content = msg.get("content", [])
if content:
@@ -372,7 +412,7 @@ def _grade_hybrid(
judge_model: str,
) -> Dict[str, Any]:
"""Run both automated and LLM judge grading, combine with weights."""
- weights = record.metadata.get("grading_weights", {"automated": 0.5, "llm_judge": 0.5})
+ weights = record.metadata.get("grading_weights") or {"automated": 0.5, "llm_judge": 0.5}
auto = _grade_automated(record, transcript, workspace_path)
llm = _grade_llm_judge(record, transcript, workspace_path, judge_backend, judge_model)
@@ -426,7 +466,24 @@ class PinchBenchScorer(LLMJudgeScorer):
self, record: EvalRecord, model_answer: str,
) -> Tuple[Optional[bool], Dict[str, Any]]:
trace = record.metadata.get("query_trace")
- transcript = _trace_to_transcript(trace) if trace else []
+ if trace:
+ transcript = _trace_to_transcript(trace)
+ else:
+ # No trace — build transcript from tool_results if available
+ tool_results = record.metadata.get("tool_results", [])
+ transcript = _tool_results_to_transcript(tool_results)
+
+ # Always append final model answer as assistant text message
+ # so grading functions that check for text responses can find it
+ if model_answer:
+ transcript.append({
+ "type": "message",
+ "message": {
+ "role": "assistant",
+ "content": [{"type": "text", "text": model_answer}],
+ },
+ })
+
result = grade_pinchbench_task(
record=record,
transcript=transcript,
@@ -442,4 +499,5 @@ __all__ = [
"PinchBenchScorer",
"events_to_transcript",
"grade_pinchbench_task",
+ "_tool_results_to_transcript",
]
diff --git a/src/openjarvis/security/file_policy.py b/src/openjarvis/security/file_policy.py
index fc4006ae..36482440 100644
--- a/src/openjarvis/security/file_policy.py
+++ b/src/openjarvis/security/file_policy.py
@@ -30,11 +30,27 @@ def is_sensitive_file(path: Union[str, Path]) -> bool:
Checks both the filename and the full name against
``DEFAULT_SENSITIVE_PATTERNS`` using :func:`fnmatch.fnmatch`.
+ Uses the Rust implementation when available, falls back to Python.
"""
- from openjarvis._rust_bridge import get_rust_module
+ try:
+ from openjarvis._rust_bridge import get_rust_module
- _rust = get_rust_module()
- return _rust.is_sensitive_file(str(path))
+ _rust = get_rust_module()
+ return _rust.is_sensitive_file(str(path))
+ except ImportError:
+ return _is_sensitive_file_py(str(path))
+
+
+def _is_sensitive_file_py(path_str: str) -> bool:
+ """Pure-Python fallback for sensitive file detection."""
+ import fnmatch
+
+ p = Path(path_str)
+ name = p.name
+ for pattern in DEFAULT_SENSITIVE_PATTERNS:
+ if fnmatch.fnmatch(name, pattern) or fnmatch.fnmatch(str(p), pattern):
+ return True
+ return False
def filter_sensitive_paths(paths: Iterable[Union[str, Path]]) -> List[Path]:
diff --git a/src/openjarvis/tools/calculator.py b/src/openjarvis/tools/calculator.py
index 14ca4bc4..2ec101eb 100644
--- a/src/openjarvis/tools/calculator.py
+++ b/src/openjarvis/tools/calculator.py
@@ -89,10 +89,15 @@ def _safe_eval_node(node: ast.AST) -> Any:
def safe_eval(expression: str) -> float:
- """Evaluate a math expression safely — always via Rust backend."""
- from openjarvis._rust_bridge import get_rust_module
- _rust = get_rust_module()
- return float(_rust.CalculatorTool().execute(expression))
+ """Evaluate a math expression safely — Rust backend with Python fallback."""
+ try:
+ from openjarvis._rust_bridge import get_rust_module
+ _rust = get_rust_module()
+ return float(_rust.CalculatorTool().execute(expression))
+ except ImportError:
+ import ast as _ast
+ tree = _ast.parse(expression, mode="eval")
+ return float(_safe_eval_node(tree.body))
@ToolRegistry.register("calculator")
diff --git a/src/openjarvis/tools/file_read.py b/src/openjarvis/tools/file_read.py
index 36fb258e..6e5b7e38 100644
--- a/src/openjarvis/tools/file_read.py
+++ b/src/openjarvis/tools/file_read.py
@@ -114,10 +114,15 @@ class FileReadTool(BaseTool):
content=f"File too large: {size} bytes (max {_MAX_SIZE_BYTES}).",
success=False,
)
- from openjarvis._rust_bridge import get_rust_module
- _rust = get_rust_module()
try:
+ from openjarvis._rust_bridge import get_rust_module
+ _rust = get_rust_module()
text = _rust.FileReadTool().execute(str(path))
+ except ImportError:
+ try:
+ text = path.read_text(encoding="utf-8")
+ except UnicodeDecodeError:
+ text = path.read_text(encoding="utf-8", errors="replace")
except Exception as exc:
return ToolResult(
tool_name="file_read",
diff --git a/src/openjarvis/tools/file_write.py b/src/openjarvis/tools/file_write.py
index d006280c..71be96df 100644
--- a/src/openjarvis/tools/file_write.py
+++ b/src/openjarvis/tools/file_write.py
@@ -155,26 +155,26 @@ class FileWriteTool(BaseTool):
success=False,
)
- from openjarvis._rust_bridge import get_rust_module
- _rust = get_rust_module()
if mode == "write":
try:
+ from openjarvis._rust_bridge import get_rust_module
+ _rust = get_rust_module()
_rust.FileWriteTool().execute(str(path), content)
+ except ImportError:
+ try:
+ path.write_text(content, encoding="utf-8")
+ except OSError as exc:
+ return ToolResult(
+ tool_name="file_write",
+ content=f"Write error: {exc}",
+ success=False,
+ )
except Exception as exc:
return ToolResult(
tool_name="file_write",
content=f"Write error: {exc}",
success=False,
)
- elif False: # dead code — all write modes go through Rust
- try:
- path.write_text(content, encoding="utf-8")
- except OSError as exc:
- return ToolResult(
- tool_name="file_write",
- content=f"Write error: {exc}",
- success=False,
- )
else:
# append mode — always Python
try:
diff --git a/src/openjarvis/tools/http_request.py b/src/openjarvis/tools/http_request.py
index 4cde4d60..d519815a 100644
--- a/src/openjarvis/tools/http_request.py
+++ b/src/openjarvis/tools/http_request.py
@@ -103,9 +103,13 @@ class HttpRequestTool(BaseTool):
body = params.get("body")
timeout = params.get("timeout", 30)
- from openjarvis._rust_bridge import get_rust_module
- _rust = get_rust_module()
- if not headers:
+ _rust = None
+ try:
+ from openjarvis._rust_bridge import get_rust_module
+ _rust = get_rust_module()
+ except ImportError:
+ pass
+ if _rust is not None and not headers:
try:
content = _rust.HttpRequestTool().execute(url, method, body)
return ToolResult(
diff --git a/src/openjarvis/tools/shell_exec.py b/src/openjarvis/tools/shell_exec.py
index 3952d26d..6611df7a 100644
--- a/src/openjarvis/tools/shell_exec.py
+++ b/src/openjarvis/tools/shell_exec.py
@@ -125,32 +125,33 @@ class ShellExecTool(BaseTool):
if val is not None:
env[key] = val
- from openjarvis._rust_bridge import get_rust_module
- _rust = get_rust_module()
- if True:
- try:
- output = _rust.ShellExecTool().execute(command, working_dir)
- return ToolResult(
- tool_name="shell_exec",
- content=output or "(no output)",
- success=True,
- metadata={
- "returncode": 0,
- "timeout_used": timeout,
- "working_dir": working_dir,
- },
- )
- except Exception as exc:
- return ToolResult(
- tool_name="shell_exec",
- content=str(exc),
- success=False,
- metadata={
- "returncode": -1,
- "timeout_used": timeout,
- "working_dir": working_dir,
- },
- )
+ try:
+ from openjarvis._rust_bridge import get_rust_module
+ _rust = get_rust_module()
+ output = _rust.ShellExecTool().execute(command, working_dir)
+ return ToolResult(
+ tool_name="shell_exec",
+ content=output or "(no output)",
+ success=True,
+ metadata={
+ "returncode": 0,
+ "timeout_used": timeout,
+ "working_dir": working_dir,
+ },
+ )
+ except ImportError:
+ pass # Fall through to subprocess below
+ except Exception as exc:
+ return ToolResult(
+ tool_name="shell_exec",
+ content=str(exc),
+ success=False,
+ metadata={
+ "returncode": -1,
+ "timeout_used": timeout,
+ "working_dir": working_dir,
+ },
+ )
try:
result = subprocess.run(
command,
diff --git a/src/openjarvis/tools/think.py b/src/openjarvis/tools/think.py
index 225c888e..699bbe03 100644
--- a/src/openjarvis/tools/think.py
+++ b/src/openjarvis/tools/think.py
@@ -40,9 +40,12 @@ class ThinkTool(BaseTool):
def execute(self, **params: Any) -> ToolResult:
thought = params.get("thought", "")
- from openjarvis._rust_bridge import get_rust_module
- _rust = get_rust_module()
- content = _rust.ThinkTool().execute(thought)
+ try:
+ from openjarvis._rust_bridge import get_rust_module
+ _rust = get_rust_module()
+ content = _rust.ThinkTool().execute(thought)
+ except ImportError:
+ content = thought
return ToolResult(
tool_name="think",
content=content,
diff --git a/tests/evals/test_benchmark_datasets.py b/tests/evals/test_benchmark_datasets.py
index ee5e73a3..e569fa48 100644
--- a/tests/evals/test_benchmark_datasets.py
+++ b/tests/evals/test_benchmark_datasets.py
@@ -313,7 +313,7 @@ class TestConfigBenchmarks:
def test_benchmarks_count(self) -> None:
from openjarvis.evals.core.config import KNOWN_BENCHMARKS
- assert len(KNOWN_BENCHMARKS) == 25
+ assert len(KNOWN_BENCHMARKS) == 26
# ---------------------------------------------------------------------------
diff --git a/tests/evals/test_pinchbench_grading.py b/tests/evals/test_pinchbench_grading.py
index aa107506..d2396bfc 100644
--- a/tests/evals/test_pinchbench_grading.py
+++ b/tests/evals/test_pinchbench_grading.py
@@ -74,7 +74,7 @@ def grade(transcript, workspace_path):
{
"type": "toolCall",
"name": "read_file",
- "arguments": {"path": "a.txt"},
+ "params": {"path": "a.txt"},
}
],
},
@@ -124,7 +124,7 @@ class TestSummarizeTranscript:
{
"type": "toolCall",
"name": "read_file",
- "arguments": {"path": "a.txt"},
+ "params": {"path": "a.txt"},
}
],
},