Skip to content

Personal AI, On Personal Devices

Python

fix(evals): pipe tool arguments to transcript + multi-session support

Jon Saad-Falcon1531ad656c19
diff --git a/src/openjarvis/evals/core/runner.py b/src/openjarvis/evals/core/runner.py
index 9effbad2..9f4d82e6 100644
--- a/src/openjarvis/evals/core/runner.py
+++ b/src/openjarvis/evals/core/runner.py
@@ -261,10 +261,29 @@ class EvalRunner:
**gen_kwargs,
)
full = full or {}
- # Store tool results for the scorer to build transcripts
- record.metadata["tool_results"] = full.get(
+ all_tool_results = list(full.get(
"tool_results", [],
- )
+ ))
+
+ # Multi-session: execute remaining sessions
+ sessions = record.metadata.get("sessions", [])
+ if (
+ record.metadata.get("multi_session")
+ and len(sessions) > 1
+ ):
+ for session in sessions[1:]:
+ prompt = session.get("prompt", "")
+ if not prompt:
+ continue
+ sfull = self._backend.generate_full(
+ prompt, **gen_kwargs,
+ )
+ sfull = sfull or {}
+ all_tool_results.extend(
+ sfull.get("tool_results", []),
+ )
+
+ record.metadata["tool_results"] = all_tool_results
# Score INSIDE context so workspace files still exist
content = full.get("content", "")
is_correct, scoring_meta = self._scorer.score(
diff --git a/src/openjarvis/evals/datasets/pinchbench.py b/src/openjarvis/evals/datasets/pinchbench.py
index fdcb8fd0..9c084bc4 100644
--- a/src/openjarvis/evals/datasets/pinchbench.py
+++ b/src/openjarvis/evals/datasets/pinchbench.py
@@ -74,6 +74,8 @@ def _parse_task_markdown(content: str, filename: str = "") -> Dict[str, Any]:
"timeout_seconds": frontmatter.get("timeout_seconds", 180),
"workspace_files": frontmatter.get("workspace_files", []),
"grading_weights": frontmatter.get("grading_weights"),
+ "multi_session": frontmatter.get("multi_session", False),
+ "sessions": frontmatter.get("sessions", []),
"prompt": sections.get("Prompt", ""),
"expected_behavior": sections.get("Expected Behavior", ""),
"grading_criteria": sections.get("Grading Criteria", ""),
@@ -160,7 +162,11 @@ class PinchBenchDataset(DatasetProvider):
self._records = [
EvalRecord(
record_id=t["id"],
- problem=t["prompt"],
+ problem=(
+ t["sessions"][0]["prompt"]
+ if t.get("multi_session") and t.get("sessions")
+ else t["prompt"]
+ ),
reference=t["expected_behavior"],
category=t["category"],
subject=t["name"],
@@ -172,6 +178,8 @@ class PinchBenchDataset(DatasetProvider):
"timeout_seconds": t["timeout_seconds"],
"workspace_files": t["workspace_files"],
"pinchbench_repo_dir": str(repo_dir),
+ "multi_session": t.get("multi_session", False),
+ "sessions": t.get("sessions", []),
},
)
for t in tasks
diff --git a/src/openjarvis/evals/scorers/pinchbench.py b/src/openjarvis/evals/scorers/pinchbench.py
index 693f3d12..4325012a 100644
--- a/src/openjarvis/evals/scorers/pinchbench.py
+++ b/src/openjarvis/evals/scorers/pinchbench.py
@@ -121,7 +121,7 @@ def _tool_results_to_transcript(
"type": "message",
"message": {
"role": "assistant",
- "content": [{"type": "toolCall", "name": mapped, "params": {}}],
+ "content": [{"type": "toolCall", "name": mapped, "params": tr.get("arguments", {})}],
},
})
transcript.append({
diff --git a/src/openjarvis/system.py b/src/openjarvis/system.py
index 2764323a..b2dc1619 100644
--- a/src/openjarvis/system.py
+++ b/src/openjarvis/system.py
@@ -251,6 +251,7 @@ class JarvisSystem:
"tool_name": tr.tool_name,
"content": tr.content,
"success": tr.success,
+ "arguments": tr.metadata.get("arguments", {}),
}
for tr in getattr(result, "tool_results", [])
],
diff --git a/src/openjarvis/tools/_stubs.py b/src/openjarvis/tools/_stubs.py
index 73a15b27..143ddaa9 100644
--- a/src/openjarvis/tools/_stubs.py
+++ b/src/openjarvis/tools/_stubs.py
@@ -241,6 +241,7 @@ class ToolExecutor:
)
latency = time.time() - t0
result.latency_seconds = latency
+ result.metadata["arguments"] = params
# Auto-detect taints in results
if result.success: