Personal AI, On Personal Devices
Python
fix(evals): pipe tool arguments to transcript + multi-session support
Jon Saad-Falcon1531ad656c19
diff --git a/src/openjarvis/evals/core/runner.py b/src/openjarvis/evals/core/runner.pyindex 9effbad2..9f4d82e6 100644--- a/src/openjarvis/evals/core/runner.py+++ b/src/openjarvis/evals/core/runner.py@@ -261,10 +261,29 @@ class EvalRunner:**gen_kwargs,)full = full or {}- # Store tool results for the scorer to build transcripts- record.metadata["tool_results"] = full.get(+ all_tool_results = list(full.get("tool_results", [],- )+ ))++ # Multi-session: execute remaining sessions+ sessions = record.metadata.get("sessions", [])+ if (+ record.metadata.get("multi_session")+ and len(sessions) > 1+ ):+ for session in sessions[1:]:+ prompt = session.get("prompt", "")+ if not prompt:+ continue+ sfull = self._backend.generate_full(+ prompt, **gen_kwargs,+ )+ sfull = sfull or {}+ all_tool_results.extend(+ sfull.get("tool_results", []),+ )++ record.metadata["tool_results"] = all_tool_results# Score INSIDE context so workspace files still existcontent = full.get("content", "")is_correct, scoring_meta = self._scorer.score(diff --git a/src/openjarvis/evals/datasets/pinchbench.py b/src/openjarvis/evals/datasets/pinchbench.pyindex fdcb8fd0..9c084bc4 100644--- a/src/openjarvis/evals/datasets/pinchbench.py+++ b/src/openjarvis/evals/datasets/pinchbench.py@@ -74,6 +74,8 @@ def _parse_task_markdown(content: str, filename: str = "") -> Dict[str, Any]:"timeout_seconds": frontmatter.get("timeout_seconds", 180),"workspace_files": frontmatter.get("workspace_files", []),"grading_weights": frontmatter.get("grading_weights"),+ "multi_session": frontmatter.get("multi_session", False),+ "sessions": frontmatter.get("sessions", []),"prompt": sections.get("Prompt", ""),"expected_behavior": sections.get("Expected Behavior", ""),"grading_criteria": sections.get("Grading Criteria", ""),@@ -160,7 +162,11 @@ class PinchBenchDataset(DatasetProvider):self._records = [EvalRecord(record_id=t["id"],- problem=t["prompt"],+ problem=(+ t["sessions"][0]["prompt"]+ if t.get("multi_session") and t.get("sessions")+ else t["prompt"]+ ),reference=t["expected_behavior"],category=t["category"],subject=t["name"],@@ -172,6 +178,8 @@ class PinchBenchDataset(DatasetProvider):"timeout_seconds": t["timeout_seconds"],"workspace_files": t["workspace_files"],"pinchbench_repo_dir": str(repo_dir),+ "multi_session": t.get("multi_session", False),+ "sessions": t.get("sessions", []),},)for t in tasksdiff --git a/src/openjarvis/evals/scorers/pinchbench.py b/src/openjarvis/evals/scorers/pinchbench.pyindex 693f3d12..4325012a 100644--- a/src/openjarvis/evals/scorers/pinchbench.py+++ b/src/openjarvis/evals/scorers/pinchbench.py@@ -121,7 +121,7 @@ def _tool_results_to_transcript("type": "message","message": {"role": "assistant",- "content": [{"type": "toolCall", "name": mapped, "params": {}}],+ "content": [{"type": "toolCall", "name": mapped, "params": tr.get("arguments", {})}],},})transcript.append({diff --git a/src/openjarvis/system.py b/src/openjarvis/system.pyindex 2764323a..b2dc1619 100644--- a/src/openjarvis/system.py+++ b/src/openjarvis/system.py@@ -251,6 +251,7 @@ class JarvisSystem:"tool_name": tr.tool_name,"content": tr.content,"success": tr.success,+ "arguments": tr.metadata.get("arguments", {}),}for tr in getattr(result, "tool_results", [])],diff --git a/src/openjarvis/tools/_stubs.py b/src/openjarvis/tools/_stubs.pyindex 73a15b27..143ddaa9 100644--- a/src/openjarvis/tools/_stubs.py+++ b/src/openjarvis/tools/_stubs.py@@ -241,6 +241,7 @@ class ToolExecutor:)latency = time.time() - t0result.latency_seconds = latency+ result.metadata["arguments"] = params# Auto-detect taints in resultsif result.success: