Skip to content

Personal AI, On Personal Devices

Python

feat(evals): add PinchBench grading helpers — transcript translation, automated/LLM judge/hybrid scoring

Jon Saad-Falcon73cb1f1bbdf1
diff --git a/src/openjarvis/evals/scorers/pinchbench.py b/src/openjarvis/evals/scorers/pinchbench.py
new file mode 100644
index 00000000..68269ec0
--- /dev/null
+++ b/src/openjarvis/evals/scorers/pinchbench.py
@@ -0,0 +1,445 @@
+"""PinchBench grading helpers and scorer.
+
+Provides transcript translation (OpenJarvis events → PinchBench format),
+automated grading (exec of embedded Python), LLM judge grading, and
+hybrid combination. Used by PinchBenchTaskEnv.run_tests() and the
+standalone PinchBenchScorer.
+
+Reference: https://github.com/pinchbench/skill
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import re
+from typing import Any, Dict, List, Optional, Tuple
+
+from openjarvis.evals.core.event_recorder import EventType
+from openjarvis.evals.core.scorer import LLMJudgeScorer
+from openjarvis.evals.core.types import EvalRecord
+
+LOGGER = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Tool name mapping: OpenJarvis name → PinchBench-expected name
+# ---------------------------------------------------------------------------
+_TOOL_NAME_MAP: Dict[str, str] = {
+ "file_read": "read_file",
+ "file_write": "write_file",
+ "image_generate": "generate_image",
+ # web_search, calculator, shell_exec, etc. use the same names
+}
+
+
+# ---------------------------------------------------------------------------
+# Transcript translation
+# ---------------------------------------------------------------------------
+
+def events_to_transcript(events: List[Any]) -> List[Dict[str, Any]]:
+ """Build PinchBench-format transcript from raw EventRecorder events.
+
+ Pairs TOOL_CALL_START/END events to extract tool name, arguments, and
+ results. Called by run_tests() before QueryTrace exists.
+ """
+ transcript: List[Dict[str, Any]] = []
+
+ for event in events:
+ etype = event.event_type
+ if isinstance(etype, str):
+ # Normalize string event types to enum comparison
+ pass
+ if etype == EventType.TOOL_CALL_START or etype == EventType.TOOL_CALL_START.value:
+ tool_name = event.metadata.get("tool", "unknown")
+ mapped = _TOOL_NAME_MAP.get(tool_name, tool_name)
+ arguments = event.metadata.get("arguments", {})
+ transcript.append({
+ "type": "message",
+ "message": {
+ "role": "assistant",
+ "content": [{"type": "toolCall", "name": mapped, "arguments": arguments}],
+ },
+ })
+ elif etype == EventType.TOOL_CALL_END or etype == EventType.TOOL_CALL_END.value:
+ result_text = str(event.metadata.get("result", ""))
+ transcript.append({
+ "type": "message",
+ "message": {
+ "role": "toolResult",
+ "content": [{"text": result_text}],
+ },
+ })
+
+ return transcript
+
+
+def _trace_to_transcript(trace: Any) -> List[Dict[str, Any]]:
+ """Convert QueryTrace to PinchBench transcript (for EvalRunner path)."""
+ transcript: List[Dict[str, Any]] = []
+ for turn in trace.turns:
+ for tc in getattr(turn, "tool_calls", []):
+ mapped = _TOOL_NAME_MAP.get(tc["name"], tc["name"])
+ transcript.append({
+ "type": "message",
+ "message": {
+ "role": "assistant",
+ "content": [{"type": "toolCall", "name": mapped, "arguments": tc.get("arguments", {})}],
+ },
+ })
+ transcript.append({
+ "type": "message",
+ "message": {
+ "role": "toolResult",
+ "content": [{"text": tc.get("result", "")}],
+ },
+ })
+ return transcript
+
+
+# ---------------------------------------------------------------------------
+# Transcript summarization (replicates PinchBench _summarize_transcript)
+# ---------------------------------------------------------------------------
+
+def _summarize_transcript(transcript: List[Dict[str, Any]]) -> str:
+ """Summarize transcript for LLM judge prompt.
+
+ Produces the same format as PinchBench's lib_grading._summarize_transcript():
+ Tool: function_name({"arg1": value1})
+ Result: [first 200 chars]
+ User: [message text]
+ """
+ parts: List[str] = []
+ for event in transcript:
+ if event.get("type") != "message":
+ continue
+ msg = event.get("message", {})
+ role = msg.get("role")
+ if role == "assistant":
+ for item in msg.get("content", []):
+ if item.get("type") == "toolCall":
+ parts.append(
+ f"Tool: {item.get('name')}({json.dumps(item.get('arguments', {}))})"
+ )
+ elif role == "toolResult":
+ content = msg.get("content", [])
+ if content:
+ preview = str(content[0])[:200]
+ parts.append(f"Result: {preview}")
+ elif role == "user":
+ content = msg.get("content", [])
+ if content:
+ parts.append(f"User: {content[0]}")
+ return "\n".join(parts)
+
+
+# ---------------------------------------------------------------------------
+# Judge prompt and response parsing (replicates PinchBench exactly)
+# ---------------------------------------------------------------------------
+
+def _build_judge_prompt(
+ *,
+ task_prompt: str,
+ expected_behavior: str,
+ rubric: str,
+ transcript_summary: str,
+) -> str:
+ """Build the LLM judge prompt matching PinchBench's format."""
+ return f"""You are a strict evaluator assessing an AI agent's performance on a task.
+Reserve a score of 1.0 for genuinely excellent performance.
+
+## Task
+{task_prompt}
+
+## Expected Behavior
+{expected_behavior}
+
+## Agent Transcript
+{transcript_summary}
+
+## Grading Rubric
+{rubric}
+
+## Instructions
+Evaluate the agent's performance against each criterion in the rubric.
+Return your evaluation as JSON with this exact structure:
+```json
+{{
+ "scores": {{"criterion_name": score, ...}},
+ "total": overall_score_0_to_1,
+ "notes": "brief justification"
+}}
+```
+Be a strict evaluator. Deduct points for unnecessary steps, verbose output,
+or inefficient tool usage."""
+
+
+def _parse_judge_response(raw: str) -> Dict[str, Any]:
+ """Parse LLM judge response with fallback chain.
+
+ Tries: JSON code block → balanced braces → regex score extraction.
+ Matches PinchBench's lib_grading._parse_judge_response() logic.
+ """
+ if not raw or not raw.strip():
+ return {"scores": {}, "total": 0.0, "notes": "Empty judge response"}
+
+ # Try JSON code block
+ code_block = re.search(r"```json\s*(.*?)\s*```", raw, re.DOTALL)
+ if code_block:
+ try:
+ parsed = json.loads(code_block.group(1))
+ if isinstance(parsed, dict):
+ return _normalize_judge_response(parsed)
+ except json.JSONDecodeError:
+ pass
+
+ # Try balanced braces extraction
+ candidates: List[str] = []
+ depth = 0
+ current: List[str] = []
+ for char in raw:
+ if char == "{":
+ if depth == 0:
+ current = []
+ depth += 1
+ if depth > 0:
+ current.append(char)
+ if char == "}":
+ depth -= 1
+ if depth == 0 and current:
+ candidates.append("".join(current))
+
+ for candidate in reversed(candidates):
+ try:
+ parsed = json.loads(candidate)
+ if isinstance(parsed, dict) and "scores" in parsed:
+ return _normalize_judge_response(parsed)
+ except json.JSONDecodeError:
+ continue
+
+ for candidate in reversed(candidates):
+ try:
+ parsed = json.loads(candidate)
+ if isinstance(parsed, dict):
+ return _normalize_judge_response(parsed)
+ except json.JSONDecodeError:
+ continue
+
+ # Regex fallback for prose scores
+ score_match = re.search(
+ r"(?:total|overall|final)\s*(?:score)?[:\s]*(0\.\d+|1\.0+)",
+ raw,
+ re.IGNORECASE,
+ )
+ if score_match:
+ try:
+ total = float(score_match.group(1))
+ if 0.0 <= total <= 1.0:
+ LOGGER.warning("Fell back to regex score extraction (total=%.2f)", total)
+ return {"scores": {}, "total": total, "notes": "Score extracted from prose"}
+ except ValueError:
+ pass
+
+ LOGGER.warning("Failed to parse judge response")
+ return {"scores": {}, "total": 0.0, "notes": "Failed to parse judge response"}
+
+
+def _normalize_judge_response(parsed: Dict[str, Any]) -> Dict[str, Any]:
+ """Normalize various judge response formats to standard structure.
+
+ Matches PinchBench's lib_grading._normalize_judge_response().
+ """
+ result: Dict[str, Any] = {"scores": {}, "total": 0.0, "notes": ""}
+
+ # Extract scores
+ scores_data = parsed.get("scores", parsed.get("criteria_scores", {}))
+ if isinstance(scores_data, dict):
+ for key, value in scores_data.items():
+ if isinstance(value, dict) and "score" in value:
+ result["scores"][key] = float(value["score"])
+ elif isinstance(value, (int, float)):
+ result["scores"][key] = float(value)
+
+ # Extract total
+ for key in ("total", "score", "overall_score"):
+ if key in parsed and isinstance(parsed[key], (int, float)):
+ result["total"] = float(parsed[key])
+ break
+ else:
+ if result["scores"]:
+ values = [v for v in result["scores"].values() if isinstance(v, (int, float))]
+ if values:
+ result["total"] = sum(values) / len(values)
+
+ # Normalize summed totals back to 0..1
+ values = [v for v in result["scores"].values() if isinstance(v, (int, float))]
+ if (
+ values
+ and result["total"] is not None
+ and result["total"] > 1.0
+ and all(0.0 <= float(v) <= 1.0 for v in values)
+ ):
+ result["total"] = sum(values) / len(values)
+
+ # Extract notes
+ for key in ("notes", "justification", "reasoning"):
+ if key in parsed:
+ result["notes"] = str(parsed[key])
+ break
+
+ return result
+
+
+# ---------------------------------------------------------------------------
+# Grading functions
+# ---------------------------------------------------------------------------
+
+def _grade_automated(
+ record: EvalRecord,
+ transcript: List[Dict[str, Any]],
+ workspace_path: str,
+) -> Dict[str, Any]:
+ """Run the embedded Python grade() function from the task definition."""
+ code = record.metadata.get("automated_checks")
+ if not code:
+ return {"score": 0.0, "breakdown": {}, "notes": "No automated checks defined"}
+
+ namespace: Dict[str, Any] = {}
+ try:
+ exec(code, namespace) # noqa: S102
+ except Exception as exc:
+ LOGGER.error("Failed to compile grading code for %s: %s", record.record_id, exc)
+ return {"score": 0.0, "breakdown": {}, "notes": f"Grading code error: {exc}"}
+
+ grade_fn = namespace.get("grade")
+ if not callable(grade_fn):
+ return {"score": 0.0, "breakdown": {}, "notes": "No grade() function found in automated checks"}
+
+ try:
+ scores = grade_fn(transcript, workspace_path)
+ except Exception as exc:
+ LOGGER.error("grade() failed for %s: %s", record.record_id, exc)
+ return {"score": 0.0, "breakdown": {}, "notes": f"grade() error: {exc}"}
+
+ if not isinstance(scores, dict) or not scores:
+ return {"score": 0.0, "breakdown": {}, "notes": "grade() returned empty or non-dict"}
+
+ mean_score = sum(scores.values()) / len(scores)
+ return {"score": mean_score, "breakdown": scores, "notes": ""}
+
+
+def _grade_llm_judge(
+ record: EvalRecord,
+ transcript: List[Dict[str, Any]],
+ workspace_path: str,
+ judge_backend: Any,
+ judge_model: str,
+) -> Dict[str, Any]:
+ """Grade using an LLM judge with the task's rubric."""
+ rubric = record.metadata.get("llm_judge_rubric")
+ if not rubric:
+ return {"score": 0.0, "breakdown": {}, "notes": "No LLM judge rubric defined"}
+
+ if judge_backend is None:
+ return {"score": 0.0, "breakdown": {}, "notes": "No judge backend configured"}
+
+ summary = _summarize_transcript(transcript)
+ prompt = _build_judge_prompt(
+ task_prompt=record.problem,
+ expected_behavior=record.reference or "",
+ rubric=rubric,
+ transcript_summary=summary,
+ )
+
+ try:
+ raw = judge_backend.generate(prompt, model=judge_model, temperature=0.0, max_tokens=2048)
+ except Exception as exc:
+ LOGGER.error("LLM judge call failed for %s: %s", record.record_id, exc)
+ return {"score": 0.0, "breakdown": {}, "notes": f"Judge error: {exc}"}
+
+ parsed = _parse_judge_response(raw)
+ return {
+ "score": parsed.get("total", 0.0),
+ "breakdown": parsed.get("scores", {}),
+ "notes": parsed.get("notes", ""),
+ }
+
+
+def _grade_hybrid(
+ record: EvalRecord,
+ transcript: List[Dict[str, Any]],
+ workspace_path: str,
+ judge_backend: Any,
+ judge_model: str,
+) -> Dict[str, Any]:
+ """Run both automated and LLM judge grading, combine with weights."""
+ weights = record.metadata.get("grading_weights", {"automated": 0.5, "llm_judge": 0.5})
+ auto = _grade_automated(record, transcript, workspace_path)
+ llm = _grade_llm_judge(record, transcript, workspace_path, judge_backend, judge_model)
+
+ auto_w = float(weights.get("automated", 0.5))
+ llm_w = float(weights.get("llm_judge", 0.5))
+ total_w = auto_w + llm_w
+
+ combined = (auto["score"] * auto_w + llm["score"] * llm_w) / total_w if total_w > 0 else 0.0
+ breakdown = {
+ **{f"automated.{k}": v for k, v in auto["breakdown"].items()},
+ **{f"llm_judge.{k}": v for k, v in llm["breakdown"].items()},
+ }
+ notes = " | ".join(filter(None, [auto.get("notes", ""), llm.get("notes", "")]))
+ return {"score": combined, "breakdown": breakdown, "notes": notes}
+
+
+def grade_pinchbench_task(
+ *,
+ record: EvalRecord,
+ transcript: List[Dict[str, Any]],
+ workspace_path: str,
+ judge_backend: Any = None,
+ judge_model: str = "anthropic/claude-opus-4-5",
+) -> Dict[str, Any]:
+ """Top-level grading entry point. Routes by grading_type.
+
+ Returns {"score": float, "breakdown": dict, "notes": str}.
+ """
+ grading_type = record.metadata.get("grading_type", "automated")
+
+ if grading_type == "automated":
+ return _grade_automated(record, transcript, workspace_path)
+ elif grading_type == "llm_judge":
+ return _grade_llm_judge(record, transcript, workspace_path, judge_backend, judge_model)
+ elif grading_type == "hybrid":
+ return _grade_hybrid(record, transcript, workspace_path, judge_backend, judge_model)
+ else:
+ return {"score": 0.0, "breakdown": {}, "notes": f"Unknown grading type: {grading_type}"}
+
+
+# ---------------------------------------------------------------------------
+# Standalone scorer (for EvalRunner non-agentic path)
+# ---------------------------------------------------------------------------
+
+class PinchBenchScorer(LLMJudgeScorer):
+ """PinchBench scorer for the non-agentic EvalRunner path."""
+
+ scorer_id = "pinchbench"
+
+ def score(
+ self, record: EvalRecord, model_answer: str,
+ ) -> Tuple[Optional[bool], Dict[str, Any]]:
+ trace = record.metadata.get("query_trace")
+ transcript = _trace_to_transcript(trace) if trace else []
+ result = grade_pinchbench_task(
+ record=record,
+ transcript=transcript,
+ workspace_path=record.metadata.get("workspace_path", ""),
+ judge_backend=self._judge_backend,
+ judge_model=self._judge_model,
+ )
+ is_correct = result["score"] >= 0.5
+ return is_correct, {**result}
+
+
+__all__ = [
+ "PinchBenchScorer",
+ "events_to_transcript",
+ "grade_pinchbench_task",
+]
diff --git a/tests/evals/test_pinchbench_grading.py b/tests/evals/test_pinchbench_grading.py
new file mode 100644
index 00000000..9f37c4d4
--- /dev/null
+++ b/tests/evals/test_pinchbench_grading.py
@@ -0,0 +1,137 @@
+"""Tests for PinchBench grading functions."""
+
+from openjarvis.evals.core.types import EvalRecord
+from openjarvis.evals.scorers.pinchbench import (
+ _grade_automated,
+ _parse_judge_response,
+ _summarize_transcript,
+ grade_pinchbench_task,
+)
+
+
+def _make_record(**meta_overrides) -> EvalRecord:
+ meta = {
+ "grading_type": "automated",
+ "automated_checks": None,
+ "llm_judge_rubric": None,
+ "grading_weights": None,
+ }
+ meta.update(meta_overrides)
+ return EvalRecord(
+ record_id="test_task",
+ problem="Do the thing",
+ reference="Expected behavior",
+ category="test",
+ subject="Test Task",
+ metadata=meta,
+ )
+
+
+class TestGradeAutomated:
+ def test_simple_pass(self, tmp_path):
+ code = '''
+def grade(transcript, workspace_path):
+ from pathlib import Path
+ f = Path(workspace_path) / "output.txt"
+ return {"file_exists": 1.0 if f.exists() else 0.0}
+'''
+ (tmp_path / "output.txt").write_text("hello")
+ record = _make_record(automated_checks=code)
+ result = _grade_automated(record, [], str(tmp_path))
+ assert result["score"] == 1.0
+ assert result["breakdown"]["file_exists"] == 1.0
+
+ def test_simple_fail(self, tmp_path):
+ code = '''
+def grade(transcript, workspace_path):
+ from pathlib import Path
+ f = Path(workspace_path) / "output.txt"
+ return {"file_exists": 1.0 if f.exists() else 0.0}
+'''
+ record = _make_record(automated_checks=code)
+ result = _grade_automated(record, [], str(tmp_path))
+ assert result["score"] == 0.0
+
+ def test_transcript_inspection(self, tmp_path):
+ code = '''
+def grade(transcript, workspace_path):
+ used_read = False
+ for entry in transcript:
+ if entry.get("type") == "message":
+ msg = entry.get("message", {})
+ if msg.get("role") == "assistant":
+ for item in msg.get("content", []):
+ if item.get("type") == "toolCall" and item.get("name") == "read_file":
+ used_read = True
+ return {"used_read_file": 1.0 if used_read else 0.0}
+'''
+ transcript = [{
+ "type": "message",
+ "message": {
+ "role": "assistant",
+ "content": [{"type": "toolCall", "name": "read_file", "arguments": {"path": "a.txt"}}],
+ },
+ }]
+ record = _make_record(automated_checks=code)
+ result = _grade_automated(record, transcript, str(tmp_path))
+ assert result["score"] == 1.0
+
+ def test_no_checks_returns_zero(self, tmp_path):
+ record = _make_record(automated_checks=None)
+ result = _grade_automated(record, [], str(tmp_path))
+ assert result["score"] == 0.0
+
+
+class TestParseJudgeResponse:
+ def test_json_code_block(self):
+ raw = '```json\n{"scores": {"quality": 0.8}, "total": 0.8, "notes": "good"}\n```'
+ parsed = _parse_judge_response(raw)
+ assert parsed["total"] == 0.8
+ assert parsed["scores"]["quality"] == 0.8
+
+ def test_bare_json(self):
+ raw = 'The agent did well. {"scores": {"a": 0.9}, "total": 0.9, "notes": ""}'
+ parsed = _parse_judge_response(raw)
+ assert parsed["total"] == 0.9
+
+ def test_regex_fallback(self):
+ raw = "The agent performed reasonably. Overall score: 0.65"
+ parsed = _parse_judge_response(raw)
+ assert parsed["total"] == 0.65
+
+ def test_empty_response(self):
+ parsed = _parse_judge_response("")
+ assert parsed["total"] == 0.0
+
+
+class TestSummarizeTranscript:
+ def test_tool_call_and_result(self):
+ transcript = [
+ {
+ "type": "message",
+ "message": {
+ "role": "assistant",
+ "content": [{"type": "toolCall", "name": "read_file", "arguments": {"path": "a.txt"}}],
+ },
+ },
+ {
+ "type": "message",
+ "message": {"role": "toolResult", "content": [{"text": "file contents"}]},
+ },
+ ]
+ summary = _summarize_transcript(transcript)
+ assert 'Tool: read_file({"path": "a.txt"})' in summary
+ assert "Result:" in summary
+
+
+class TestGradeRouter:
+ def test_routes_automated(self, tmp_path):
+ code = 'def grade(t, w): return {"ok": 1.0}'
+ record = _make_record(grading_type="automated", automated_checks=code)
+ result = grade_pinchbench_task(record=record, transcript=[], workspace_path=str(tmp_path))
+ assert result["score"] == 1.0
+
+ def test_unknown_type(self, tmp_path):
+ record = _make_record(grading_type="unknown")
+ result = grade_pinchbench_task(record=record, transcript=[], workspace_path=str(tmp_path))
+ assert result["score"] == 0.0
diff --git a/tests/evals/test_pinchbench_transcript.py b/tests/evals/test_pinchbench_transcript.py
index 7218216b..d9fefc88 100644
--- a/tests/evals/test_pinchbench_transcript.py
+++ b/tests/evals/test_pinchbench_transcript.py
@@ -30,3 +30,59 @@ def test_turn_trace_tool_calls_from_dict_missing():
d = {"turn_index": 0}
turn = TurnTrace.from_dict(d)
assert turn.tool_calls == []
+
+
+from openjarvis.evals.core.event_recorder import AgentEvent, EventType
+
+
+def _make_event(etype, **metadata):
+ """Helper to create a mock AgentEvent."""
+ return AgentEvent(event_type=etype, timestamp=0.0, metadata=metadata)
+
+
+def test_events_to_transcript_tool_call_pair():
+ """TOOL_CALL_START + END produces assistant toolCall + toolResult messages."""
+ from openjarvis.evals.scorers.pinchbench import events_to_transcript
+
+ events = [
+ _make_event(EventType.TOOL_CALL_START, tool="file_read", arguments={"path": "a.txt"}),
+ _make_event(EventType.TOOL_CALL_END, tool="file_read", result="file contents"),
+ ]
+ transcript = events_to_transcript(events)
+ assert len(transcript) == 2
+ assert transcript[0]["type"] == "message"
+ assert transcript[0]["message"]["role"] == "assistant"
+ assert transcript[0]["message"]["content"][0]["type"] == "toolCall"
+ # Tool name mapped: file_read -> read_file
+ assert transcript[0]["message"]["content"][0]["name"] == "read_file"
+ assert transcript[1]["message"]["role"] == "toolResult"
+ assert transcript[1]["message"]["content"][0]["text"] == "file contents"
+
+
+def test_events_to_transcript_tool_name_mapping():
+ """OpenJarvis tool names are mapped to PinchBench-expected names."""
+ from openjarvis.evals.scorers.pinchbench import events_to_transcript
+
+ events = [
+ _make_event(EventType.TOOL_CALL_START, tool="image_generate", arguments={"prompt": "cat"}),
+ _make_event(EventType.TOOL_CALL_END, tool="image_generate", result="ok"),
+ ]
+ transcript = events_to_transcript(events)
+ assert transcript[0]["message"]["content"][0]["name"] == "generate_image"
+
+
+def test_events_to_transcript_empty():
+ """Empty events produce empty transcript."""
+ from openjarvis.evals.scorers.pinchbench import events_to_transcript
+ assert events_to_transcript([]) == []
+
+
+def test_events_to_transcript_ignores_non_tool_events():
+ """Non-tool events are skipped."""
+ from openjarvis.evals.scorers.pinchbench import events_to_transcript
+
+ events = [
+ _make_event(EventType.LM_INFERENCE_START),
+ _make_event(EventType.LM_INFERENCE_END, prompt_tokens=10, completion_tokens=5),
+ ]
+ assert events_to_transcript(events) == []