Personal AI, On Personal Devices
Python
feat(evals): add PinchBench grading helpers — transcript translation, automated/LLM judge/hybrid scoring
Jon Saad-Falcon73cb1f1bbdf1
diff --git a/src/openjarvis/evals/scorers/pinchbench.py b/src/openjarvis/evals/scorers/pinchbench.pynew file mode 100644index 00000000..68269ec0--- /dev/null+++ b/src/openjarvis/evals/scorers/pinchbench.py@@ -0,0 +1,445 @@+"""PinchBench grading helpers and scorer.++Provides transcript translation (OpenJarvis events → PinchBench format),+automated grading (exec of embedded Python), LLM judge grading, and+hybrid combination. Used by PinchBenchTaskEnv.run_tests() and the+standalone PinchBenchScorer.++Reference: https://github.com/pinchbench/skill+"""++from __future__ import annotations++import json+import logging+import re+from typing import Any, Dict, List, Optional, Tuple++from openjarvis.evals.core.event_recorder import EventType+from openjarvis.evals.core.scorer import LLMJudgeScorer+from openjarvis.evals.core.types import EvalRecord++LOGGER = logging.getLogger(__name__)++# ---------------------------------------------------------------------------+# Tool name mapping: OpenJarvis name → PinchBench-expected name+# ---------------------------------------------------------------------------+_TOOL_NAME_MAP: Dict[str, str] = {+ "file_read": "read_file",+ "file_write": "write_file",+ "image_generate": "generate_image",+ # web_search, calculator, shell_exec, etc. use the same names+}+++# ---------------------------------------------------------------------------+# Transcript translation+# ---------------------------------------------------------------------------++def events_to_transcript(events: List[Any]) -> List[Dict[str, Any]]:+ """Build PinchBench-format transcript from raw EventRecorder events.++ Pairs TOOL_CALL_START/END events to extract tool name, arguments, and+ results. Called by run_tests() before QueryTrace exists.+ """+ transcript: List[Dict[str, Any]] = []++ for event in events:+ etype = event.event_type+ if isinstance(etype, str):+ # Normalize string event types to enum comparison+ pass+ if etype == EventType.TOOL_CALL_START or etype == EventType.TOOL_CALL_START.value:+ tool_name = event.metadata.get("tool", "unknown")+ mapped = _TOOL_NAME_MAP.get(tool_name, tool_name)+ arguments = event.metadata.get("arguments", {})+ transcript.append({+ "type": "message",+ "message": {+ "role": "assistant",+ "content": [{"type": "toolCall", "name": mapped, "arguments": arguments}],+ },+ })+ elif etype == EventType.TOOL_CALL_END or etype == EventType.TOOL_CALL_END.value:+ result_text = str(event.metadata.get("result", ""))+ transcript.append({+ "type": "message",+ "message": {+ "role": "toolResult",+ "content": [{"text": result_text}],+ },+ })++ return transcript+++def _trace_to_transcript(trace: Any) -> List[Dict[str, Any]]:+ """Convert QueryTrace to PinchBench transcript (for EvalRunner path)."""+ transcript: List[Dict[str, Any]] = []+ for turn in trace.turns:+ for tc in getattr(turn, "tool_calls", []):+ mapped = _TOOL_NAME_MAP.get(tc["name"], tc["name"])+ transcript.append({+ "type": "message",+ "message": {+ "role": "assistant",+ "content": [{"type": "toolCall", "name": mapped, "arguments": tc.get("arguments", {})}],+ },+ })+ transcript.append({+ "type": "message",+ "message": {+ "role": "toolResult",+ "content": [{"text": tc.get("result", "")}],+ },+ })+ return transcript+++# ---------------------------------------------------------------------------+# Transcript summarization (replicates PinchBench _summarize_transcript)+# ---------------------------------------------------------------------------++def _summarize_transcript(transcript: List[Dict[str, Any]]) -> str:+ """Summarize transcript for LLM judge prompt.++ Produces the same format as PinchBench's lib_grading._summarize_transcript():+ Tool: function_name({"arg1": value1})+ Result: [first 200 chars]+ User: [message text]+ """+ parts: List[str] = []+ for event in transcript:+ if event.get("type") != "message":+ continue+ msg = event.get("message", {})+ role = msg.get("role")+ if role == "assistant":+ for item in msg.get("content", []):+ if item.get("type") == "toolCall":+ parts.append(+ f"Tool: {item.get('name')}({json.dumps(item.get('arguments', {}))})"+ )+ elif role == "toolResult":+ content = msg.get("content", [])+ if content:+ preview = str(content[0])[:200]+ parts.append(f"Result: {preview}")+ elif role == "user":+ content = msg.get("content", [])+ if content:+ parts.append(f"User: {content[0]}")+ return "\n".join(parts)+++# ---------------------------------------------------------------------------+# Judge prompt and response parsing (replicates PinchBench exactly)+# ---------------------------------------------------------------------------++def _build_judge_prompt(+ *,+ task_prompt: str,+ expected_behavior: str,+ rubric: str,+ transcript_summary: str,+) -> str:+ """Build the LLM judge prompt matching PinchBench's format."""+ return f"""You are a strict evaluator assessing an AI agent's performance on a task.+Reserve a score of 1.0 for genuinely excellent performance.++## Task+{task_prompt}++## Expected Behavior+{expected_behavior}++## Agent Transcript+{transcript_summary}++## Grading Rubric+{rubric}++## Instructions+Evaluate the agent's performance against each criterion in the rubric.+Return your evaluation as JSON with this exact structure:+```json+{{+ "scores": {{"criterion_name": score, ...}},+ "total": overall_score_0_to_1,+ "notes": "brief justification"+}}+```+Be a strict evaluator. Deduct points for unnecessary steps, verbose output,+or inefficient tool usage."""+++def _parse_judge_response(raw: str) -> Dict[str, Any]:+ """Parse LLM judge response with fallback chain.++ Tries: JSON code block → balanced braces → regex score extraction.+ Matches PinchBench's lib_grading._parse_judge_response() logic.+ """+ if not raw or not raw.strip():+ return {"scores": {}, "total": 0.0, "notes": "Empty judge response"}++ # Try JSON code block+ code_block = re.search(r"```json\s*(.*?)\s*```", raw, re.DOTALL)+ if code_block:+ try:+ parsed = json.loads(code_block.group(1))+ if isinstance(parsed, dict):+ return _normalize_judge_response(parsed)+ except json.JSONDecodeError:+ pass++ # Try balanced braces extraction+ candidates: List[str] = []+ depth = 0+ current: List[str] = []+ for char in raw:+ if char == "{":+ if depth == 0:+ current = []+ depth += 1+ if depth > 0:+ current.append(char)+ if char == "}":+ depth -= 1+ if depth == 0 and current:+ candidates.append("".join(current))++ for candidate in reversed(candidates):+ try:+ parsed = json.loads(candidate)+ if isinstance(parsed, dict) and "scores" in parsed:+ return _normalize_judge_response(parsed)+ except json.JSONDecodeError:+ continue++ for candidate in reversed(candidates):+ try:+ parsed = json.loads(candidate)+ if isinstance(parsed, dict):+ return _normalize_judge_response(parsed)+ except json.JSONDecodeError:+ continue++ # Regex fallback for prose scores+ score_match = re.search(+ r"(?:total|overall|final)\s*(?:score)?[:\s]*(0\.\d+|1\.0+)",+ raw,+ re.IGNORECASE,+ )+ if score_match:+ try:+ total = float(score_match.group(1))+ if 0.0 <= total <= 1.0:+ LOGGER.warning("Fell back to regex score extraction (total=%.2f)", total)+ return {"scores": {}, "total": total, "notes": "Score extracted from prose"}+ except ValueError:+ pass++ LOGGER.warning("Failed to parse judge response")+ return {"scores": {}, "total": 0.0, "notes": "Failed to parse judge response"}+++def _normalize_judge_response(parsed: Dict[str, Any]) -> Dict[str, Any]:+ """Normalize various judge response formats to standard structure.++ Matches PinchBench's lib_grading._normalize_judge_response().+ """+ result: Dict[str, Any] = {"scores": {}, "total": 0.0, "notes": ""}++ # Extract scores+ scores_data = parsed.get("scores", parsed.get("criteria_scores", {}))+ if isinstance(scores_data, dict):+ for key, value in scores_data.items():+ if isinstance(value, dict) and "score" in value:+ result["scores"][key] = float(value["score"])+ elif isinstance(value, (int, float)):+ result["scores"][key] = float(value)++ # Extract total+ for key in ("total", "score", "overall_score"):+ if key in parsed and isinstance(parsed[key], (int, float)):+ result["total"] = float(parsed[key])+ break+ else:+ if result["scores"]:+ values = [v for v in result["scores"].values() if isinstance(v, (int, float))]+ if values:+ result["total"] = sum(values) / len(values)++ # Normalize summed totals back to 0..1+ values = [v for v in result["scores"].values() if isinstance(v, (int, float))]+ if (+ values+ and result["total"] is not None+ and result["total"] > 1.0+ and all(0.0 <= float(v) <= 1.0 for v in values)+ ):+ result["total"] = sum(values) / len(values)++ # Extract notes+ for key in ("notes", "justification", "reasoning"):+ if key in parsed:+ result["notes"] = str(parsed[key])+ break++ return result+++# ---------------------------------------------------------------------------+# Grading functions+# ---------------------------------------------------------------------------++def _grade_automated(+ record: EvalRecord,+ transcript: List[Dict[str, Any]],+ workspace_path: str,+) -> Dict[str, Any]:+ """Run the embedded Python grade() function from the task definition."""+ code = record.metadata.get("automated_checks")+ if not code:+ return {"score": 0.0, "breakdown": {}, "notes": "No automated checks defined"}++ namespace: Dict[str, Any] = {}+ try:+ exec(code, namespace) # noqa: S102+ except Exception as exc:+ LOGGER.error("Failed to compile grading code for %s: %s", record.record_id, exc)+ return {"score": 0.0, "breakdown": {}, "notes": f"Grading code error: {exc}"}++ grade_fn = namespace.get("grade")+ if not callable(grade_fn):+ return {"score": 0.0, "breakdown": {}, "notes": "No grade() function found in automated checks"}++ try:+ scores = grade_fn(transcript, workspace_path)+ except Exception as exc:+ LOGGER.error("grade() failed for %s: %s", record.record_id, exc)+ return {"score": 0.0, "breakdown": {}, "notes": f"grade() error: {exc}"}++ if not isinstance(scores, dict) or not scores:+ return {"score": 0.0, "breakdown": {}, "notes": "grade() returned empty or non-dict"}++ mean_score = sum(scores.values()) / len(scores)+ return {"score": mean_score, "breakdown": scores, "notes": ""}+++def _grade_llm_judge(+ record: EvalRecord,+ transcript: List[Dict[str, Any]],+ workspace_path: str,+ judge_backend: Any,+ judge_model: str,+) -> Dict[str, Any]:+ """Grade using an LLM judge with the task's rubric."""+ rubric = record.metadata.get("llm_judge_rubric")+ if not rubric:+ return {"score": 0.0, "breakdown": {}, "notes": "No LLM judge rubric defined"}++ if judge_backend is None:+ return {"score": 0.0, "breakdown": {}, "notes": "No judge backend configured"}++ summary = _summarize_transcript(transcript)+ prompt = _build_judge_prompt(+ task_prompt=record.problem,+ expected_behavior=record.reference or "",+ rubric=rubric,+ transcript_summary=summary,+ )++ try:+ raw = judge_backend.generate(prompt, model=judge_model, temperature=0.0, max_tokens=2048)+ except Exception as exc:+ LOGGER.error("LLM judge call failed for %s: %s", record.record_id, exc)+ return {"score": 0.0, "breakdown": {}, "notes": f"Judge error: {exc}"}++ parsed = _parse_judge_response(raw)+ return {+ "score": parsed.get("total", 0.0),+ "breakdown": parsed.get("scores", {}),+ "notes": parsed.get("notes", ""),+ }+++def _grade_hybrid(+ record: EvalRecord,+ transcript: List[Dict[str, Any]],+ workspace_path: str,+ judge_backend: Any,+ judge_model: str,+) -> Dict[str, Any]:+ """Run both automated and LLM judge grading, combine with weights."""+ weights = record.metadata.get("grading_weights", {"automated": 0.5, "llm_judge": 0.5})+ auto = _grade_automated(record, transcript, workspace_path)+ llm = _grade_llm_judge(record, transcript, workspace_path, judge_backend, judge_model)++ auto_w = float(weights.get("automated", 0.5))+ llm_w = float(weights.get("llm_judge", 0.5))+ total_w = auto_w + llm_w++ combined = (auto["score"] * auto_w + llm["score"] * llm_w) / total_w if total_w > 0 else 0.0+ breakdown = {+ **{f"automated.{k}": v for k, v in auto["breakdown"].items()},+ **{f"llm_judge.{k}": v for k, v in llm["breakdown"].items()},+ }+ notes = " | ".join(filter(None, [auto.get("notes", ""), llm.get("notes", "")]))+ return {"score": combined, "breakdown": breakdown, "notes": notes}+++def grade_pinchbench_task(+ *,+ record: EvalRecord,+ transcript: List[Dict[str, Any]],+ workspace_path: str,+ judge_backend: Any = None,+ judge_model: str = "anthropic/claude-opus-4-5",+) -> Dict[str, Any]:+ """Top-level grading entry point. Routes by grading_type.++ Returns {"score": float, "breakdown": dict, "notes": str}.+ """+ grading_type = record.metadata.get("grading_type", "automated")++ if grading_type == "automated":+ return _grade_automated(record, transcript, workspace_path)+ elif grading_type == "llm_judge":+ return _grade_llm_judge(record, transcript, workspace_path, judge_backend, judge_model)+ elif grading_type == "hybrid":+ return _grade_hybrid(record, transcript, workspace_path, judge_backend, judge_model)+ else:+ return {"score": 0.0, "breakdown": {}, "notes": f"Unknown grading type: {grading_type}"}+++# ---------------------------------------------------------------------------+# Standalone scorer (for EvalRunner non-agentic path)+# ---------------------------------------------------------------------------++class PinchBenchScorer(LLMJudgeScorer):+ """PinchBench scorer for the non-agentic EvalRunner path."""++ scorer_id = "pinchbench"++ def score(+ self, record: EvalRecord, model_answer: str,+ ) -> Tuple[Optional[bool], Dict[str, Any]]:+ trace = record.metadata.get("query_trace")+ transcript = _trace_to_transcript(trace) if trace else []+ result = grade_pinchbench_task(+ record=record,+ transcript=transcript,+ workspace_path=record.metadata.get("workspace_path", ""),+ judge_backend=self._judge_backend,+ judge_model=self._judge_model,+ )+ is_correct = result["score"] >= 0.5+ return is_correct, {**result}+++__all__ = [+ "PinchBenchScorer",+ "events_to_transcript",+ "grade_pinchbench_task",+]diff --git a/tests/evals/test_pinchbench_grading.py b/tests/evals/test_pinchbench_grading.pynew file mode 100644index 00000000..9f37c4d4--- /dev/null+++ b/tests/evals/test_pinchbench_grading.py@@ -0,0 +1,137 @@+"""Tests for PinchBench grading functions."""++from openjarvis.evals.core.types import EvalRecord+from openjarvis.evals.scorers.pinchbench import (+ _grade_automated,+ _parse_judge_response,+ _summarize_transcript,+ grade_pinchbench_task,+)+++def _make_record(**meta_overrides) -> EvalRecord:+ meta = {+ "grading_type": "automated",+ "automated_checks": None,+ "llm_judge_rubric": None,+ "grading_weights": None,+ }+ meta.update(meta_overrides)+ return EvalRecord(+ record_id="test_task",+ problem="Do the thing",+ reference="Expected behavior",+ category="test",+ subject="Test Task",+ metadata=meta,+ )+++class TestGradeAutomated:+ def test_simple_pass(self, tmp_path):+ code = '''+def grade(transcript, workspace_path):+ from pathlib import Path+ f = Path(workspace_path) / "output.txt"+ return {"file_exists": 1.0 if f.exists() else 0.0}+'''+ (tmp_path / "output.txt").write_text("hello")+ record = _make_record(automated_checks=code)+ result = _grade_automated(record, [], str(tmp_path))+ assert result["score"] == 1.0+ assert result["breakdown"]["file_exists"] == 1.0++ def test_simple_fail(self, tmp_path):+ code = '''+def grade(transcript, workspace_path):+ from pathlib import Path+ f = Path(workspace_path) / "output.txt"+ return {"file_exists": 1.0 if f.exists() else 0.0}+'''+ record = _make_record(automated_checks=code)+ result = _grade_automated(record, [], str(tmp_path))+ assert result["score"] == 0.0++ def test_transcript_inspection(self, tmp_path):+ code = '''+def grade(transcript, workspace_path):+ used_read = False+ for entry in transcript:+ if entry.get("type") == "message":+ msg = entry.get("message", {})+ if msg.get("role") == "assistant":+ for item in msg.get("content", []):+ if item.get("type") == "toolCall" and item.get("name") == "read_file":+ used_read = True+ return {"used_read_file": 1.0 if used_read else 0.0}+'''+ transcript = [{+ "type": "message",+ "message": {+ "role": "assistant",+ "content": [{"type": "toolCall", "name": "read_file", "arguments": {"path": "a.txt"}}],+ },+ }]+ record = _make_record(automated_checks=code)+ result = _grade_automated(record, transcript, str(tmp_path))+ assert result["score"] == 1.0++ def test_no_checks_returns_zero(self, tmp_path):+ record = _make_record(automated_checks=None)+ result = _grade_automated(record, [], str(tmp_path))+ assert result["score"] == 0.0+++class TestParseJudgeResponse:+ def test_json_code_block(self):+ raw = '```json\n{"scores": {"quality": 0.8}, "total": 0.8, "notes": "good"}\n```'+ parsed = _parse_judge_response(raw)+ assert parsed["total"] == 0.8+ assert parsed["scores"]["quality"] == 0.8++ def test_bare_json(self):+ raw = 'The agent did well. {"scores": {"a": 0.9}, "total": 0.9, "notes": ""}'+ parsed = _parse_judge_response(raw)+ assert parsed["total"] == 0.9++ def test_regex_fallback(self):+ raw = "The agent performed reasonably. Overall score: 0.65"+ parsed = _parse_judge_response(raw)+ assert parsed["total"] == 0.65++ def test_empty_response(self):+ parsed = _parse_judge_response("")+ assert parsed["total"] == 0.0+++class TestSummarizeTranscript:+ def test_tool_call_and_result(self):+ transcript = [+ {+ "type": "message",+ "message": {+ "role": "assistant",+ "content": [{"type": "toolCall", "name": "read_file", "arguments": {"path": "a.txt"}}],+ },+ },+ {+ "type": "message",+ "message": {"role": "toolResult", "content": [{"text": "file contents"}]},+ },+ ]+ summary = _summarize_transcript(transcript)+ assert 'Tool: read_file({"path": "a.txt"})' in summary+ assert "Result:" in summary+++class TestGradeRouter:+ def test_routes_automated(self, tmp_path):+ code = 'def grade(t, w): return {"ok": 1.0}'+ record = _make_record(grading_type="automated", automated_checks=code)+ result = grade_pinchbench_task(record=record, transcript=[], workspace_path=str(tmp_path))+ assert result["score"] == 1.0++ def test_unknown_type(self, tmp_path):+ record = _make_record(grading_type="unknown")+ result = grade_pinchbench_task(record=record, transcript=[], workspace_path=str(tmp_path))+ assert result["score"] == 0.0diff --git a/tests/evals/test_pinchbench_transcript.py b/tests/evals/test_pinchbench_transcript.pyindex 7218216b..d9fefc88 100644--- a/tests/evals/test_pinchbench_transcript.py+++ b/tests/evals/test_pinchbench_transcript.py@@ -30,3 +30,59 @@ def test_turn_trace_tool_calls_from_dict_missing():d = {"turn_index": 0}turn = TurnTrace.from_dict(d)assert turn.tool_calls == []+++from openjarvis.evals.core.event_recorder import AgentEvent, EventType+++def _make_event(etype, **metadata):+ """Helper to create a mock AgentEvent."""+ return AgentEvent(event_type=etype, timestamp=0.0, metadata=metadata)+++def test_events_to_transcript_tool_call_pair():+ """TOOL_CALL_START + END produces assistant toolCall + toolResult messages."""+ from openjarvis.evals.scorers.pinchbench import events_to_transcript++ events = [+ _make_event(EventType.TOOL_CALL_START, tool="file_read", arguments={"path": "a.txt"}),+ _make_event(EventType.TOOL_CALL_END, tool="file_read", result="file contents"),+ ]+ transcript = events_to_transcript(events)+ assert len(transcript) == 2+ assert transcript[0]["type"] == "message"+ assert transcript[0]["message"]["role"] == "assistant"+ assert transcript[0]["message"]["content"][0]["type"] == "toolCall"+ # Tool name mapped: file_read -> read_file+ assert transcript[0]["message"]["content"][0]["name"] == "read_file"+ assert transcript[1]["message"]["role"] == "toolResult"+ assert transcript[1]["message"]["content"][0]["text"] == "file contents"+++def test_events_to_transcript_tool_name_mapping():+ """OpenJarvis tool names are mapped to PinchBench-expected names."""+ from openjarvis.evals.scorers.pinchbench import events_to_transcript++ events = [+ _make_event(EventType.TOOL_CALL_START, tool="image_generate", arguments={"prompt": "cat"}),+ _make_event(EventType.TOOL_CALL_END, tool="image_generate", result="ok"),+ ]+ transcript = events_to_transcript(events)+ assert transcript[0]["message"]["content"][0]["name"] == "generate_image"+++def test_events_to_transcript_empty():+ """Empty events produce empty transcript."""+ from openjarvis.evals.scorers.pinchbench import events_to_transcript+ assert events_to_transcript([]) == []+++def test_events_to_transcript_ignores_non_tool_events():+ """Non-tool events are skipped."""+ from openjarvis.evals.scorers.pinchbench import events_to_transcript++ events = [+ _make_event(EventType.LM_INFERENCE_START),+ _make_event(EventType.LM_INFERENCE_END, prompt_tokens=10, completion_tokens=5),+ ]+ assert events_to_transcript(events) == []