Personal AI, On Personal Devices
Python
test(evals): add PinchBench integration tests — full grading pipeline
Jon Saad-Falcon784b88013061
diff --git a/src/openjarvis/evals/execution/pinchbench_env.py b/src/openjarvis/evals/execution/pinchbench_env.pyindex f492746e..cd11235e 100644--- a/src/openjarvis/evals/execution/pinchbench_env.py+++ b/src/openjarvis/evals/execution/pinchbench_env.py@@ -50,13 +50,16 @@ class PinchBenchTaskEnv:for file_spec in workspace_files:if "content" in file_spec:# Inline content- dest = self._workspace / file_spec.get("path", file_spec.get("dest", "file.txt"))+ dest_key = file_spec.get("path", file_spec.get("dest", "file.txt"))+ dest = self._workspace / dest_keydest.parent.mkdir(parents=True, exist_ok=True)dest.write_text(file_spec["content"])elif "source" in file_spec:# Asset file from reposource = repo_dir / "assets" / file_spec["source"]- dest_key = file_spec.get("dest", file_spec.get("path", file_spec["source"]))+ dest_key = file_spec.get(+ "dest", file_spec.get("path", file_spec["source"])+ )dest = self._workspace / dest_keydest.parent.mkdir(parents=True, exist_ok=True)if source.exists():@@ -65,7 +68,11 @@ class PinchBenchTaskEnv:LOGGER.warning("Asset not found: %s", source)self._record.metadata["workspace_path"] = str(self._workspace)- LOGGER.info("PinchBench workspace: %s (task %s)", self._workspace, self._record.record_id)+ LOGGER.info(+ "PinchBench workspace: %s (task %s)",+ self._workspace,+ self._record.record_id,+ )return selfdef run_tests(self) -> None:diff --git a/tests/evals/test_pinchbench_grading.py b/tests/evals/test_pinchbench_grading.pyindex 9f37c4d4..aa107506 100644--- a/tests/evals/test_pinchbench_grading.py+++ b/tests/evals/test_pinchbench_grading.py@@ -61,7 +61,8 @@ def grade(transcript, workspace_path):msg = entry.get("message", {})if msg.get("role") == "assistant":for item in msg.get("content", []):- if item.get("type") == "toolCall" and item.get("name") == "read_file":+ name = item.get("name")+ if item.get("type") == "toolCall" and name == "read_file":used_read = Truereturn {"used_read_file": 1.0 if used_read else 0.0}'''@@ -69,7 +70,13 @@ def grade(transcript, workspace_path):"type": "message","message": {"role": "assistant",- "content": [{"type": "toolCall", "name": "read_file", "arguments": {"path": "a.txt"}}],+ "content": [+ {+ "type": "toolCall",+ "name": "read_file",+ "arguments": {"path": "a.txt"},+ }+ ],},}]record = _make_record(automated_checks=code)@@ -84,7 +91,9 @@ def grade(transcript, workspace_path):class TestParseJudgeResponse:def test_json_code_block(self):- raw = '```json\n{"scores": {"quality": 0.8}, "total": 0.8, "notes": "good"}\n```'+ raw = (+ '```json\n{"scores": {"quality": 0.8}, "total": 0.8, "notes": "good"}\n```'+ )parsed = _parse_judge_response(raw)assert parsed["total"] == 0.8assert parsed["scores"]["quality"] == 0.8@@ -111,12 +120,21 @@ class TestSummarizeTranscript:"type": "message","message": {"role": "assistant",- "content": [{"type": "toolCall", "name": "read_file", "arguments": {"path": "a.txt"}}],+ "content": [+ {+ "type": "toolCall",+ "name": "read_file",+ "arguments": {"path": "a.txt"},+ }+ ],},},{"type": "message",- "message": {"role": "toolResult", "content": [{"text": "file contents"}]},+ "message": {+ "role": "toolResult",+ "content": [{"text": "file contents"}],+ },},]summary = _summarize_transcript(transcript)@@ -128,10 +146,14 @@ class TestGradeRouter:def test_routes_automated(self, tmp_path):code = 'def grade(t, w): return {"ok": 1.0}'record = _make_record(grading_type="automated", automated_checks=code)- result = grade_pinchbench_task(record=record, transcript=[], workspace_path=str(tmp_path))+ result = grade_pinchbench_task(+ record=record, transcript=[], workspace_path=str(tmp_path)+ )assert result["score"] == 1.0def test_unknown_type(self, tmp_path):record = _make_record(grading_type="unknown")- result = grade_pinchbench_task(record=record, transcript=[], workspace_path=str(tmp_path))+ result = grade_pinchbench_task(+ record=record, transcript=[], workspace_path=str(tmp_path)+ )assert result["score"] == 0.0diff --git a/tests/evals/test_pinchbench_integration.py b/tests/evals/test_pinchbench_integration.pynew file mode 100644index 00000000..4edf9334--- /dev/null+++ b/tests/evals/test_pinchbench_integration.py@@ -0,0 +1,121 @@+"""Integration tests for PinchBench eval pipeline.++These tests use synthetic task files to verify the full pipeline+without requiring the actual PinchBench repo or cloud API keys.+"""++import textwrap+from pathlib import Path++from openjarvis.evals.core.event_recorder import EventRecorder, EventType+from openjarvis.evals.datasets.pinchbench import PinchBenchDataset+++def _create_test_repo(tmp_path: Path) -> Path:+ """Create a minimal PinchBench repo structure for testing."""+ repo = tmp_path / "skill"+ tasks_dir = repo / "tasks"+ tasks_dir.mkdir(parents=True)+ assets_dir = repo / "assets"+ assets_dir.mkdir()++ # Write a simple automated task+ (tasks_dir / "task_00_sanity.md").write_text(textwrap.dedent("""\+ ---+ id: task_00_sanity+ name: Sanity Check+ category: basic+ grading_type: automated+ timeout_seconds: 60+ workspace_files: []+ ---++ ## Prompt++ Write "hello" to a file called output.txt.++ ## Expected Behavior++ A file output.txt exists with the word hello.++ ## Grading Criteria++ - [ ] output.txt exists+ - [ ] Contains hello++ ## Automated Checks++ ```python+ def grade(transcript, workspace_path):+ from pathlib import Path+ f = Path(workspace_path) / "output.txt"+ exists = f.exists()+ has_hello = "hello" in f.read_text().lower() if exists else False+ return {+ "file_exists": 1.0 if exists else 0.0,+ "has_hello": 1.0 if has_hello else 0.0,+ }+ ```+ """))++ return repo+++def test_dataset_loads_from_local_path(tmp_path):+ """Dataset loads tasks from a local path."""+ repo = _create_test_repo(tmp_path)+ ds = PinchBenchDataset(path=str(repo))+ ds.load()+ assert ds.size() == 1+ records = list(ds.iter_records())+ assert records[0].record_id == "task_00_sanity"+ assert "hello" in records[0].problem.lower() or "output.txt" in records[0].problem+++def test_task_env_creates_workspace(tmp_path):+ """Task env creates and cleans up workspace."""+ repo = _create_test_repo(tmp_path)+ ds = PinchBenchDataset(path=str(repo))+ ds.load()+ record = list(ds.iter_records())[0]++ env = ds.create_task_env(record)+ with env:+ ws = Path(record.metadata["workspace_path"])+ assert ws.exists()+ # After exit, workspace is cleaned up+ assert not ws.exists()+++def test_full_grading_pipeline(tmp_path):+ """Full pipeline: workspace setup -> simulate agent -> grade."""+ repo = _create_test_repo(tmp_path)+ ds = PinchBenchDataset(path=str(repo))+ ds.load()+ record = list(ds.iter_records())[0]++ env = ds.create_task_env(record)+ with env:+ ws = Path(record.metadata["workspace_path"])+ # Simulate agent writing the output file+ (ws / "output.txt").write_text("hello world")++ # Simulate EventRecorder with a tool call+ recorder = EventRecorder()+ recorder.record(+ EventType.TOOL_CALL_START,+ tool="file_write",+ arguments={"path": "output.txt", "content": "hello world"},+ )+ recorder.record(+ EventType.TOOL_CALL_END,+ tool="file_write",+ result="ok",+ )+ env.set_event_recorder(recorder)+ env.run_tests()++ assert record.metadata["is_resolved"] is True+ assert record.metadata["pinchbench_score"] == 1.0+ assert record.metadata["pinchbench_breakdown"]["file_exists"] == 1.0+ assert record.metadata["pinchbench_breakdown"]["has_hello"] == 1.0diff --git a/tests/evals/test_pinchbench_transcript.py b/tests/evals/test_pinchbench_transcript.pyindex d9fefc88..d7850c6b 100644--- a/tests/evals/test_pinchbench_transcript.py+++ b/tests/evals/test_pinchbench_transcript.py@@ -1,5 +1,6 @@"""Tests for TurnTrace tool_calls field and PinchBench transcript translation."""+from openjarvis.evals.core.event_recorder import AgentEvent, EventTypefrom openjarvis.evals.core.trace import TurnTrace@@ -32,9 +33,6 @@ def test_turn_trace_tool_calls_from_dict_missing():assert turn.tool_calls == []-from openjarvis.evals.core.event_recorder import AgentEvent, EventType--def _make_event(etype, **metadata):"""Helper to create a mock AgentEvent."""return AgentEvent(event_type=etype, timestamp=0.0, metadata=metadata)@@ -45,7 +43,9 @@ def test_events_to_transcript_tool_call_pair():from openjarvis.evals.scorers.pinchbench import events_to_transcriptevents = [- _make_event(EventType.TOOL_CALL_START, tool="file_read", arguments={"path": "a.txt"}),+ _make_event(+ EventType.TOOL_CALL_START, tool="file_read", arguments={"path": "a.txt"}+ ),_make_event(EventType.TOOL_CALL_END, tool="file_read", result="file contents"),]transcript = events_to_transcript(events)@@ -64,7 +64,11 @@ def test_events_to_transcript_tool_name_mapping():from openjarvis.evals.scorers.pinchbench import events_to_transcriptevents = [- _make_event(EventType.TOOL_CALL_START, tool="image_generate", arguments={"prompt": "cat"}),+ _make_event(+ EventType.TOOL_CALL_START,+ tool="image_generate",+ arguments={"prompt": "cat"},+ ),_make_event(EventType.TOOL_CALL_END, tool="image_generate", result="ok"),]transcript = events_to_transcript(events)