Skip to content

Personal AI, On Personal Devices

Python

test(evals): add PinchBench integration tests — full grading pipeline

Jon Saad-Falcon784b88013061
diff --git a/src/openjarvis/evals/execution/pinchbench_env.py b/src/openjarvis/evals/execution/pinchbench_env.py
index f492746e..cd11235e 100644
--- a/src/openjarvis/evals/execution/pinchbench_env.py
+++ b/src/openjarvis/evals/execution/pinchbench_env.py
@@ -50,13 +50,16 @@ class PinchBenchTaskEnv:
for file_spec in workspace_files:
if "content" in file_spec:
# Inline content
- dest = self._workspace / file_spec.get("path", file_spec.get("dest", "file.txt"))
+ dest_key = file_spec.get("path", file_spec.get("dest", "file.txt"))
+ dest = self._workspace / dest_key
dest.parent.mkdir(parents=True, exist_ok=True)
dest.write_text(file_spec["content"])
elif "source" in file_spec:
# Asset file from repo
source = repo_dir / "assets" / file_spec["source"]
- dest_key = file_spec.get("dest", file_spec.get("path", file_spec["source"]))
+ dest_key = file_spec.get(
+ "dest", file_spec.get("path", file_spec["source"])
+ )
dest = self._workspace / dest_key
dest.parent.mkdir(parents=True, exist_ok=True)
if source.exists():
@@ -65,7 +68,11 @@ class PinchBenchTaskEnv:
LOGGER.warning("Asset not found: %s", source)
self._record.metadata["workspace_path"] = str(self._workspace)
- LOGGER.info("PinchBench workspace: %s (task %s)", self._workspace, self._record.record_id)
+ LOGGER.info(
+ "PinchBench workspace: %s (task %s)",
+ self._workspace,
+ self._record.record_id,
+ )
return self
def run_tests(self) -> None:
diff --git a/tests/evals/test_pinchbench_grading.py b/tests/evals/test_pinchbench_grading.py
index 9f37c4d4..aa107506 100644
--- a/tests/evals/test_pinchbench_grading.py
+++ b/tests/evals/test_pinchbench_grading.py
@@ -61,7 +61,8 @@ def grade(transcript, workspace_path):
msg = entry.get("message", {})
if msg.get("role") == "assistant":
for item in msg.get("content", []):
- if item.get("type") == "toolCall" and item.get("name") == "read_file":
+ name = item.get("name")
+ if item.get("type") == "toolCall" and name == "read_file":
used_read = True
return {"used_read_file": 1.0 if used_read else 0.0}
'''
@@ -69,7 +70,13 @@ def grade(transcript, workspace_path):
"type": "message",
"message": {
"role": "assistant",
- "content": [{"type": "toolCall", "name": "read_file", "arguments": {"path": "a.txt"}}],
+ "content": [
+ {
+ "type": "toolCall",
+ "name": "read_file",
+ "arguments": {"path": "a.txt"},
+ }
+ ],
},
}]
record = _make_record(automated_checks=code)
@@ -84,7 +91,9 @@ def grade(transcript, workspace_path):
class TestParseJudgeResponse:
def test_json_code_block(self):
- raw = '```json\n{"scores": {"quality": 0.8}, "total": 0.8, "notes": "good"}\n```'
+ raw = (
+ '```json\n{"scores": {"quality": 0.8}, "total": 0.8, "notes": "good"}\n```'
+ )
parsed = _parse_judge_response(raw)
assert parsed["total"] == 0.8
assert parsed["scores"]["quality"] == 0.8
@@ -111,12 +120,21 @@ class TestSummarizeTranscript:
"type": "message",
"message": {
"role": "assistant",
- "content": [{"type": "toolCall", "name": "read_file", "arguments": {"path": "a.txt"}}],
+ "content": [
+ {
+ "type": "toolCall",
+ "name": "read_file",
+ "arguments": {"path": "a.txt"},
+ }
+ ],
},
},
{
"type": "message",
- "message": {"role": "toolResult", "content": [{"text": "file contents"}]},
+ "message": {
+ "role": "toolResult",
+ "content": [{"text": "file contents"}],
+ },
},
]
summary = _summarize_transcript(transcript)
@@ -128,10 +146,14 @@ class TestGradeRouter:
def test_routes_automated(self, tmp_path):
code = 'def grade(t, w): return {"ok": 1.0}'
record = _make_record(grading_type="automated", automated_checks=code)
- result = grade_pinchbench_task(record=record, transcript=[], workspace_path=str(tmp_path))
+ result = grade_pinchbench_task(
+ record=record, transcript=[], workspace_path=str(tmp_path)
+ )
assert result["score"] == 1.0
def test_unknown_type(self, tmp_path):
record = _make_record(grading_type="unknown")
- result = grade_pinchbench_task(record=record, transcript=[], workspace_path=str(tmp_path))
+ result = grade_pinchbench_task(
+ record=record, transcript=[], workspace_path=str(tmp_path)
+ )
assert result["score"] == 0.0
diff --git a/tests/evals/test_pinchbench_integration.py b/tests/evals/test_pinchbench_integration.py
new file mode 100644
index 00000000..4edf9334
--- /dev/null
+++ b/tests/evals/test_pinchbench_integration.py
@@ -0,0 +1,121 @@
+"""Integration tests for PinchBench eval pipeline.
+
+These tests use synthetic task files to verify the full pipeline
+without requiring the actual PinchBench repo or cloud API keys.
+"""
+
+import textwrap
+from pathlib import Path
+
+from openjarvis.evals.core.event_recorder import EventRecorder, EventType
+from openjarvis.evals.datasets.pinchbench import PinchBenchDataset
+
+
+def _create_test_repo(tmp_path: Path) -> Path:
+ """Create a minimal PinchBench repo structure for testing."""
+ repo = tmp_path / "skill"
+ tasks_dir = repo / "tasks"
+ tasks_dir.mkdir(parents=True)
+ assets_dir = repo / "assets"
+ assets_dir.mkdir()
+
+ # Write a simple automated task
+ (tasks_dir / "task_00_sanity.md").write_text(textwrap.dedent("""\
+ ---
+ id: task_00_sanity
+ name: Sanity Check
+ category: basic
+ grading_type: automated
+ timeout_seconds: 60
+ workspace_files: []
+ ---
+
+ ## Prompt
+
+ Write "hello" to a file called output.txt.
+
+ ## Expected Behavior
+
+ A file output.txt exists with the word hello.
+
+ ## Grading Criteria
+
+ - [ ] output.txt exists
+ - [ ] Contains hello
+
+ ## Automated Checks
+
+ ```python
+ def grade(transcript, workspace_path):
+ from pathlib import Path
+ f = Path(workspace_path) / "output.txt"
+ exists = f.exists()
+ has_hello = "hello" in f.read_text().lower() if exists else False
+ return {
+ "file_exists": 1.0 if exists else 0.0,
+ "has_hello": 1.0 if has_hello else 0.0,
+ }
+ ```
+ """))
+
+ return repo
+
+
+def test_dataset_loads_from_local_path(tmp_path):
+ """Dataset loads tasks from a local path."""
+ repo = _create_test_repo(tmp_path)
+ ds = PinchBenchDataset(path=str(repo))
+ ds.load()
+ assert ds.size() == 1
+ records = list(ds.iter_records())
+ assert records[0].record_id == "task_00_sanity"
+ assert "hello" in records[0].problem.lower() or "output.txt" in records[0].problem
+
+
+def test_task_env_creates_workspace(tmp_path):
+ """Task env creates and cleans up workspace."""
+ repo = _create_test_repo(tmp_path)
+ ds = PinchBenchDataset(path=str(repo))
+ ds.load()
+ record = list(ds.iter_records())[0]
+
+ env = ds.create_task_env(record)
+ with env:
+ ws = Path(record.metadata["workspace_path"])
+ assert ws.exists()
+ # After exit, workspace is cleaned up
+ assert not ws.exists()
+
+
+def test_full_grading_pipeline(tmp_path):
+ """Full pipeline: workspace setup -> simulate agent -> grade."""
+ repo = _create_test_repo(tmp_path)
+ ds = PinchBenchDataset(path=str(repo))
+ ds.load()
+ record = list(ds.iter_records())[0]
+
+ env = ds.create_task_env(record)
+ with env:
+ ws = Path(record.metadata["workspace_path"])
+ # Simulate agent writing the output file
+ (ws / "output.txt").write_text("hello world")
+
+ # Simulate EventRecorder with a tool call
+ recorder = EventRecorder()
+ recorder.record(
+ EventType.TOOL_CALL_START,
+ tool="file_write",
+ arguments={"path": "output.txt", "content": "hello world"},
+ )
+ recorder.record(
+ EventType.TOOL_CALL_END,
+ tool="file_write",
+ result="ok",
+ )
+ env.set_event_recorder(recorder)
+ env.run_tests()
+
+ assert record.metadata["is_resolved"] is True
+ assert record.metadata["pinchbench_score"] == 1.0
+ assert record.metadata["pinchbench_breakdown"]["file_exists"] == 1.0
+ assert record.metadata["pinchbench_breakdown"]["has_hello"] == 1.0
diff --git a/tests/evals/test_pinchbench_transcript.py b/tests/evals/test_pinchbench_transcript.py
index d9fefc88..d7850c6b 100644
--- a/tests/evals/test_pinchbench_transcript.py
+++ b/tests/evals/test_pinchbench_transcript.py
@@ -1,5 +1,6 @@
"""Tests for TurnTrace tool_calls field and PinchBench transcript translation."""
+from openjarvis.evals.core.event_recorder import AgentEvent, EventType
from openjarvis.evals.core.trace import TurnTrace
@@ -32,9 +33,6 @@ def test_turn_trace_tool_calls_from_dict_missing():
assert turn.tool_calls == []
-from openjarvis.evals.core.event_recorder import AgentEvent, EventType
-
-
def _make_event(etype, **metadata):
"""Helper to create a mock AgentEvent."""
return AgentEvent(event_type=etype, timestamp=0.0, metadata=metadata)
@@ -45,7 +43,9 @@ def test_events_to_transcript_tool_call_pair():
from openjarvis.evals.scorers.pinchbench import events_to_transcript
events = [
- _make_event(EventType.TOOL_CALL_START, tool="file_read", arguments={"path": "a.txt"}),
+ _make_event(
+ EventType.TOOL_CALL_START, tool="file_read", arguments={"path": "a.txt"}
+ ),
_make_event(EventType.TOOL_CALL_END, tool="file_read", result="file contents"),
]
transcript = events_to_transcript(events)
@@ -64,7 +64,11 @@ def test_events_to_transcript_tool_name_mapping():
from openjarvis.evals.scorers.pinchbench import events_to_transcript
events = [
- _make_event(EventType.TOOL_CALL_START, tool="image_generate", arguments={"prompt": "cat"}),
+ _make_event(
+ EventType.TOOL_CALL_START,
+ tool="image_generate",
+ arguments={"prompt": "cat"},
+ ),
_make_event(EventType.TOOL_CALL_END, tool="image_generate", result="ok"),
]
transcript = events_to_transcript(events)