Skip to content

Personal AI, On Personal Devices

Python

feat(evals): add PinchBench task environment — workspace setup and run_tests() grading

Jon Saad-Falcon6ae720baa9df
diff --git a/src/openjarvis/evals/execution/pinchbench_env.py b/src/openjarvis/evals/execution/pinchbench_env.py
new file mode 100644
index 00000000..f492746e
--- /dev/null
+++ b/src/openjarvis/evals/execution/pinchbench_env.py
@@ -0,0 +1,123 @@
+"""PinchBench task environment — per-task workspace setup and grading."""
+
+from __future__ import annotations
+
+import logging
+import os
+import shutil
+import tempfile
+from pathlib import Path
+from types import TracebackType
+from typing import Any, Optional, Type
+
+from openjarvis.evals.core.types import EvalRecord
+
+LOGGER = logging.getLogger(__name__)
+
+
+class PinchBenchTaskEnv:
+ """Per-task workspace environment for PinchBench.
+
+ Context manager that creates an isolated workspace directory,
+ populates fixture files from the task definition, and runs
+ grading after agent execution via run_tests().
+ """
+
+ def __init__(
+ self,
+ record: EvalRecord,
+ judge_backend: Any = None,
+ judge_model: str = "anthropic/claude-opus-4-5",
+ ) -> None:
+ self._record = record
+ self._judge_backend = judge_backend
+ self._judge_model = judge_model
+ self._workspace: Optional[Path] = None
+ self._event_recorder: Any = None
+
+ def set_event_recorder(self, recorder: Any) -> None:
+ """Receive the EventRecorder from AgenticRunner for transcript building."""
+ self._event_recorder = recorder
+
+ def __enter__(self) -> PinchBenchTaskEnv:
+ # Create isolated workspace
+ self._workspace = Path(tempfile.mkdtemp(prefix="pinchbench_"))
+
+ # Populate fixture files
+ repo_dir = Path(self._record.metadata.get("pinchbench_repo_dir", ""))
+ workspace_files = self._record.metadata.get("workspace_files", [])
+
+ for file_spec in workspace_files:
+ if "content" in file_spec:
+ # Inline content
+ dest = self._workspace / file_spec.get("path", file_spec.get("dest", "file.txt"))
+ dest.parent.mkdir(parents=True, exist_ok=True)
+ dest.write_text(file_spec["content"])
+ elif "source" in file_spec:
+ # Asset file from repo
+ source = repo_dir / "assets" / file_spec["source"]
+ dest_key = file_spec.get("dest", file_spec.get("path", file_spec["source"]))
+ dest = self._workspace / dest_key
+ dest.parent.mkdir(parents=True, exist_ok=True)
+ if source.exists():
+ shutil.copy2(str(source), str(dest))
+ else:
+ LOGGER.warning("Asset not found: %s", source)
+
+ self._record.metadata["workspace_path"] = str(self._workspace)
+ LOGGER.info("PinchBench workspace: %s (task %s)", self._workspace, self._record.record_id)
+ return self
+
+ def run_tests(self) -> None:
+ """Grade the agent's work using PinchBench grading logic.
+
+ Called by AgenticRunner after agent execution, before QueryTrace
+ is constructed. Builds transcript from raw EventRecorder events.
+ """
+ from openjarvis.evals.scorers.pinchbench import (
+ events_to_transcript,
+ grade_pinchbench_task,
+ )
+
+ workspace_path = self._record.metadata.get("workspace_path", "")
+ events = self._event_recorder.get_events() if self._event_recorder else []
+ transcript = events_to_transcript(events)
+
+ result = grade_pinchbench_task(
+ record=self._record,
+ transcript=transcript,
+ workspace_path=workspace_path,
+ judge_backend=self._judge_backend,
+ judge_model=self._judge_model,
+ )
+
+ self._record.metadata["is_resolved"] = result["score"] >= 0.5
+ self._record.metadata["reward"] = result["score"]
+ self._record.metadata["pinchbench_score"] = result["score"]
+ self._record.metadata["pinchbench_breakdown"] = result["breakdown"]
+ self._record.metadata["pinchbench_notes"] = result.get("notes", "")
+
+ LOGGER.info(
+ "PinchBench grading for %s: score=%.2f resolved=%s",
+ self._record.record_id,
+ result["score"],
+ result["score"] >= 0.5,
+ )
+
+ def __exit__(
+ self,
+ exc_type: Optional[Type[BaseException]],
+ exc_val: Optional[BaseException],
+ exc_tb: Optional[TracebackType],
+ ) -> None:
+ if self._workspace and self._workspace.exists():
+ keep = os.environ.get("PINCHBENCH_KEEP_WORKSPACES", "").strip()
+ if keep and keep != "0":
+ LOGGER.info("Keeping workspace: %s", self._workspace)
+ else:
+ shutil.rmtree(self._workspace, ignore_errors=True)
+ self._workspace = None
+ self._event_recorder = None
+
+
+__all__ = ["PinchBenchTaskEnv"]