Personal AI, On Personal Devices
Python
feat(evals): add PinchBench task environment — workspace setup and run_tests() grading
Jon Saad-Falcon6ae720baa9df
diff --git a/src/openjarvis/evals/execution/pinchbench_env.py b/src/openjarvis/evals/execution/pinchbench_env.pynew file mode 100644index 00000000..f492746e--- /dev/null+++ b/src/openjarvis/evals/execution/pinchbench_env.py@@ -0,0 +1,123 @@+"""PinchBench task environment — per-task workspace setup and grading."""++from __future__ import annotations++import logging+import os+import shutil+import tempfile+from pathlib import Path+from types import TracebackType+from typing import Any, Optional, Type++from openjarvis.evals.core.types import EvalRecord++LOGGER = logging.getLogger(__name__)+++class PinchBenchTaskEnv:+ """Per-task workspace environment for PinchBench.++ Context manager that creates an isolated workspace directory,+ populates fixture files from the task definition, and runs+ grading after agent execution via run_tests().+ """++ def __init__(+ self,+ record: EvalRecord,+ judge_backend: Any = None,+ judge_model: str = "anthropic/claude-opus-4-5",+ ) -> None:+ self._record = record+ self._judge_backend = judge_backend+ self._judge_model = judge_model+ self._workspace: Optional[Path] = None+ self._event_recorder: Any = None++ def set_event_recorder(self, recorder: Any) -> None:+ """Receive the EventRecorder from AgenticRunner for transcript building."""+ self._event_recorder = recorder++ def __enter__(self) -> PinchBenchTaskEnv:+ # Create isolated workspace+ self._workspace = Path(tempfile.mkdtemp(prefix="pinchbench_"))++ # Populate fixture files+ repo_dir = Path(self._record.metadata.get("pinchbench_repo_dir", ""))+ workspace_files = self._record.metadata.get("workspace_files", [])++ for file_spec in workspace_files:+ if "content" in file_spec:+ # Inline content+ dest = self._workspace / file_spec.get("path", file_spec.get("dest", "file.txt"))+ dest.parent.mkdir(parents=True, exist_ok=True)+ dest.write_text(file_spec["content"])+ elif "source" in file_spec:+ # Asset file from repo+ source = repo_dir / "assets" / file_spec["source"]+ dest_key = file_spec.get("dest", file_spec.get("path", file_spec["source"]))+ dest = self._workspace / dest_key+ dest.parent.mkdir(parents=True, exist_ok=True)+ if source.exists():+ shutil.copy2(str(source), str(dest))+ else:+ LOGGER.warning("Asset not found: %s", source)++ self._record.metadata["workspace_path"] = str(self._workspace)+ LOGGER.info("PinchBench workspace: %s (task %s)", self._workspace, self._record.record_id)+ return self++ def run_tests(self) -> None:+ """Grade the agent's work using PinchBench grading logic.++ Called by AgenticRunner after agent execution, before QueryTrace+ is constructed. Builds transcript from raw EventRecorder events.+ """+ from openjarvis.evals.scorers.pinchbench import (+ events_to_transcript,+ grade_pinchbench_task,+ )++ workspace_path = self._record.metadata.get("workspace_path", "")+ events = self._event_recorder.get_events() if self._event_recorder else []+ transcript = events_to_transcript(events)++ result = grade_pinchbench_task(+ record=self._record,+ transcript=transcript,+ workspace_path=workspace_path,+ judge_backend=self._judge_backend,+ judge_model=self._judge_model,+ )++ self._record.metadata["is_resolved"] = result["score"] >= 0.5+ self._record.metadata["reward"] = result["score"]+ self._record.metadata["pinchbench_score"] = result["score"]+ self._record.metadata["pinchbench_breakdown"] = result["breakdown"]+ self._record.metadata["pinchbench_notes"] = result.get("notes", "")++ LOGGER.info(+ "PinchBench grading for %s: score=%.2f resolved=%s",+ self._record.record_id,+ result["score"],+ result["score"] >= 0.5,+ )++ def __exit__(+ self,+ exc_type: Optional[Type[BaseException]],+ exc_val: Optional[BaseException],+ exc_tb: Optional[TracebackType],+ ) -> None:+ if self._workspace and self._workspace.exists():+ keep = os.environ.get("PINCHBENCH_KEEP_WORKSPACES", "").strip()+ if keep and keep != "0":+ LOGGER.info("Keeping workspace: %s", self._workspace)+ else:+ shutil.rmtree(self._workspace, ignore_errors=True)+ self._workspace = None+ self._event_recorder = None+++__all__ = ["PinchBenchTaskEnv"]