Personal AI, On Personal Devices
Python
feat(evals): add PinchBench dataset provider — repo clone and task markdown parsing
Jon Saad-Falcon8e8d4a68e4e6
diff --git a/src/openjarvis/evals/datasets/pinchbench.py b/src/openjarvis/evals/datasets/pinchbench.pynew file mode 100644index 00000000..fdcb8fd0--- /dev/null+++ b/src/openjarvis/evals/datasets/pinchbench.py@@ -0,0 +1,202 @@+"""PinchBench dataset provider — real-world agent task benchmark.++Clones the pinchbench/skill repo at runtime and parses task markdown files+into EvalRecords for use with AgenticRunner.++Reference: https://github.com/pinchbench/skill+"""++from __future__ import annotations++import logging+import random+import re+import shutil+import subprocess+from pathlib import Path+from typing import Any, Dict, Iterable, List, Optional++import yaml++from openjarvis.evals.core.dataset import DatasetProvider+from openjarvis.evals.core.types import EvalRecord++LOGGER = logging.getLogger(__name__)++PINCHBENCH_REPO = "https://github.com/pinchbench/skill.git"+CACHE_DIR = Path.home() / ".cache" / "pinchbench"+++def _parse_task_markdown(content: str, filename: str = "") -> Dict[str, Any]:+ """Parse a PinchBench task markdown file into a dict.++ Extracts YAML frontmatter and markdown sections (## Prompt,+ ## Expected Behavior, ## Automated Checks, ## LLM Judge Rubric).+ """+ # Split frontmatter+ parts = content.split("---", 2)+ if len(parts) < 3:+ raise ValueError(f"Missing YAML frontmatter in {filename}")++ frontmatter = yaml.safe_load(parts[1])+ body = parts[2]++ # Parse sections by ## headers+ sections: Dict[str, str] = {}+ current_header: Optional[str] = None+ current_lines: List[str] = []++ for line in body.split("\n"):+ header_match = re.match(r"^##\s+(.+)$", line)+ if header_match:+ if current_header is not None:+ sections[current_header] = "\n".join(current_lines).strip()+ current_header = header_match.group(1).strip()+ current_lines = []+ else:+ current_lines.append(line)++ if current_header is not None:+ sections[current_header] = "\n".join(current_lines).strip()++ # Extract Python code block from Automated Checks section+ automated_checks = None+ checks_section = sections.get("Automated Checks", "")+ code_match = re.search(r"```python\s*\n(.*?)```", checks_section, re.DOTALL)+ if code_match:+ automated_checks = code_match.group(1).strip()++ return {+ "id": frontmatter.get("id", ""),+ "name": frontmatter.get("name", ""),+ "category": frontmatter.get("category", ""),+ "grading_type": frontmatter.get("grading_type", "automated"),+ "timeout_seconds": frontmatter.get("timeout_seconds", 180),+ "workspace_files": frontmatter.get("workspace_files", []),+ "grading_weights": frontmatter.get("grading_weights"),+ "prompt": sections.get("Prompt", ""),+ "expected_behavior": sections.get("Expected Behavior", ""),+ "grading_criteria": sections.get("Grading Criteria", ""),+ "automated_checks": automated_checks,+ "llm_judge_rubric": sections.get("LLM Judge Rubric"),+ }+++class PinchBenchDataset(DatasetProvider):+ """PinchBench real-world agent benchmark.++ Clones pinchbench/skill from GitHub (or uses a local path) and+ parses task markdown files into EvalRecords.+ """++ dataset_id = "pinchbench"+ dataset_name = "PinchBench"++ def __init__(self, path: Optional[str] = None) -> None:+ self._local_path = Path(path) if path else None+ self._repo_dir: Path = self._local_path or CACHE_DIR+ self._records: List[EvalRecord] = []++ def verify_requirements(self) -> List[str]:+ issues: List[str] = []+ if self._local_path is None and shutil.which("git") is None:+ issues.append("git binary not found. Install git to clone PinchBench tasks.")+ if self._repo_dir.exists() and not (self._repo_dir / "tasks").is_dir():+ issues.append(+ f"PinchBench cache at {self._repo_dir} is corrupted (missing tasks/). "+ "Delete and re-run to re-clone."+ )+ return issues++ def _ensure_repo(self) -> Path:+ """Clone the repo if not already cached. Returns repo dir."""+ if self._local_path is not None:+ if not self._local_path.exists():+ raise FileNotFoundError(f"PinchBench path not found: {self._local_path}")+ return self._local_path++ if not self._repo_dir.exists():+ LOGGER.info("Cloning PinchBench from %s ...", PINCHBENCH_REPO)+ self._repo_dir.parent.mkdir(parents=True, exist_ok=True)+ subprocess.run(+ ["git", "clone", "--depth", "1", PINCHBENCH_REPO, str(self._repo_dir)],+ check=True,+ capture_output=True,+ )+ LOGGER.info("PinchBench cloned to %s", self._repo_dir)++ return self._repo_dir++ def load(+ self,+ *,+ max_samples: Optional[int] = None,+ split: Optional[str] = None,+ seed: Optional[int] = None,+ ) -> None:+ repo_dir = self._ensure_repo()+ tasks_dir = repo_dir / "tasks"++ if not tasks_dir.is_dir():+ raise FileNotFoundError(f"No tasks/ directory in {repo_dir}")++ task_files = sorted(tasks_dir.glob("task_*.md"))+ if not task_files:+ raise FileNotFoundError(f"No task_*.md files in {tasks_dir}")++ tasks = []+ for tf in task_files:+ try:+ parsed = _parse_task_markdown(tf.read_text(), filename=tf.name)+ tasks.append(parsed)+ except Exception as exc:+ LOGGER.warning("Skipping %s: %s", tf.name, exc)++ if seed is not None:+ random.Random(seed).shuffle(tasks)+ if max_samples is not None:+ tasks = tasks[:max_samples]++ self._records = [+ EvalRecord(+ record_id=t["id"],+ problem=t["prompt"],+ reference=t["expected_behavior"],+ category=t["category"],+ subject=t["name"],+ metadata={+ "grading_type": t["grading_type"],+ "grading_weights": t["grading_weights"],+ "automated_checks": t["automated_checks"],+ "llm_judge_rubric": t["llm_judge_rubric"],+ "timeout_seconds": t["timeout_seconds"],+ "workspace_files": t["workspace_files"],+ "pinchbench_repo_dir": str(repo_dir),+ },+ )+ for t in tasks+ ]++ LOGGER.info("PinchBench: loaded %d tasks", len(self._records))++ def iter_records(self) -> Iterable[EvalRecord]:+ return iter(self._records)++ def size(self) -> int:+ return len(self._records)++ def set_judge(self, judge_backend: Any, judge_model: str) -> None:+ """Set the judge backend/model for LLM-judge and hybrid grading."""+ self._judge_backend = judge_backend+ self._judge_model = judge_model++ def create_task_env(self, record: EvalRecord):+ from openjarvis.evals.execution.pinchbench_env import PinchBenchTaskEnv+ return PinchBenchTaskEnv(+ record,+ judge_backend=getattr(self, "_judge_backend", None),+ judge_model=getattr(self, "_judge_model", "anthropic/claude-opus-4-5"),+ )+++__all__ = ["PinchBenchDataset"]diff --git a/tests/evals/test_pinchbench_dataset.py b/tests/evals/test_pinchbench_dataset.pynew file mode 100644index 00000000..4abcd9a7--- /dev/null+++ b/tests/evals/test_pinchbench_dataset.py@@ -0,0 +1,130 @@+"""Tests for PinchBench dataset provider."""++import textwrap++from openjarvis.evals.datasets.pinchbench import _parse_task_markdown+++def test_parse_task_markdown_basic():+ """Parse a minimal task markdown file."""+ md = textwrap.dedent("""\+ ---+ id: task_00_test+ name: Test Task+ category: basic+ grading_type: automated+ timeout_seconds: 60+ workspace_files: []+ ---++ ## Prompt++ Do the thing.++ ## Expected Behavior++ The thing should be done.++ ## Grading Criteria++ - [ ] Thing was done++ ## Automated Checks++ ```python+ def grade(transcript, workspace_path):+ return {"done": 1.0}+ ```+ """)+ task = _parse_task_markdown(md, filename="task_00_test.md")+ assert task["id"] == "task_00_test"+ assert task["name"] == "Test Task"+ assert task["category"] == "basic"+ assert task["grading_type"] == "automated"+ assert "Do the thing." in task["prompt"]+ assert "The thing should be done." in task["expected_behavior"]+ assert "def grade" in task["automated_checks"]+++def test_parse_task_markdown_hybrid():+ """Parse a hybrid-graded task with weights."""+ md = textwrap.dedent("""\+ ---+ id: task_16_triage+ name: Email Triage+ category: email+ grading_type: hybrid+ timeout_seconds: 300+ workspace_files:+ - source: emails/email_01.txt+ dest: inbox/email_01.txt+ grading_weights:+ automated: 0.4+ llm_judge: 0.6+ ---++ ## Prompt++ Triage the emails.++ ## Expected Behavior++ Create a report.++ ## Grading Criteria++ - [ ] Report created++ ## Automated Checks++ ```python+ def grade(transcript, workspace_path):+ return {"report": 1.0}+ ```++ ## LLM Judge Rubric++ ### Criterion 1: Quality (Weight: 100%)++ **Score 1.0**: Excellent+ """)+ task = _parse_task_markdown(md, filename="task_16_triage.md")+ assert task["grading_type"] == "hybrid"+ assert task["grading_weights"] == {"automated": 0.4, "llm_judge": 0.6}+ assert len(task["workspace_files"]) == 1+ assert "Quality" in task["llm_judge_rubric"]+++def test_parse_task_markdown_no_automated_checks():+ """LLM-judge-only tasks have no automated checks."""+ md = textwrap.dedent("""\+ ---+ id: task_03_blog+ name: Blog Post+ category: writing+ grading_type: llm_judge+ timeout_seconds: 180+ workspace_files: []+ ---++ ## Prompt++ Write a blog post.++ ## Expected Behavior++ A good blog post.++ ## Grading Criteria++ - [ ] Well written++ ## LLM Judge Rubric++ ### Writing (Weight: 100%)++ **Score 1.0**: Excellent+ """)+ task = _parse_task_markdown(md, filename="task_03_blog.md")+ assert task["automated_checks"] is None+ assert "Writing" in task["llm_judge_rubric"]