"CLI-Anything: Making ALL Software Agent-Native" -- CLI-Hub: https://clianything.cc/
Python
Add Audacity eval harness
sehawq1cdc4e22e93d
diff --git a/audacity/agent-harness/cli_anything/audacity/README.md b/audacity/agent-harness/cli_anything/audacity/README.mdindex 30cff5408..4d36bcd29 100644--- a/audacity/agent-harness/cli_anything/audacity/README.md+++ b/audacity/agent-harness/cli_anything/audacity/README.md@@ -53,6 +53,28 @@ python3 -m pytest cli/tests/test_core.py -vpython3 -m pytest cli/tests/test_full_e2e.py -v```+## Evaluation (Eval Harness)++Run a lightweight evaluation suite and generate reports:++```bash+# Default output: eval_results/<timestamp>/+python3 -m cli.audacity_cli eval++# Custom output directory+python3 -m cli.audacity_cli eval --out ./eval_out++# Compare to a baseline and fail on regression+python3 -m cli.audacity_cli eval --baseline baseline.json --fail-on-regression++# Update (write) baseline JSON+python3 -m cli.audacity_cli eval --baseline baseline.json --update-baseline+```++Outputs:+- `eval_report.json` and `eval_report.md`+- `artifacts/` directory for task outputs+## Command Groups| Group | Commands |diff --git a/audacity/agent-harness/cli_anything/audacity/audacity_cli.py b/audacity/agent-harness/cli_anything/audacity/audacity_cli.pyindex b4d5dc952..4e5aac2b3 100644--- a/audacity/agent-harness/cli_anything/audacity/audacity_cli.py+++ b/audacity/agent-harness/cli_anything/audacity/audacity_cli.py@@ -673,6 +673,51 @@ def session_history():output(history, "Undo history:")+# -- Eval ------------------------------------------------------------------[email protected]("eval")[email protected]("--out", "out_dir", type=str, default=None,+ help="Output directory for eval reports")[email protected]("--baseline", "baseline_path", type=str, default=None,+ help="Path to baseline JSON for regression comparison")[email protected]("--update-baseline", is_flag=True,+ help="Write baseline JSON from this run")[email protected]("--fail-on-regression", is_flag=True,+ help="Exit with code 2 if regression is detected")+@handle_error+def eval_cmd(out_dir, baseline_path, update_baseline, fail_on_regression):+ """Run evaluation tasks and generate reports."""+ from cli_anything.audacity.eval.runner import run_eval++ result = run_eval(+ output_dir=out_dir,+ baseline_path=baseline_path,+ update_baseline=update_baseline,+ )+ report = result.get("report", {})+ summary = report.get("summary", {})+ comparison = result.get("comparison")+ regression = bool(comparison.get("regression")) if comparison else False++ output_data = {+ "summary": summary,+ "output_dir": result.get("paths", {}).get("output_dir"),+ "report_json": result.get("paths", {}).get("report_json"),+ "report_md": result.get("paths", {}).get("report_md"),+ "baseline_written": result.get("paths", {}).get("baseline_written"),+ "regression": regression,+ }++ msg = (+ f"Eval completed: {summary.get('passed', 0)}/{summary.get('total', 0)} passed"+ )+ if regression:+ msg += " (regression detected)"+ output(output_data, msg)++ if regression and fail_on_regression:+ sys.exit(2)++# -- REPL ------------------------------------------------------------------@cli.command()@click.option("--project", "project_path", type=str, default=None)@@ -742,6 +787,7 @@ def _repl_help(skin=None):"media probe|check": "Media file operations","export presets|preset-info|render": "Export/render commands","session status|undo|redo|history": "Session management",+ "eval": "Run evaluation harness and generate reports","help": "Show this help","quit": "Exit REPL",}diff --git a/audacity/agent-harness/cli_anything/audacity/eval/__init__.py b/audacity/agent-harness/cli_anything/audacity/eval/__init__.pynew file mode 100644index 000000000..56e34798e--- /dev/null+++ b/audacity/agent-harness/cli_anything/audacity/eval/__init__.py@@ -0,0 +1 @@+"""Evaluation harness for Audacity CLI."""diff --git a/audacity/agent-harness/cli_anything/audacity/eval/runner.py b/audacity/agent-harness/cli_anything/audacity/eval/runner.pynew file mode 100644index 000000000..048b8543f--- /dev/null+++ b/audacity/agent-harness/cli_anything/audacity/eval/runner.py@@ -0,0 +1,299 @@+"""Evaluation and regression harness for Audacity CLI."""++from __future__ import annotations++import importlib+import json+import pkgutil+import tempfile+import time+from dataclasses import dataclass+from datetime import datetime+from pathlib import Path+from typing import Any, Callable, Dict, List, Optional++from cli_anything.audacity.utils.file_io import safe_write_json+++@dataclass+class TaskSpec:+ task_id: str+ name: str+ description: str+ run: Callable[["EvalContext"], Dict[str, Any]]+++@dataclass+class EvalContext:+ output_dir: Path+ artifacts_dir: Path+ work_dir: Path+ task_id: str = ""++ def task_work_dir(self) -> Path:+ path = self.work_dir / self.task_id+ path.mkdir(parents=True, exist_ok=True)+ return path++ def task_artifacts_dir(self) -> Path:+ path = self.artifacts_dir / self.task_id+ path.mkdir(parents=True, exist_ok=True)+ return path++ def task_artifact_path(self, filename: str) -> Path:+ return self.task_artifacts_dir() / filename+++def _iso_now() -> str:+ return datetime.now().isoformat()+++def default_output_dir() -> Path:+ stamp = datetime.now().strftime("%Y%m%d_%H%M%S")+ return Path("eval_results") / stamp+++def discover_tasks() -> List[TaskSpec]:+ tasks_pkg = importlib.import_module("cli_anything.audacity.eval.tasks")+ task_specs: List[TaskSpec] = []+ seen_ids = set()++ for mod in pkgutil.iter_modules(tasks_pkg.__path__):+ if mod.ispkg:+ continue+ module_name = f"{tasks_pkg.__name__}.{mod.name}"+ module = importlib.import_module(module_name)+ task_meta = getattr(module, "TASK", None)+ run_fn = getattr(module, "run", None)+ if not isinstance(task_meta, dict) or not callable(run_fn):+ continue++ task_id = str(task_meta.get("id") or mod.name)+ if task_id in seen_ids:+ raise ValueError(f"Duplicate task id: {task_id}")+ seen_ids.add(task_id)++ task_specs.append(TaskSpec(+ task_id=task_id,+ name=str(task_meta.get("name", task_id)),+ description=str(task_meta.get("description", "")),+ run=run_fn,+ ))++ task_specs.sort(key=lambda t: t.task_id)+ return task_specs+++def _run_task(task: TaskSpec, ctx: EvalContext) -> Dict[str, Any]:+ ctx.task_id = task.task_id+ ctx.task_work_dir()+ ctx.task_artifacts_dir()++ started = time.time()+ ok = False+ metrics: Dict[str, Any] = {}+ artifacts: List[str] = []+ notes = ""+ error = ""++ try:+ result = task.run(ctx) or {}+ ok = bool(result.get("ok", False))+ metrics = result.get("metrics", {}) or {}+ artifacts = result.get("artifacts", []) or []+ notes = result.get("notes", "") or ""+ except Exception as exc: # pylint: disable=broad-except+ error = f"{type(exc).__name__}: {exc}"++ duration_ms = int((time.time() - started) * 1000)+ status = "pass" if ok else "fail"++ return {+ "id": task.task_id,+ "name": task.name,+ "description": task.description,+ "status": status,+ "duration_ms": duration_ms,+ "metrics": metrics,+ "artifacts": artifacts,+ "notes": notes,+ "error": error,+ }+++def _build_summary(results: List[Dict[str, Any]]) -> Dict[str, Any]:+ total = len(results)+ passed = sum(1 for r in results if r.get("status") == "pass")+ failed = total - passed+ success_rate = float(passed) / float(total) if total else 0.0+ return {+ "total": total,+ "passed": passed,+ "failed": failed,+ "success_rate": round(success_rate, 4),+ }+++def _report_to_markdown(report: Dict[str, Any]) -> str:+ lines: List[str] = []+ lines.append("# Audacity Eval Report")+ lines.append("")+ lines.append(f"Run at: {report.get('started_at', '')}")+ lines.append("")+ summary = report.get("summary", {})+ lines.append(+ f"Summary: {summary.get('passed', 0)}/{summary.get('total', 0)} passed "+ f"({summary.get('success_rate', 0.0):.2%})"+ )+ lines.append("")+ lines.append("| Task | Status | Duration (ms) |")+ lines.append("| --- | --- | --- |")+ for task in report.get("tasks", []):+ status = str(task.get("status", "")).upper()+ lines.append(+ f"| {task.get('id', '')} | {status} | {task.get('duration_ms', 0)} |"+ )++ comparison = report.get("baseline_comparison")+ if comparison:+ lines.append("")+ lines.append("## Baseline Comparison")+ lines.append("")+ lines.append(f"Baseline: {comparison.get('baseline_path', '')}")+ lines.append(+ f"Success rate delta: {comparison.get('success_rate_delta', 0.0):.4f}"+ )+ if comparison.get("regressions"):+ lines.append("")+ lines.append("Regressions:")+ for reg in comparison.get("regressions", []):+ lines.append(+ f"- {reg.get('task_id', '')}: {reg.get('reason', '')}"+ )+ else:+ lines.append("")+ lines.append("Regressions: none")++ lines.append("")+ return "\n".join(lines)+++def _report_to_baseline(report: Dict[str, Any]) -> Dict[str, Any]:+ task_map = {}+ for task in report.get("tasks", []):+ task_map[task.get("id", "")] = {+ "status": task.get("status"),+ "metrics": task.get("metrics", {}),+ }+ return {+ "summary": report.get("summary", {}),+ "tasks": task_map,+ }+++def load_baseline(path: str) -> Dict[str, Any]:+ baseline_path = Path(path)+ if not baseline_path.exists():+ raise FileNotFoundError(f"Baseline file not found: {path}")+ with baseline_path.open("r", encoding="utf-8") as f:+ return json.load(f)+++def compare_baseline(baseline: Dict[str, Any], report: Dict[str, Any]) -> Dict[str, Any]:+ baseline_tasks = baseline.get("tasks", {}) or {}+ report_tasks = {t.get("id", ""): t for t in report.get("tasks", [])}++ regressions: List[Dict[str, Any]] = []++ for task_id, baseline_task in baseline_tasks.items():+ if baseline_task.get("status") != "pass":+ continue+ current = report_tasks.get(task_id)+ if current and current.get("status") == "fail":+ regressions.append({+ "task_id": task_id,+ "reason": "pass_to_fail",+ })++ baseline_rate = float(baseline.get("summary", {}).get("success_rate", 0.0))+ current_rate = float(report.get("summary", {}).get("success_rate", 0.0))+ rate_delta = round(current_rate - baseline_rate, 4)+ if rate_delta < 0:+ regressions.append({+ "task_id": "__summary__",+ "reason": "success_rate_decrease",+ "delta": rate_delta,+ })++ return {+ "success_rate_delta": rate_delta,+ "regressions": regressions,+ "regression": len(regressions) > 0,+ }+++def run_eval(+ output_dir: Optional[str] = None,+ baseline_path: Optional[str] = None,+ update_baseline: bool = False,+) -> Dict[str, Any]:+ out_dir = Path(output_dir) if output_dir else default_output_dir()+ out_dir.mkdir(parents=True, exist_ok=True)+ artifacts_dir = out_dir / "artifacts"+ artifacts_dir.mkdir(parents=True, exist_ok=True)++ tasks = discover_tasks()+ if not tasks:+ raise RuntimeError("No eval tasks discovered.")++ started_at = _iso_now()+ results: List[Dict[str, Any]] = []++ with tempfile.TemporaryDirectory() as tmp_dir:+ ctx = EvalContext(+ output_dir=out_dir,+ artifacts_dir=artifacts_dir,+ work_dir=Path(tmp_dir),+ )+ for task in tasks:+ results.append(_run_task(task, ctx))++ summary = _build_summary(results)+ report = {+ "schema_version": 1,+ "started_at": started_at,+ "summary": summary,+ "tasks": results,+ }++ comparison = None+ if baseline_path:+ baseline = load_baseline(baseline_path)+ comparison = compare_baseline(baseline, report)+ comparison["baseline_path"] = str(baseline_path)+ report["baseline_comparison"] = comparison++ report_json_path = out_dir / "eval_report.json"+ report_md_path = out_dir / "eval_report.md"++ safe_write_json(report_json_path, report, indent=2, default=str)+ report_md_path.write_text(_report_to_markdown(report), encoding="utf-8")++ baseline_written = None+ if update_baseline:+ baseline_out = Path(baseline_path) if baseline_path else (out_dir / "baseline.json")+ baseline_out.parent.mkdir(parents=True, exist_ok=True)+ baseline_data = _report_to_baseline(report)+ safe_write_json(baseline_out, baseline_data, indent=2, default=str)+ baseline_written = str(baseline_out)++ return {+ "report": report,+ "comparison": comparison,+ "paths": {+ "output_dir": str(out_dir),+ "report_json": str(report_json_path),+ "report_md": str(report_md_path),+ "baseline_written": baseline_written,+ },+ }diff --git a/audacity/agent-harness/cli_anything/audacity/eval/tasks/__init__.py b/audacity/agent-harness/cli_anything/audacity/eval/tasks/__init__.pynew file mode 100644index 000000000..60114feb8--- /dev/null+++ b/audacity/agent-harness/cli_anything/audacity/eval/tasks/__init__.py@@ -0,0 +1 @@+"""Eval tasks for Audacity CLI."""diff --git a/audacity/agent-harness/cli_anything/audacity/eval/tasks/effects_registry.py b/audacity/agent-harness/cli_anything/audacity/eval/tasks/effects_registry.pynew file mode 100644index 000000000..da115e60b--- /dev/null+++ b/audacity/agent-harness/cli_anything/audacity/eval/tasks/effects_registry.py@@ -0,0 +1,45 @@+"""Eval task: effects registry."""++from cli_anything.audacity.core.project import create_project+from cli_anything.audacity.core.tracks import add_track+from cli_anything.audacity.core.effects import EFFECT_REGISTRY, add_effect, list_effects+++TASK = {+ "id": "effects_registry",+ "name": "Effects registry",+ "description": "Validate core effects and apply to track",+}+++def run(ctx):+ _ = ctx.task_work_dir()++ project = create_project(+ name="Eval Effects",+ sample_rate=44100,+ bit_depth=16,+ channels=2,+ )+ add_track(project, name="Track 1")++ has_normalize = "normalize" in EFFECT_REGISTRY+ has_fade_in = "fade_in" in EFFECT_REGISTRY++ add_effect(project, "normalize", track_index=0, params={"target_db": -3.0})+ add_effect(project, "fade_in", track_index=0, params={"duration": 0.2})++ effects = list_effects(project, track_index=0)+ ok = has_normalize and has_fade_in and len(effects) == 2++ metrics = {+ "effects_count": len(effects),+ "has_normalize": has_normalize,+ "has_fade_in": has_fade_in,+ }++ return {+ "ok": ok,+ "metrics": metrics,+ "artifacts": [],+ }diff --git a/audacity/agent-harness/cli_anything/audacity/eval/tasks/export_wav.py b/audacity/agent-harness/cli_anything/audacity/eval/tasks/export_wav.pynew file mode 100644index 000000000..b8f5b2987--- /dev/null+++ b/audacity/agent-harness/cli_anything/audacity/eval/tasks/export_wav.py@@ -0,0 +1,56 @@+"""Eval task: WAV export."""++import os++from cli_anything.audacity.core.project import create_project+from cli_anything.audacity.core.tracks import add_track+from cli_anything.audacity.core.clips import add_clip+from cli_anything.audacity.core.export import render_mix+from cli_anything.audacity.utils.audio_utils import generate_sine_wave, write_wav+++TASK = {+ "id": "export_wav",+ "name": "WAV export",+ "description": "Render a simple project to WAV",+}+++def run(ctx):+ work_dir = ctx.task_work_dir()+ artifacts_dir = ctx.task_artifacts_dir()++ source_wav = work_dir / "source.wav"+ samples = generate_sine_wave(+ frequency=440.0,+ duration=0.5,+ sample_rate=8000,+ amplitude=0.5,+ channels=1,+ )+ write_wav(str(source_wav), samples, sample_rate=8000, channels=1, bit_depth=16)++ project = create_project(+ name="Eval Export",+ sample_rate=8000,+ bit_depth=16,+ channels=1,+ )+ add_track(project, name="Track 1")+ add_clip(project, track_index=0, source=str(source_wav))++ output_path = artifacts_dir / "mix.wav"+ result = render_mix(project, str(output_path), preset="wav", overwrite=True)++ ok = os.path.exists(output_path) and result.get("file_size", 0) > 0+ metrics = {+ "file_size": result.get("file_size"),+ "duration": result.get("duration"),+ "tracks_rendered": result.get("tracks_rendered"),+ }++ return {+ "ok": ok,+ "metrics": metrics,+ "artifacts": [str(output_path)],+ }diff --git a/audacity/agent-harness/cli_anything/audacity/eval/tasks/project_roundtrip.py b/audacity/agent-harness/cli_anything/audacity/eval/tasks/project_roundtrip.pynew file mode 100644index 000000000..875abc495--- /dev/null+++ b/audacity/agent-harness/cli_anything/audacity/eval/tasks/project_roundtrip.py@@ -0,0 +1,48 @@+"""Eval task: project roundtrip."""++from cli_anything.audacity.core.project import (+ create_project,+ get_project_info,+ open_project,+ save_project,+)+++TASK = {+ "id": "project_roundtrip",+ "name": "Project roundtrip",+ "description": "Create -> save -> open -> info validation",+}+++def run(ctx):+ work_dir = ctx.task_work_dir()+ project_path = work_dir / "project.json"++ project = create_project(+ name="Eval Project",+ sample_rate=44100,+ bit_depth=16,+ channels=2,+ )+ save_project(project, str(project_path))+ loaded = open_project(str(project_path))+ info = get_project_info(loaded)++ ok = (+ info.get("name") == "Eval Project"+ and info.get("settings", {}).get("sample_rate") == 44100+ and info.get("track_count") == 0+ )++ metrics = {+ "track_count": info.get("track_count"),+ "clip_count": info.get("clip_count"),+ "duration": info.get("duration"),+ }++ return {+ "ok": ok,+ "metrics": metrics,+ "artifacts": [],+ }diff --git a/audacity/agent-harness/cli_anything/audacity/eval/tasks/track_clip_flow.py b/audacity/agent-harness/cli_anything/audacity/eval/tasks/track_clip_flow.pynew file mode 100644index 000000000..be883e580--- /dev/null+++ b/audacity/agent-harness/cli_anything/audacity/eval/tasks/track_clip_flow.py@@ -0,0 +1,54 @@+"""Eval task: track + clip flow."""++from cli_anything.audacity.core.project import create_project+from cli_anything.audacity.core.tracks import add_track+from cli_anything.audacity.core.clips import add_clip, list_clips, split_clip+from cli_anything.audacity.utils.audio_utils import generate_sine_wave, write_wav+++TASK = {+ "id": "track_clip_flow",+ "name": "Track and clip flow",+ "description": "Add track, add clip, split clip",+}+++def run(ctx):+ work_dir = ctx.task_work_dir()+ wav_path = work_dir / "tone.wav"++ samples = generate_sine_wave(+ frequency=440.0,+ duration=0.5,+ sample_rate=8000,+ amplitude=0.5,+ channels=1,+ )+ write_wav(str(wav_path), samples, sample_rate=8000, channels=1, bit_depth=16)++ project = create_project(+ name="Eval Track Clip",+ sample_rate=8000,+ bit_depth=16,+ channels=1,+ )+ add_track(project, name="Track 1")+ clip = add_clip(project, track_index=0, source=str(wav_path))++ before = list_clips(project, 0)+ split_time = clip["start_time"] + (clip["end_time"] - clip["start_time"]) / 2.0+ split_clip(project, track_index=0, clip_index=0, split_time=split_time)+ after = list_clips(project, 0)++ ok = len(before) == 1 and len(after) == 2+ metrics = {+ "clips_before": len(before),+ "clips_after": len(after),+ "duration": after[0].get("duration", 0.0) if after else 0.0,+ }++ return {+ "ok": ok,+ "metrics": metrics,+ "artifacts": [],+ }diff --git a/audacity/agent-harness/cli_anything/audacity/tests/test_eval.py b/audacity/agent-harness/cli_anything/audacity/tests/test_eval.pynew file mode 100644index 000000000..a59303ead--- /dev/null+++ b/audacity/agent-harness/cli_anything/audacity/tests/test_eval.py@@ -0,0 +1,51 @@+"""Tests for Audacity eval harness."""++import json++from cli_anything.audacity.eval.runner import compare_baseline, discover_tasks, run_eval+++def test_discover_tasks():+ tasks = discover_tasks()+ assert len(tasks) >= 3+ task_ids = [t.task_id for t in tasks]+ assert len(task_ids) == len(set(task_ids))+++def test_run_eval_creates_reports(tmp_path):+ out_dir = tmp_path / "eval_out"+ result = run_eval(output_dir=str(out_dir))+ paths = result.get("paths", {})++ report_json = paths.get("report_json")+ report_md = paths.get("report_md")++ assert report_json and report_md+ assert (out_dir / "eval_report.json").exists()+ assert (out_dir / "eval_report.md").exists()++ data = json.loads((out_dir / "eval_report.json").read_text(encoding="utf-8"))+ summary = data.get("summary", {})+ assert summary.get("total") == len(data.get("tasks", []))+ assert summary.get("passed") + summary.get("failed") == summary.get("total")+++def test_compare_baseline_detects_regression():+ baseline = {+ "summary": {"success_rate": 1.0},+ "tasks": {+ "task_a": {"status": "pass"},+ "task_b": {"status": "pass"},+ },+ }+ report = {+ "summary": {"success_rate": 0.5},+ "tasks": [+ {"id": "task_a", "status": "fail", "duration_ms": 1},+ {"id": "task_b", "status": "pass", "duration_ms": 1},+ ],+ }++ comparison = compare_baseline(baseline, report)+ assert comparison.get("regression") is True+ assert any(r.get("task_id") == "task_a" for r in comparison.get("regressions", []))diff --git a/audacity/agent-harness/cli_anything/audacity/utils/file_io.py b/audacity/agent-harness/cli_anything/audacity/utils/file_io.pynew file mode 100644index 000000000..38af054e1--- /dev/null+++ b/audacity/agent-harness/cli_anything/audacity/utils/file_io.py@@ -0,0 +1,50 @@+"""Helpers for safe JSON writes."""++from __future__ import annotations++import json+from pathlib import Path+from typing import Any, Callable, Optional, Union+++def safe_write_json(+ path: Union[str, Path],+ data: Any,+ *,+ indent: int = 2,+ default: Optional[Callable] = None,+) -> None:+ """Write JSON to disk with best-effort file locking on Unix.++ On platforms without fcntl (e.g., Windows), it falls back to a normal write.+ """+ path_str = str(path)+ try:+ f = open(path_str, "r+")+ except FileNotFoundError:+ f = open(path_str, "w")++ with f:+ locked = False+ fcntl_mod = None+ try:+ import fcntl as _fcntl+ _fcntl.flock(f.fileno(), _fcntl.LOCK_EX)+ locked = True+ fcntl_mod = _fcntl+ except (ImportError, OSError):+ pass++ try:+ f.seek(0)+ f.truncate()+ if default is None:+ json.dump(data, f, indent=indent)+ else:+ json.dump(data, f, indent=indent, default=default)+ finally:+ if locked and fcntl_mod is not None:+ try:+ fcntl_mod.flock(f.fileno(), fcntl_mod.LOCK_UN)+ except OSError:+ pass