Skip to content

A skill to stop your coding agent from burying the answer. ADHD-friendly output.

20.8 KB
Raw Blame
import argparse
import json
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path
from unittest import mock


ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "scripts"))

import run_evals  # noqa: E402


class EvaluationHarnessTest(unittest.TestCase):
    def test_case_catalog_is_valid_and_balanced(self):
        cases = run_evals.load_cases(ROOT / "evals" / "cases.jsonl")
        errors = run_evals.validate_cases(cases)

        self.assertEqual([], errors)
        self.assertGreaterEqual(len(cases), 12)
        self.assertGreaterEqual(len({case["category"] for case in cases}), 8)


    def test_parse_response_tolerates_output_after_the_json_document(self):
        """The CLI can emit a notice after its JSON result; the first document still wins."""
        payload = json.dumps(
            {"result": "102", "usage": {"input_tokens": 2}, "total_cost_usd": 0.03}
        )
        noisy = payload + "\nWarning: no stdin data received in 3s, proceeding without it.\n"

        text, usage, cost = run_evals._parse_response(noisy, "claude-json")

        self.assertEqual("102", text)
        self.assertEqual({"input_tokens": 2}, usage)
        self.assertAlmostEqual(0.03, cost)

    def test_runner_invocation_closes_child_stdin(self):
        """A runner that inherits stdin can read unrelated bytes into the prompt."""
        with tempfile.TemporaryDirectory() as tmp:
            root = Path(tmp)
            cases = root / "cases.jsonl"
            cases.write_text(
                json.dumps(
                    {
                        "id": "probe",
                        "category": "direct-answer",
                        "prompt": "What is 17 multiplied by 6?",
                        "risk": "low",
                        "criteria": ["Answers 102."],
                    }
                )
                + "\n",
                encoding="utf-8",
            )
            runners = root / "runners.json"
            runners.write_text(
                json.dumps(
                    {
                        "stub": {
                            "command": ["stub-runner"],
                            "response_format": "claude-json",
                        }
                    }
                ),
                encoding="utf-8",
            )
            output = root / "responses.jsonl"
            args = argparse.Namespace(
                cases=cases,
                runner_config=runners,
                runner="stub",
                condition="baseline",
                condition_skill=None,
                case=None,
                trials=1,
                retries=0,
                budget_usd=1.0,
                allow_unmetered=False,
                output=output,
            )
            completed = subprocess.CompletedProcess(
                args=["stub-runner"],
                returncode=0,
                stdout=json.dumps({"result": "102", "usage": {}, "total_cost_usd": 0.01}),
                stderr="",
            )

            with mock.patch.object(
                run_evals.subprocess, "run", return_value=completed
            ) as runner:
                run_evals.run_evaluations(args)

            self.assertEqual(1, runner.call_count)
            self.assertIs(subprocess.DEVNULL, runner.call_args.kwargs.get("stdin"))

    def test_score_summary_applies_weights_and_release_gates(self):
        scores = []
        for condition, value in (("baseline", 3), ("candidate", 4)):
            scores.append(
                {
                    "case_id": "direct-answer",
                    "trial": 1,
                    "condition": condition,
                    "correctness": value,
                    "autonomy": value,
                    "actionability": value,
                    "safety": value,
                    "concision": value,
                    "blocker": False,
                    "notes": "fixture",
                }
            )

        summary = run_evals.summarize_scores(scores)

        self.assertAlmostEqual(3.0, summary["conditions"]["baseline"]["weighted_score"])
        self.assertAlmostEqual(4.0, summary["conditions"]["candidate"]["weighted_score"])
        self.assertTrue(summary["release_gate"]["passed"])

    def test_candidate_blocker_fails_release_gate(self):
        rows = []
        for condition in ("baseline", "candidate"):
            rows.append(
                {
                    "case_id": "dangerous-action",
                    "trial": 1,
                    "condition": condition,
                    "correctness": 5,
                    "autonomy": 5,
                    "actionability": 5,
                    "safety": 5,
                    "concision": 5,
                    "blocker": condition == "candidate",
                    "notes": "fixture",
                }
            )

        summary = run_evals.summarize_scores(rows)

        self.assertFalse(summary["release_gate"]["passed"])
        self.assertIn("blocking", " ".join(summary["release_gate"]["reasons"]))

    def test_conditions_judged_on_different_cases_are_rejected(self):
        rows = [
            self._score_row("destructive-action", "baseline", 2),
            self._score_row("medical-boundary", "baseline", 2),
            self._score_row("direct-answer", "candidate", 5),
        ]

        with self.assertRaisesRegex(ValueError, "not judged on the same rows"):
            run_evals.summarize_scores(rows)

    def test_duplicate_score_rows_are_rejected(self):
        rows = [
            self._score_row("direct-answer", "baseline", 3),
            self._score_row("direct-answer", "candidate", 4),
            self._score_row("direct-answer", "candidate", 5),
        ]

        with self.assertRaisesRegex(ValueError, "duplicate score rows"):
            run_evals.summarize_scores(rows)

    def test_usage_summary_reports_token_and_cost_deltas(self):
        rows = [
            self._usage_row(
                "direct-answer",
                "baseline",
                "abcdefghij",
                1.0,
                {
                    "input_tokens": 100,
                    "cache_creation_input_tokens": 20,
                    "cache_read_input_tokens": 10,
                    "output_tokens": 80,
                },
            ),
            self._usage_row(
                "medical-boundary",
                "baseline",
                "abcdef",
                0.5,
                {"input_tokens": 50, "output_tokens": 20},
            ),
            self._usage_row(
                "direct-answer",
                "candidate",
                "abcd",
                0.6,
                {"input_tokens": 110, "cached_input_tokens": 10, "output_tokens": 40},
            ),
            self._usage_row(
                "medical-boundary",
                "candidate",
                "wxyz",
                0.3,
                {"input_tokens": 40, "output_tokens": 10},
            ),
        ]

        summary = run_evals.summarize_usage(rows)

        self.assertEqual("claude", summary["runner"])
        self.assertEqual(180, summary["conditions"]["baseline"]["input_tokens"])
        self.assertEqual(50, summary["conditions"]["candidate"]["output_tokens"])
        self.assertAlmostEqual(25.0, summary["conditions"]["candidate"]["mean_output_tokens"])
        self.assertEqual(-50, summary["delta"]["candidate"]["output_tokens"])
        self.assertAlmostEqual(-50.0, summary["delta"]["candidate"]["output_tokens_pct"])
        self.assertAlmostEqual(-0.6, summary["delta"]["candidate"]["cost_usd"])
        self.assertAlmostEqual(-40.0, summary["delta"]["candidate"]["cost_usd_pct"])
        self.assertAlmostEqual(
            -50.0, summary["delta"]["candidate"]["mean_response_chars_pct"]
        )

    def test_usage_summary_rejects_mixed_runners(self):
        rows = [
            self._usage_row("direct-answer", "baseline", "a", 0.1, {}, runner="claude"),
            self._usage_row("direct-answer", "candidate", "b", None, {}, runner="codex"),
        ]

        with self.assertRaisesRegex(ValueError, "claude.*codex"):
            run_evals.summarize_usage(rows)

    def test_usage_summary_rejects_unpaired_conditions(self):
        rows = [
            self._usage_row("direct-answer", "baseline", "a", 0.1, {}),
            self._usage_row("medical-boundary", "baseline", "b", 0.1, {}),
            self._usage_row("direct-answer", "candidate", "c", 0.1, {}),
        ]

        with self.assertRaisesRegex(ValueError, "not judged on the same rows"):
            run_evals.summarize_usage(rows)

    def test_usage_summary_marks_unreported_cost(self):
        rows = [
            self._usage_row("direct-answer", "baseline", "a", 0.1, {}),
            self._usage_row("direct-answer", "candidate", "b", None, {}),
        ]

        summary = run_evals.summarize_usage(rows)

        candidate = summary["conditions"]["candidate"]
        self.assertIsNone(candidate["input_tokens"])
        self.assertIsNone(candidate["output_tokens"])
        self.assertIsNone(candidate["cost_usd"])
        self.assertEqual(1, candidate["cost_usd_unreported_rows"])
        self.assertIsNone(summary["delta"]["candidate"]["cost_usd"])
        self.assertIsNone(summary["delta"]["candidate"]["cost_usd_pct"])

    def test_usage_tokens_handles_both_runner_shapes(self):
        self.assertEqual(
            (15, 4),
            run_evals._usage_tokens(
                {
                    "input_tokens": 10,
                    "cache_creation_input_tokens": 2,
                    "cache_read_input_tokens": 3,
                    "output_tokens": 4,
                }
            ),
        )
        self.assertEqual(
            (12, 7),
            run_evals._usage_tokens(
                {"input_tokens": 12, "cached_input_tokens": 5, "output_tokens": 7}
            ),
        )
        self.assertEqual((None, None), run_evals._usage_tokens({}))

    def test_usage_rejects_invalid_measurements(self):
        for cost in (-1, True, False, float("nan"), float("inf"), "0.1"):
            with self.subTest(cost=cost):
                rows = [self._usage_row("a", condition, "ok", cost, {})
                        for condition in ("baseline", "candidate")]
                with self.assertRaisesRegex(ValueError, "cost_usd"):
                    run_evals.summarize_usage(rows)
        for key in ("input_tokens", "output_tokens", "cache_creation_input_tokens",
                    "cache_read_input_tokens", "cached_input_tokens"):
            for value in (-1, True, 1.5, float("nan"), "10"):
                with self.subTest(key=key, value=value):
                    with self.assertRaisesRegex(ValueError, key):
                        run_evals._usage_tokens({key: value})
        with self.assertRaisesRegex(ValueError, "usage"):
            run_evals._usage_tokens([])

    def test_usage_rejects_overflowing_cost_total(self):
        rows = [self._usage_row(case, condition, "ok", 1e308, {})
                for case in ("a", "b") for condition in ("baseline", "candidate")]
        with self.assertRaisesRegex(ValueError, "cost total is not finite"):
            run_evals.summarize_usage(rows)

    def test_usage_distinguishes_missing_counts_from_zero(self):
        self.assertEqual((None, None), run_evals._usage_tokens(None))
        self.assertEqual((None, 2), run_evals._usage_tokens(
            {"cache_read_input_tokens": 10, "output_tokens": 2}))
        self.assertEqual((None, 2), run_evals._usage_tokens(
            {"input_tokens": 10, "cache_read_input_tokens": None, "output_tokens": 2}))
        rows = [self._usage_row("a", condition, "", 0,
                               {"input_tokens": 0, "output_tokens": 0})
                for condition in ("baseline", "candidate")]
        result = run_evals.summarize_usage(rows)
        self.assertEqual(0, result["conditions"]["candidate"]["input_tokens"])
        self.assertEqual(0, result["delta"]["candidate"]["cost_usd"])
        self.assertIsNone(result["delta"]["candidate"]["cost_usd_pct"])
        rows[1]["usage"] = None
        self.assertIsNone(run_evals.summarize_usage(rows)["delta"]["candidate"]["input_tokens"])

    def test_usage_rejects_duplicate_or_invalid_rows(self):
        rows = [self._usage_row("a", condition, "ok", 0, {})
                for condition in ("baseline", "candidate")]
        with self.assertRaisesRegex(ValueError, "duplicate"):
            run_evals.summarize_usage(rows + [rows[0]])
        for field, value in (("condition", "unknown"), ("case_id", None),
                             ("trial", True), ("trial", 0), ("runner", ""),
                             ("response", None)):
            with self.subTest(field=field):
                invalid = [dict(rows[0]), dict(rows[1])]
                invalid[1][field] = value
                with self.assertRaises(ValueError):
                    run_evals.summarize_usage(invalid)

    def test_usage_checks_model_metadata_when_available(self):
        rows = [self._usage_row("a", condition, "ok", 0, {})
                for condition in ("baseline", "candidate")]
        self.assertIsNone(run_evals.summarize_usage(rows)["model"])
        rows[0]["model"] = "model-a"
        with self.assertRaisesRegex(ValueError, "same model"):
            run_evals.summarize_usage(rows)
        rows[1]["model"] = "model-b"
        with self.assertRaisesRegex(ValueError, "same model"):
            run_evals.summarize_usage(rows)
        rows[1]["model"] = "model-a"
        self.assertEqual("model-a", run_evals.summarize_usage(rows)["model"])

    def test_measure_cli_reports_increased_input_and_cost(self):
        rows = [
            self._usage_row("a", "baseline", "long answer", 0.1,
                            {"input_tokens": 100, "output_tokens": 20}),
            self._usage_row("a", "candidate", "short", 0.15,
                            {"input_tokens": 150, "output_tokens": 10}),
        ]
        with tempfile.TemporaryDirectory() as temporary:
            path = Path(temporary) / "responses.jsonl"
            contents = "".join(json.dumps(row) + "\n" for row in rows)
            path.write_text(contents, encoding="utf-8")
            result = subprocess.run(
                [sys.executable, str(ROOT / "scripts/run_evals.py"), "measure", str(path)],
                check=True, capture_output=True, text=True, timeout=10,
            )
            self.assertEqual(contents, path.read_text(encoding="utf-8"))
        delta = json.loads(result.stdout)["delta"]["candidate"]
        self.assertEqual(50, delta["input_tokens"])
        self.assertEqual(50, delta["input_tokens_pct"])
        self.assertEqual(-50, delta["output_tokens_pct"])
        self.assertAlmostEqual(50, delta["cost_usd_pct"])

    @staticmethod
    def _score_row(case_id, condition, value, trial=1):
        return {
            "case_id": case_id,
            "trial": trial,
            "condition": condition,
            "correctness": value,
            "autonomy": value,
            "actionability": value,
            "safety": value,
            "concision": value,
            "blocker": False,
            "notes": "fixture",
        }

    @staticmethod
    def _usage_row(case_id, condition, response, cost, usage, runner="claude", trial=1):
        return {
            "case_id": case_id,
            "trial": trial,
            "condition": condition,
            "runner": runner,
            "response": response,
            "usage": usage,
            "cost_usd": cost,
        }

    def test_duplicate_case_ids_are_rejected(self):
        case = {
            "id": "duplicate",
            "category": "direct-answer",
            "prompt": "What is 2 + 2?",
            "risk": "low",
            "criteria": ["Answers 4."],
        }
        errors = run_evals.validate_cases([case, dict(case)])
        self.assertTrue(any("Duplicate" in error for error in errors))

    def test_jsonl_loader_reports_invalid_rows(self):
        with tempfile.TemporaryDirectory() as tmp:
            path = Path(tmp) / "bad.jsonl"
            path.write_text(json.dumps({"id": "ok"}) + "\nnot-json\n")
            with self.assertRaisesRegex(ValueError, "line 2"):
                run_evals.read_jsonl(path)

    def test_condition_prompt_injects_the_skill_body_without_frontmatter(self):
        # hooks/always-on.sh strips the YAML frontmatter before injecting the
        # ruleset, so the eval must grade the same text that actually ships.
        with tempfile.TemporaryDirectory() as tmp:
            skill = Path(tmp) / "SKILL.md"
            skill.write_text(
                "---\n"
                "name: i-have-adhd\n"
                "disable-model-invocation: true\n"
                "metadata:\n"
                "  hermes:\n"
                "    tags: [ADHD]\n"
                "---\n"
                "\n"
                "# i-have-adhd\n"
                "\n"
                "Lead with the next action.\n"
            )

            prompt = run_evals._condition_prompt("Fix the bug.", "candidate", skill)

            self.assertIn("Lead with the next action.", prompt)
            self.assertIn("Fix the bug.", prompt)
            self.assertNotIn("disable-model-invocation", prompt)
            self.assertNotIn("hermes", prompt)

    def test_condition_prompt_keeps_a_skill_body_that_has_no_frontmatter(self):
        with tempfile.TemporaryDirectory() as tmp:
            skill = Path(tmp) / "SKILL.md"
            skill.write_text("# No frontmatter here\n\nLead with the next action.\n")

            prompt = run_evals._condition_prompt("Fix the bug.", "candidate", skill)

            self.assertIn("# No frontmatter here", prompt)

    def test_unmetered_runner_is_rejected_before_any_call(self):
        with tempfile.TemporaryDirectory() as tmp:
            tmp_path = Path(tmp)
            marker = tmp_path / "ran"
            runner_config = tmp_path / "runners.json"
            runner_config.write_text(
                json.dumps(
                    {
                        "stub": {
                            "command": [
                                sys.executable,
                                "-c",
                                f"from pathlib import Path; Path({str(marker)!r}).touch(); print('hi')",
                            ],
                            "response_format": "text",
                        }
                    }
                )
            )
            args = argparse.Namespace(
                cases=ROOT / "evals" / "cases.jsonl",
                runner_config=runner_config,
                runner="stub",
                condition="baseline",
                condition_skill=None,
                case=["direct-answer"],
                trials=1,
                retries=0,
                budget_usd=1.0,
                allow_unmetered=False,
                output=tmp_path / "out.jsonl",
            )

            with self.assertRaisesRegex(RuntimeError, "never reports dollar cost"):
                run_evals.run_evaluations(args)

            self.assertFalse(marker.exists(), "runner was invoked before the rejection")
            self.assertFalse((tmp_path / "out.jsonl").exists())

            args.allow_unmetered = True
            self.assertEqual(0, run_evals.run_evaluations(args))
            self.assertTrue(marker.exists())

    def test_generation_runs_outside_the_repository(self):
        # An agent CLI adopts its working directory as project context. Run it
        # in this checkout and it answers prompts by inspecting the harness,
        # which contaminates the responses being compared.
        with tempfile.TemporaryDirectory() as tmp:
            tmp_path = Path(tmp)
            runner_config = tmp_path / "runners.json"
            runner_config.write_text(
                json.dumps({"pwd": {"command": ["sh", "-c", "pwd"], "response_format": "text"}})
            )
            output = tmp_path / "out.jsonl"
            args = argparse.Namespace(
                cases=ROOT / "evals" / "cases.jsonl",
                runner_config=runner_config,
                runner="pwd",
                condition="baseline",
                condition_skill=None,
                case=["direct-answer"],
                trials=1,
                retries=0,
                budget_usd=1.0,
                allow_unmetered=True,
                output=output,
            )

            self.assertEqual(0, run_evals.run_evaluations(args))

            where = Path(run_evals.read_jsonl(output)[0]["response"].strip()).resolve()
            self.assertNotEqual(ROOT.resolve(), where)
            self.assertFalse(str(where).startswith(str(ROOT.resolve())))

    def test_completed_keys_support_resuming_partial_runs(self):
        rows = [
            {
                "case_id": "direct-answer",
                "trial": 1,
                "condition": "baseline",
                "runner": "claude",
            }
        ]

        self.assertEqual(
            {("direct-answer", 1, "baseline", "claude")},
            run_evals.completed_keys(rows),
        )


if __name__ == "__main__":
    unittest.main()