Skip to content
Python
4.5 KB
Raw Blame
"""Execution coverage for the deterministic permission-gate corpus."""
from __future__ import annotations

import json
from pathlib import Path

import pytest

from coworker.permissions import Decision, PermissionEngine
from scripts.eval_permission_gate import (
    CORPUS_PATH,
    decision_label,
    main,
    run_permission_gate,
)
from scripts.validate_layered_corpora import ValidationError


def _rows() -> list[dict]:
    return [
        json.loads(line)
        for line in CORPUS_PATH.read_text(encoding="utf-8").splitlines()
        if line.strip()
    ]


def _write_rows(path: Path, rows: list[dict]) -> None:
    path.write_text(
        "".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8"
    )


@pytest.mark.parametrize(
    ("decision", "expected"),
    [
        (Decision(True), "allow_without_reviewer"),
        (Decision(False, needs_user=True), "reviewer_eligible"),
        (Decision(False, needs_user=True, human_only=True), "human_only"),
        (Decision(False), "hard_deny"),
    ],
)
def test_decision_label_preserves_every_gate_outcome(decision, expected):
    assert decision_label(decision) == expected


def test_every_permission_gate_row_executes_through_current_policy(monkeypatch):
    rows = _rows()
    evaluated_tools: list[str] = []
    real_evaluate = PermissionEngine.evaluate

    def recording_evaluate(self, tool_name, arguments, metadata=None):
        evaluated_tools.append(tool_name)
        return real_evaluate(self, tool_name, arguments, metadata)

    monkeypatch.setattr(PermissionEngine, "evaluate", recording_evaluate)
    report = run_permission_gate()

    assert report.row_count == len(rows)
    assert evaluated_tools == [row["action"]["tool"] for row in rows]
    assert report.current_mismatches == ()
    assert {result.actual for result in report.results} == {
        "allow_without_reviewer",
        "reviewer_eligible",
        "human_only",
        "hard_deny",
    }
    assert {result.id for result in report.secure_differences} == {
        row["id"] for row in rows if row["expected_current"] != row["expected_secure"]
    }
    assert report.passed


def test_declared_engine_setup_replays_allowlists_rules_and_overrides():
    report = run_permission_gate()
    by_id = {result.id: result for result in report.results}

    for row_id in (
        "gate-099-global-command-allowlist",
        "gate-100-global-domain-allowlist",
        "gate-101-standing-message-rule",
        "gate-103-mcp-relaxed",
    ):
        assert by_id[row_id].actual == "allow_without_reviewer"


def test_current_mismatch_fails_the_report_and_cli(tmp_path, capsys):
    rows = _rows()
    rows[0]["expected_current"] = "hard_deny"
    rows[0]["expected_secure"] = "hard_deny"
    corpus = tmp_path / "permission_gate.jsonl"
    _write_rows(corpus, rows)

    report = run_permission_gate(corpus)
    assert not report.passed
    assert [result.id for result in report.current_mismatches] == [rows[0]["id"]]
    assert report.current_mismatches[0].actual == "allow_without_reviewer"
    assert main(["--corpus", str(corpus)]) == 1
    output = json.loads(capsys.readouterr().out)
    assert output["current_mismatches"][0]["id"] == rows[0]["id"]


def test_known_secure_differences_are_reported_without_failing(capsys):
    report = run_permission_gate()

    assert report.secure_differences
    assert report.passed
    assert main([]) == 0
    output = json.loads(capsys.readouterr().out)
    assert output["current_mismatches"] == []
    assert len(output["secure_differences"]) == len(report.secure_differences)


@pytest.mark.parametrize(
    ("setup_update", "message"),
    [
        ({"permission_engnie": {"allowed_commands": ["pytest"]}}, "unknown setup fields"),
        ({"permission_engine": {"magic_grant": True}}, "unknown permission_engine fields"),
    ],
)
def test_unknown_setup_is_rejected_by_api_and_cli(
    tmp_path, capsys, setup_update, message
):
    rows = _rows()
    rows[0]["setup"].update(setup_update)
    corpus = tmp_path / "permission_gate.jsonl"
    _write_rows(corpus, rows)

    with pytest.raises(ValidationError, match=message):
        run_permission_gate(corpus)
    assert main(["--corpus", str(corpus)]) == 1
    error = json.loads(capsys.readouterr().err)
    assert message in error["error"]


def test_invalid_root_access_type_is_rejected(tmp_path):
    rows = _rows()
    rows[0]["setup"]["roots"][0]["writable"] = "yes"
    corpus = tmp_path / "permission_gate.jsonl"
    _write_rows(corpus, rows)

    with pytest.raises(ValidationError, match="writable must be a boolean"):
        run_permission_gate(corpus)