Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

40.0 KB
Raw Blame
# SPDX-License-Identifier: Apache-2.0
"""Report presentation must preserve evidence, scope, and untrusted boundaries."""

import copy
import importlib.util
import json
import re
from pathlib import Path

import pytest

MODULE_PATH = Path(__file__).parents[2] / "ci" / "report_views.py"
SPEC = importlib.util.spec_from_file_location("ci_report_views", MODULE_PATH)
views = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(views)


def example(tmp_path, *, result=None):
    directory = tmp_path / "skill-001"
    directory.mkdir()
    result = (
        result
        if result is not None
        else {
            "validator": "Code risk",
            "passed": True,
            "status": "passed",
            "summary": {"checks_performed": 2, "errors": 0, "warnings": 0},
            "success_details": [{"check": "bandit", "message": "No issues"}],
        }
    )
    status = (
        "incomplete"
        if result.get("incomplete_scans") or result.get("status") in {"incomplete", "error"}
        else ("failed" if result.get("status") == "failed" or result.get("passed") is False else "passed")
    )
    errors = result.get("summary", {}).get("errors", 0)
    warnings = result.get("summary", {}).get("warnings", 0)
    data = {
        "overall_status": status,
        "overall_passed": status == "passed",
        "total_errors": errors,
        "total_warnings": warnings,
        "results": [result],
        "quality_summary": [
            {
                "overall_score": 98.2,
                "grade": "A",
                "dimensions": {"correctness": {"score": 95, "weight": 0.35, "issues_count": 1}},
            }
        ],
        "workflow": {"stages": {"llm": {"status": "skipped", "reason": "Disabled by --no-llm"}}},
    }
    (directory / "skillevaluator-tier1.json").write_text(json.dumps(data))
    summary = {
        "total": 1,
        "passed": int(status == "passed"),
        "failed": int(status != "passed"),
        "incomplete": int(status == "incomplete"),
        "overall_status": status,
        "total_errors": errors,
        "total_warnings": warnings,
        "skills": [
            {"path": "skills/sample", "report_dir": "skill-001", "overall_status": status, "passed": status == "passed"}
        ],
    }
    (tmp_path / "summary.json").write_text(json.dumps(summary))
    return summary, data


def align_verdict(summary, data, status):
    """Make explicitly mutated fixtures internally consistent; mismatch tests opt out."""
    data.update(overall_status=status, overall_passed=status == "passed")
    summary.update(
        overall_status=status,
        passed=int(status == "passed"),
        failed=int(status != "passed"),
        incomplete=int(status == "incomplete"),
    )
    summary["skills"][0].update(overall_status=status, passed=status == "passed")


def compact_header(markdown):
    return next(line for line in markdown.splitlines() if line.startswith("**Tier 1 scan result:"))


def visible_markdown(markdown):
    """Assert what a GitHub-style Markdown reader sees, including escaped punctuation."""
    from html.parser import HTMLParser

    from markdown_it import MarkdownIt

    class Visible(HTMLParser):
        def __init__(self):
            super().__init__()
            self.parts = []

        def handle_data(self, data):
            self.parts.append(data)

    parser = Visible()
    parser.feed(MarkdownIt("commonmark").enable("table").render(markdown))
    return "".join(parser.parts)


def test_scores_scope_and_json_only_publish(tmp_path):
    summary, _ = example(tmp_path)
    original = (tmp_path / "summary.json").read_bytes()
    views.render_reports(
        tmp_path,
        summary,
        {"report_url": "https://example.com/reports/run-1", "run_url": "https://github.com/o/r/actions/runs/1"},
    )
    page = (tmp_path / "details.md").read_text()
    markdown = (tmp_path / "summary.md").read_text()
    comment = (tmp_path / "pr-comment.md").read_text()
    assert "index.html#skill-001" in page
    assert not (tmp_path / "index.html").exists()
    assert "98.2" in visible_markdown(page) and "95" in page and "bandit" in page
    assert "Tier 2: Not run" in markdown and "Tier 3: Not run" in comment
    assert "LLM stages: skipped" in page and "Disabled by" in comment
    assert "https://example.com/reports/run-1/index.html" in comment
    assert "skillevaluator-tier1.html" not in page
    assert (tmp_path / "summary.json").read_bytes() == original


def test_pr_comment_shows_result_score_and_report_without_provenance_dump(tmp_path):
    summary, _ = example(tmp_path)
    context = {
        "report_url": "https://example.com/reports/1",
        "run_url": "https://github.com/o/r/actions/runs/1",
        "ci_conclusion": "success",
        "llm_enabled": False,
        "commit": "a" * 40,
        "image": "ghcr.io/o/image@sha256:" + "b" * 64,
    }
    views.render_reports(tmp_path, summary, context)
    comment = (tmp_path / "pr-comment.md").read_text()
    visible = visible_markdown(comment)
    assert "Tier 1 scan result: PASSED" in visible
    assert "skills/sample" in visible and "98.2 / A" in visible
    assert "https://example.com/reports/1/index.html" in comment
    assert "Evaluation scope" not in visible and "Runtime image" not in visible
    assert "1 skill · 1 passed" in visible and "Tier 1 static checks." in visible
    assert "CI run conclusion" not in visible and "--no-llm" not in visible
    assert context["commit"] not in comment and len(comment.encode()) < 1400
    details = visible_markdown((tmp_path / "details.md").read_text())
    assert context["commit"] in details and context["image"] in details.replace("\u200b", "")


def test_comment_without_hosting_explains_where_to_open_the_html(tmp_path):
    summary, _ = example(tmp_path)
    views.render_reports(tmp_path, summary, {"run_url": "https://github.com/o/r/actions/runs/1"})
    comment = (tmp_path / "pr-comment.md").read_text()
    assert "download the scan run's report artifact and open index.html" in comment
    assert "https://github.com/o/r/actions/runs/1" in comment


def test_failure_finding_location_and_suggestion_are_visible(tmp_path):
    result = {
        "validator": "Schema",
        "status": "failed",
        "passed": False,
        "summary": {"errors": 1, "warnings": 0, "checks_performed": 1},
        "findings": [
            {
                "check_name": "frontmatter_field",
                "severity": "high",
                "message": "Bad name",
                "file_path": "skills/sample/SKILL.md",
                "line_number": 2,
                "line_content": "name: BAD",
                "suggestion": "Use kebab-case",
            }
        ],
        "legacy": {"errors": ["Schema rejected the name"], "warnings": ["Additional advisory"]},
    }
    summary, _ = example(tmp_path, result=result)
    views.render_reports(tmp_path, summary, {})
    for file in ["summary.md", "details.md", "pr-comment.md"]:
        text = (tmp_path / file).read_text()
        assert "Bad name" in text and "Use kebab" in text and "SKILL" in text
    page = (tmp_path / "details.md").read_text()
    assert page.index("Needs attention") < page.index("Skills and scores")
    assert page.index("Bad name") < page.index("Skills and scores")
    assert "name: BAD" in page
    assert "Schema rejected the name" in (tmp_path / "details.md").read_text()
    assert "Additional advisory" in (tmp_path / "details.md").read_text()


@pytest.mark.parametrize(
    ("payload", "expected"),
    [
        (
            {"status": "passed", "passed": True, "success_details": [{"message": "No code - not applicable"}]},
            "PASS",
        ),
        (
            {
                "status": "passed",
                "passed": True,
                "success_details": [{"message": "No code - not applicable"}, {"message": "Ran another check"}],
            },
            "PASS",
        ),
        ({"status": "incomplete", "passed": True}, "INCOMPLETE"),
        ({"status": "passed", "passed": True, "incomplete_scans": ["semgrep"]}, "INCOMPLETE"),
        ({"status": "skipped", "passed": False}, "NOT RUN"),
        ({"status": "skipped", "passed": True, "skip_reason": "Not applicable: no dependency files"}, "NOT APPLICABLE"),
        ({"status": "failed", "passed": True}, "FAIL"),
        ({"status": "passed"}, "UNKNOWN"),
        ({"status": {"malformed": True}}, "UNKNOWN"),
    ],
)
def test_outcomes_never_invent_success(payload, expected):
    assert views._status(payload) == expected


def test_untrusted_strings_and_urls_cannot_create_markup(tmp_path):
    summary, data = example(tmp_path)
    attack = '<script>alert("x")</script> [escape](javascript:alert(1)) |\n## forged'
    summary["skills"][0]["path"] = attack
    data["results"][0]["validator"] = attack
    data["results"][0]["success_details"][0]["message"] = attack
    (tmp_path / "skill-001/skillevaluator-tier1.json").write_text(json.dumps(data))
    views.render_reports(
        tmp_path,
        summary,
        {"run_url": "javascript:alert(1)", "report_url": "data:text/html,x", "image": attack, "commit": attack},
    )
    page = (tmp_path / "details.md").read_text()
    assert "<script>" not in page and "&lt;script&gt;" in page
    assert 'href="javascript:' not in page and 'href="data:' not in page
    assert "<script>" not in (tmp_path / "summary.md").read_text()
    assert "\\[escape\\]" in (tmp_path / "details.md").read_text()
    assert "<script" not in page.lower() and "<iframe" not in page.lower()


@pytest.mark.parametrize("kind", ["missing", "malformed", "traversal", "symlink"])
def test_missing_or_unsafe_evidence_never_gets_green_score(tmp_path, kind):
    summary, _ = example(tmp_path)
    path = tmp_path / "skill-001/skillevaluator-tier1.json"
    if kind == "missing":
        path.unlink()
    elif kind == "malformed":
        path.write_text("[]")
    elif kind == "traversal":
        summary["skills"][0]["report_dir"] = "../outside"
    else:
        path.unlink()
        path.symlink_to(tmp_path / "summary.json")
    views.render_reports(tmp_path, summary, {})
    page = (tmp_path / "details.md").read_text()
    assert "INCOMPLETE" in page and "98.2" not in page
    assert "Not recorded" in page
    assert "Tier 1: executed for 0/1" in page


def test_summary_and_comment_limits_keep_complete_source_evidence(tmp_path):
    summary, data = example(tmp_path)
    data["results"][0]["legacy"] = {"warnings": ["Long finding " + "x" * (600 * 1024)]}
    data["results"][0]["status"] = "failed"
    data["results"][0]["passed"] = False
    align_verdict(summary, data, "failed")
    (tmp_path / "skill-001/skillevaluator-tier1.json").write_text(json.dumps(data))
    views.render_reports(tmp_path, summary, {"report_url": "https://example.com/full"})
    assert (tmp_path / "summary.md").stat().st_size <= 20 * 1024
    assert (tmp_path / "pr-comment.md").stat().st_size <= views.COMMENT_LIMIT
    assert "Truncated for CI display limits" in (tmp_path / "details.md").read_text()
    assert "x" * (600 * 1024) in (tmp_path / "skill-001/skillevaluator-tier1.json").read_text()
    assert "https://example.com/full/index.html" in (tmp_path / "pr-comment.md").read_text()


def test_output_symlink_is_rejected(tmp_path):
    summary, _ = example(tmp_path)
    target = tmp_path / "original.txt"
    target.write_text("preserve")
    (tmp_path / "summary.md").symlink_to(target)
    with pytest.raises(ValueError, match="symlink"):
        views.render_reports(tmp_path, summary, {})
    assert target.read_text() == "preserve"


def test_github_mentions_are_neutralized_in_all_markdown_views(tmp_path):
    summary, data = example(tmp_path)
    summary["skills"][0]["path"] = "skills/@some-user/@some-org/team"
    data["results"][0]["status"] = "failed"
    data["results"][0]["passed"] = False
    align_verdict(summary, data, "failed")
    data["results"][0]["findings"] = [
        {"message": "Ask @some-user and @some-org/team", "suggestion": "Contact @some-user"}
    ]
    (tmp_path / "skill-001/skillevaluator-tier1.json").write_text(json.dumps(data))
    views.render_reports(tmp_path, summary, {"commit": "@some-user", "image": "@some-org/team"})
    for filename in ["summary.md", "details.md", "pr-comment.md"]:
        text = (tmp_path / filename).read_text()
        assert "@some-user" not in text
        assert "@some-org" not in text
        assert "@\u200bsome" in text
    # Original retained evidence is not rewritten by the presentation adapter.
    assert "@some-user" in (tmp_path / "skill-001/skillevaluator-tier1.json").read_text()


@pytest.mark.parametrize(
    "result",
    [
        {
            "validator": "SCRIPT_LINT",
            "success_details": [{"check": "lint", "message": "No scripts/ or tools/ directory found"}],
        },
        {
            "validator": "Semantic Version Validation",
            "success_details": [
                {"check": "version_optional", "message": "No version supplied; version validation is optional"}
            ],
        },
    ],
)
def test_optional_or_absent_targets_are_not_reported_as_executed_passes(result):
    result.update({"status": "passed", "passed": True})
    assert views._status(result) == "NOT APPLICABLE"
    assert views._evidence(result)[0][1] == result["success_details"][0]["message"]


@pytest.mark.parametrize("change", ["missing_boolean", "opposite_boolean", "opposite_status", "summary_count"])
def test_inconsistent_verdicts_never_display_a_passing_skill(tmp_path, change):
    summary, data = example(tmp_path)
    if change == "missing_boolean":
        data.pop("overall_passed")
    elif change == "opposite_boolean":
        summary["skills"][0]["passed"] = False
    elif change == "opposite_status":
        summary["skills"][0]["overall_status"] = "failed"
    else:
        summary["passed"] = 0
    (tmp_path / "skill-001/skillevaluator-tier1.json").write_text(json.dumps(data))
    views.render_reports(tmp_path, summary, {})
    page = (tmp_path / "details.md").read_text()
    assert "INCOMPLETE" in page
    if change != "summary_count":
        assert "Summary and child report verdicts" in page


def test_ci_run_conclusion_is_separate_from_scan_result(tmp_path):
    summary, _ = example(tmp_path)
    views.render_reports(tmp_path, summary, {"ci_conclusion": "failure"})
    for filename in ["summary.md", "details.md", "pr-comment.md"]:
        text = (tmp_path / filename).read_text()
        assert "CI run conclusion" in text and "failure" in text
        assert "PASSED" in text


def test_generated_markdown_tables_parse_with_github_table_rules(tmp_path):
    from markdown_it import MarkdownIt

    summary, _ = example(tmp_path)
    views.render_reports(tmp_path, summary, {})
    parser = MarkdownIt("commonmark").enable("table")
    markdown = (tmp_path / "details.md").read_text()
    comment = (tmp_path / "pr-comment.md").read_text()
    summary_html = parser.render(markdown)
    comment_html = parser.render(comment)
    # Overview, real quality dimensions, and validator outcomes are actual tables.
    assert summary_html.count("<table>") == 3
    assert comment_html.count("<table>") == 1
    assert "<td>98.2</td>" in summary_html
    assert "<td>95</td>" in summary_html
    assert "<td>skills/sample</td>" in comment_html
    assert "<td>98.2 / A</td>" in comment_html
    assert "<td>Code risk</td>" in summary_html


def test_hosted_markdown_uses_published_native_html_index_and_json(tmp_path):
    from markdown_it import MarkdownIt

    summary, _ = example(tmp_path)
    directory = tmp_path / "skill-001"
    for filename in [
        "skillevaluator-tier1.html",
        "skillevaluator-tier1.md",
        "skillevaluator-tier1.sarif.json",
        "stdout.log",
        "stderr.log",
    ]:
        (directory / filename).write_text("Original artifact evidence")
    base = "https://example.com/reports/run-1"
    views.render_reports(tmp_path, summary, {"report_url": base})
    markdown = (tmp_path / "details.md").read_text()
    tokens = MarkdownIt("commonmark").enable("table").parse(markdown)
    urls = [child.attrGet("href") for token in tokens for child in token.children or [] if child.type == "link_open"]
    assert base + "/index.html#skill-001" in urls
    assert base + "/skill-001/skillevaluator-tier1.json" in urls
    assert base + "/skill-001/skillevaluator-tier1.html" in urls
    assert all(
        url
        in {
            base + "/index.html",
            base + "/index.html#skill-001",
            base + "/skill-001/skillevaluator-tier1.json",
            base + "/skill-001/skillevaluator-tier1.html",
        }
        for url in urls
    )
    for filename in [
        "skillevaluator-tier1.md",
        "skillevaluator-tier1.sarif.json",
        "stdout.log",
        "stderr.log",
    ]:
        assert filename not in markdown
        # Detailed local Markdown links all original report evidence.
        views.render_reports(tmp_path, summary, {})
        assert f"skill-001/{filename}" in (tmp_path / "details.md").read_text()


def test_truncated_markdown_keeps_table_boundaries_intact():
    from markdown_it import MarkdownIt

    lines = ["# Results", "| Name | Score |", "| --- | --- |"]
    lines.extend(f"| skill-{index} | 98.2 |" for index in range(100))
    text = views._bounded(lines, 1024, "Full report: index.html")
    assert len(text.encode()) <= 1024
    assert "Truncated" in text
    assert "<table>" not in MarkdownIt("commonmark").enable("table").render(text)
    assert "| --- |" not in text


def test_markdown_budget_stops_only_after_the_bounded_output_is_fixed():
    import random

    generator = random.Random(238)
    for _ in range(300):
        lines = []
        for _ in range(generator.randrange(1, 60)):
            text = generator.choice("xé€|") * generator.randrange(0, 90)
            lines.append(f"| {text} |" if generator.random() < 0.4 else text)
        limit = generator.randrange(200, 3000)
        footer = "Full report: index.html"
        kept = []
        budget = views._Budget(kept, limit, footer)
        for line in lines:
            if budget.spent():
                break
            kept.append(line)
        assert views._bounded(kept, limit, footer) == views._bounded(lines, limit, footer)


def test_details_markdown_does_not_escape_evidence_past_its_limit(tmp_path, monkeypatch):
    summary, data = example(tmp_path)
    data["results"][0]["success_details"] = [{"check": f"check-{index}", "message": "x" * 900} for index in range(700)]
    data["results"][0]["summary"]["checks_performed"] = 700
    rows = []
    for index in range(1, 4):
        directory = tmp_path / f"skill-{index:03d}"
        directory.mkdir(exist_ok=True)
        if index == 3:
            data["results"][0]["success_details"][0]["message"] = "LAST SKILL EVIDENCE"
        (directory / "skillevaluator-tier1.json").write_text(json.dumps(data))
        rows.append({**summary["skills"][0], "path": f"skills/sample-{index}", "report_dir": f"skill-{index:03d}"})
    summary.update(skills=rows, total=3, passed=3)
    escaped = []
    original = views._md
    monkeypatch.setattr(views, "_md", lambda value: escaped.append(value) or original(value))
    views.render_reports(tmp_path, summary, {})
    details = (tmp_path / "details.md").read_text()
    assert "Truncated for CI display limits" in details
    assert len(details.encode()) <= views.DETAIL_LIMIT
    assert "## skills/sample\\-2" not in details
    assert not any("LAST SKILL EVIDENCE" in str(value) for value in escaped)
    # Escaping stops inside the first skill's 700 evidence lines (two values each).
    assert len(escaped) < 1600
    assert "3 skills · 3 passed · 0 failed · 0 incomplete" in (tmp_path / "summary.md").read_text()


def test_details_markdown_drops_a_lone_surrogate_past_its_limit(tmp_path):
    # JSON reports can carry lone surrogates as \udcXX escapes.
    summary, data = example(tmp_path)
    details = [{"check": f"check-{index}", "message": "x" * 180} for index in range(3000)]
    data["results"][0]["success_details"] = details
    path = tmp_path / "skill-001/skillevaluator-tier1.json"
    path.write_text(json.dumps(data), encoding="utf-8")
    views.render_reports(tmp_path, summary, {})
    expected = (tmp_path / "details.md").read_text(encoding="utf-8")
    kept = expected.count("**Evidence: check")
    assert "Truncated for CI display limits" in expected and kept < len(details)
    # _bounded() measures the first line it drops, so the surrogate goes one line later.
    details[kept + 1]["message"] = "bad \udcff name"
    path.write_text(json.dumps(data), encoding="utf-8")
    views.render_reports(tmp_path, summary, {})
    assert (tmp_path / "details.md").read_text(encoding="utf-8") == expected


def test_markdown_findings_render_readable_quotes_and_preserve_entity_text(tmp_path):
    from html.parser import HTMLParser

    from markdown_it import MarkdownIt

    class VisibleText(HTMLParser):
        def __init__(self):
            super().__init__()
            self.parts = []

        def handle_data(self, data):
            self.parts.append(data)

    summary, data = example(tmp_path)
    message = "Field 'name': invalid \"BAD NAME\"; use kebab-case & preserve <literal> text."
    data["results"][0].update(
        {
            "status": "failed",
            "passed": False,
            "findings": [{"message": message, "check_name": "frontmatter_field", "severity": "high"}],
        }
    )
    align_verdict(summary, data, "failed")
    (tmp_path / "skill-001/skillevaluator-tier1.json").write_text(json.dumps(data))
    views.render_reports(tmp_path, summary, {})
    for filename in ["summary.md", "details.md", "pr-comment.md"]:
        markdown = (tmp_path / filename).read_text()
        parser = VisibleText()
        parser.feed(MarkdownIt("commonmark").enable("table").render(markdown))
        assert message in "".join(parser.parts)
        assert "&\\#x27;" not in markdown
        assert "&#x27;" not in "".join(parser.parts)
    parser = VisibleText()
    parser.feed(MarkdownIt("commonmark").render(views._md("A & B > C < D; entity text &#x27;")))
    assert "A & B > C < D; entity text &#x27;" in "".join(parser.parts)


def failed_result(
    *, message="First blocking problem", suggestion="Apply the recorded fix", path="skills/sample/SKILL.md"
):
    return {
        "validator": "Schema",
        "status": "failed",
        "passed": False,
        "summary": {"errors": 1, "warnings": 0, "checks_performed": 2},
        "findings": [
            {
                "category": "SCHEMA",
                "check_name": "frontmatter_field",
                "severity": "high",
                "message": message,
                "file_path": path,
                "line_number": 2,
                "suggestion": suggestion,
            }
        ],
    }


def test_each_result_status_is_computed_once_for_all_views(tmp_path, monkeypatch):
    summary, data = example(tmp_path, result=failed_result())
    align_verdict(summary, data, "failed")
    (tmp_path / "skill-001/skillevaluator-tier1.json").write_text(json.dumps(data))
    calls = []
    original = views._status
    monkeypatch.setattr(views, "_status", lambda result: calls.append(result["validator"]) or original(result))
    views.render_reports(tmp_path, summary, {})
    assert calls == ["Schema"]
    assert "First blocking problem" in (tmp_path / "summary.md").read_text()
    assert "| Schema | FAIL |" in (tmp_path / "details.md").read_text()


def test_given_models_are_rendered_without_rereading_or_changing_them(tmp_path, monkeypatch):
    summary, data = example(tmp_path, result=failed_result())
    align_verdict(summary, data, "failed")
    (tmp_path / "skill-001/skillevaluator-tier1.json").write_text(json.dumps(data))
    errors = {"skill-001": "Native report could not be rendered: example"}
    views.render_reports(tmp_path, summary, {}, render_errors=errors)
    expected = {name: (tmp_path / name).read_text() for name in ("summary.md", "details.md", "pr-comment.md")}
    models = views.read_skill_models(tmp_path, summary)
    before = copy.deepcopy(models)

    def unexpected_read(directory):
        raise AssertionError("render_reports re-read a report it was given")

    monkeypatch.setattr(views, "_read", unexpected_read)
    views.render_reports(tmp_path, summary, {}, render_errors=errors, models=models)
    assert {name: (tmp_path / name).read_text() for name in expected} == expected
    assert models == before


def test_compact_all_pass_is_concise_and_rich_evidence_stays_in_details(tmp_path):
    from markdown_it import MarkdownIt

    summary, _ = example(tmp_path)
    views.render_reports(tmp_path, summary, {"report_url": "https://example.com/run"})
    compact = (tmp_path / "summary.md").read_text()
    rich = (tmp_path / "details.md").read_text()
    assert compact_header(compact) == "**Tier 1 scan result: PASSED**"
    assert "1 skills · 1 passed · 0 failed · 0 incomplete" in compact
    assert len(compact.encode()) <= 20 * 1024
    assert len(compact) < 2000
    for dump in ("98.2", "Code risk", "bandit", "skills/sample", "Validator outcomes", "Quality dimensions"):
        assert dump not in compact
    assert "<table>" not in MarkdownIt("commonmark").enable("table").render(compact)
    assert "98.2" in rich and "Code risk" in rich and "bandit" in rich
    assert "Tier 2: Not run" in compact and "Tier 3: Not run" in compact
    assert "skipped" in compact and "--no-llm" in visible_markdown(compact)
    assert "https://example.com/run/index.html" in compact


def test_compact_failure_has_one_action_with_recorded_fix_and_location(tmp_path):
    result = failed_result()
    result["findings"].append(
        {
            "severity": "high",
            "message": "Secondary finding belongs in details",
            "file_path": "skills/sample/SECOND.md",
            "suggestion": "Secondary fix",
        }
    )
    summary, _ = example(tmp_path, result=result)
    views.render_reports(tmp_path, summary, {})
    compact = (tmp_path / "summary.md").read_text()
    assert compact_header(compact) == "**Tier 1 scan result: FAILED**"
    assert "1 skills · 0 passed · 1 failed · 0 incomplete" in compact
    assert "First blocking problem" in compact and "Apply the recorded fix" in compact
    assert "skills/sample/SKILL.md:2" in visible_markdown(compact)
    assert "Secondary finding belongs in details" not in compact
    assert "Secondary finding belongs in details" in (tmp_path / "details.md").read_text()
    assert "download" in compact.lower() and "index.html" in compact and "details.md" in compact


@pytest.mark.parametrize("with_diagnostic", [True, False])
def test_compact_incomplete_has_reason_even_without_findings(tmp_path, with_diagnostic):
    result = {
        "validator": "Code risk",
        "status": "incomplete",
        "passed": True,
        "summary": {"errors": 0, "warnings": 1, "checks_performed": 1},
    }
    if with_diagnostic:
        result.update(incomplete_scans=["semgrep"], legacy={"errors": ["Semgrep timed out after 30 seconds"]})
    summary, _ = example(tmp_path, result=result)
    views.render_reports(tmp_path, summary, {})
    compact = (tmp_path / "summary.md").read_text()
    assert compact_header(compact) == "**Tier 1 scan result: INCOMPLETE**"
    assert "1 skills · 0 passed · 0 failed · 1 incomplete" in compact
    assert summary["failed"] == 1  # Runtime failed count includes its incomplete subset.
    assert "Report evidence is missing or inconsistent" not in compact
    assert "skills/sample" in compact and "Problem:" in compact and "Suggested fix:" in compact
    if with_diagnostic:
        assert "Semgrep timed out after 30 seconds" in visible_markdown(compact)
    else:
        assert "incomplete" in compact.lower()


def test_compact_failed_validator_without_recorded_findings_still_has_action(tmp_path):
    summary, _ = example(tmp_path, result={"validator": "Scanner", "status": "failed", "passed": False})
    views.render_reports(tmp_path, summary, {})
    text = (tmp_path / "summary.md").read_text()
    assert "FAILED" in compact_header(text) and "skills/sample" in text
    assert "without a recorded reason" in text and "Suggested fix:" in text


def test_compact_configuration_error_is_not_green_and_has_artifact_fallback(tmp_path):
    summary = {
        "total": 0,
        "passed": 0,
        "failed": 0,
        "incomplete": 0,
        "overall_status": "error",
        "error": "No skill directories configured",
        "skills": [],
    }
    views.render_reports(tmp_path, summary, {"run_url": "https://github.com/o/r/actions/runs/1"})
    compact = (tmp_path / "summary.md").read_text()
    assert compact_header(compact) == "**Tier 1 scan result: ERROR**"
    assert "No skill directories configured" in compact and "configuration" in compact
    assert "artifact" in compact and "index.html" in compact
    assert "https://github.com/o/r/actions/runs/1" in compact
    assert "No blocking issues found" not in compact


def test_comment_retains_run_error_with_all_optional_header_fields(tmp_path):
    summary = {
        "total": 0,
        "passed": 0,
        "failed": 0,
        "incomplete": 0,
        "overall_status": "error",
        "error": "Deleted SKILL.md left tracked content",
        "skills": [],
    }
    context = {
        "run_url": "https://github.com/o/r/actions/runs/1",
        "report_url": "https://example.com/reports/1",
        "ci_conclusion": "failure",
        "commit": "a" * 40,
        "image": "ghcr.io/o/image@sha256:" + "b" * 64,
    }
    views.render_reports(tmp_path, summary, context)
    comment = visible_markdown((tmp_path / "pr-comment.md").read_text())
    assert summary["error"] in comment
    details = visible_markdown((tmp_path / "details.md").read_text())
    assert context["commit"] in details and context["image"] in details.replace("\u200b", "")
    assert context["commit"] not in comment and context["image"] not in comment
    assert "Run error" in comment


@pytest.mark.parametrize(
    "recorded,expected",
    [
        ("tools/check.py", "skills/sample/tools/check.py"),
        ("skills/sample/tools/check.py", "skills/sample/tools/check.py"),
        ("/workspace/skills/sample/tools/check.py", "skills/sample/tools/check.py"),
        ("../../outside", None),
        ("/tmp/foreign", None),
        ("file\nforged", None),
    ],
)
def test_relative_finding_locations_are_resolved_without_escaping_checkout(tmp_path, recorded, expected):
    assert views.finding_repository_path("skills/sample", recorded) == expected
    summary, _ = example(tmp_path, result=failed_result(path=recorded))
    views.render_reports(tmp_path, summary, {})
    markdown = visible_markdown((tmp_path / "summary.md").read_text())
    if expected:
        assert f"File: {expected}:2" in markdown
    else:
        assert "File:" not in markdown


def test_compact_lists_at_most_five_affected_skills_with_explicit_remainder(tmp_path):
    summary, data = example(tmp_path, result=failed_result())
    rows = []
    for index in range(1, 9):
        directory = tmp_path / f"skill-{index:03d}"
        directory.mkdir(exist_ok=True)
        child = copy.deepcopy(data)
        child["results"][0]["findings"][0]["file_path"] = f"skills/broken-{index}/SKILL.md"
        (directory / "skillevaluator-tier1.json").write_text(json.dumps(child))
        rows.append(
            {
                "path": f"skills/broken-{index}",
                "report_dir": directory.name,
                "overall_status": "failed",
                "passed": False,
            }
        )
    summary.update(total=8, passed=0, failed=8, incomplete=0, skills=rows)
    views.render_reports(tmp_path, summary, {})
    text = visible_markdown((tmp_path / "summary.md").read_text())
    for index in range(1, 6):
        assert f"skills/broken-{index}" in text
    for index in range(6, 9):
        assert f"skills/broken-{index}" not in text
    assert re.search(r"3 (?:more|additional) affected skills", text)
    assert "8 skills · 0 passed · 8 failed · 0 incomplete" in text


@pytest.mark.parametrize("kind", ["missing", "contradictory_child", "rollup_counts"])
def test_compact_unproven_evidence_never_claims_pass(tmp_path, kind):
    summary, data = example(tmp_path)
    if kind == "missing":
        (tmp_path / "skill-001/skillevaluator-tier1.json").unlink()
    elif kind == "contradictory_child":
        data["results"][0].update(status="failed", passed=False)
        (tmp_path / "skill-001/skillevaluator-tier1.json").write_text(json.dumps(data))
    else:
        summary.update(total=999, passed=999)
    views.render_reports(tmp_path, summary, {})
    text = (tmp_path / "summary.md").read_text()
    assert "PASSED" not in compact_header(text)
    assert "missing or inconsistent" in text
    assert "1 skills" in text
    if kind != "rollup_counts":
        assert "skills/sample" in text and "Problem:" in text and "Suggested fix:" in text


def test_compact_large_untrusted_fields_stay_bounded_and_useful(tmp_path):
    result = failed_result(
        message="Priority finding " + "界" * 50000,
        suggestion="Recorded fix " + "界" * 50000,
        path="skills/sample/SKILL.md" + "界" * 50000,
    )
    summary, data = example(tmp_path, result=result)
    data["workflow"]["stages"]["llm"]["reason"] = "Large reason " + "界" * 50000
    (tmp_path / "skill-001/skillevaluator-tier1.json").write_text(json.dumps(data))
    views.render_reports(tmp_path, summary, {"report_url": "https://example.com/full"})
    text = (tmp_path / "summary.md").read_text()
    assert len(text.encode()) <= 20 * 1024
    assert "FAILED" in compact_header(text) and "skills/sample" in text
    assert "Priority finding" in text and "Recorded fix" in text
    assert "https://example.com/full/index.html" in text
    assert "界" * 50000 not in text


def test_compact_adversarial_failure_fields_render_as_text(tmp_path):
    from markdown_it import MarkdownIt

    attack = "<script>alert(1)</script> [escape](javascript:alert(1)) @some-org/team |\n## forged"
    summary, _ = example(tmp_path, result=failed_result(message=attack, suggestion=attack, path=attack))
    views.render_reports(tmp_path, summary, {"run_url": "javascript:alert(1)", "report_url": "data:text/html,x"})
    text = (tmp_path / "summary.md").read_text()
    rendered = MarkdownIt("commonmark").enable("table").render(text)
    assert "<script" not in rendered and 'href="javascript:' not in rendered and 'href="data:' not in rendered
    assert "<h2>forged" not in rendered
    assert "@some-org" not in visible_markdown(text) and "@\u200bsome-org" in visible_markdown(text)
    assert "&lt;script&gt;" in text


@pytest.mark.parametrize(
    "field,folder,normalize",
    [("name", "valid-folder", True), ("description", "valid-folder", False), ("name", "INVALID FOLDER", False)],
)
def test_compact_concrete_name_replacement_requires_exact_known_evidence(tmp_path, field, folder, normalize):
    canonical = "Field 'name': Value error, Skill name 'INVALID NAME' must be in kebab-case format (lowercase letters, numbers, and hyphens only, starting with a letter)"
    result = failed_result(message=canonical, suggestion="Recorded suggestion")
    result["findings"][0]["metadata"] = {"field": field}
    summary, _ = example(tmp_path, result=result)
    summary["skills"][0]["path"] = "skills/" + folder
    views.render_reports(tmp_path, summary, {})
    text = visible_markdown((tmp_path / "summary.md").read_text())
    if normalize:
        assert "name: valid-folder" in text and "match the skill directory" in text
    else:
        assert "Recorded suggestion" in text and "Set name:" not in text


def test_compact_near_matching_name_error_keeps_recorded_fix(tmp_path):
    canonical = "Field 'name': Value error, Skill name 'INVALID NAME' must be in kebab-case format (lowercase letters, numbers, and hyphens only, starting with a letter)"
    result = failed_result(message=canonical + " Additional diagnostic context.", suggestion="Recorded suggestion")
    result["findings"][0]["metadata"] = {"field": "name"}
    summary, _ = example(tmp_path, result=result)
    views.render_reports(tmp_path, summary, {})
    text = visible_markdown((tmp_path / "summary.md").read_text())
    assert "Recorded suggestion" in text and "Set name:" not in text


def test_compact_percent_encoded_huge_hosted_url_preserves_headline_and_bound(tmp_path):
    summary, _ = example(tmp_path)
    hosted = "https://example.com/reports/" + "€" * 3000
    views.render_reports(tmp_path, summary, {"report_url": hosted})
    compact = (tmp_path / "summary.md").read_text()
    assert len(compact.encode()) <= 20 * 1024
    assert compact_header(compact) == "**Tier 1 scan result: PASSED**"
    assert "1 skills · 1 passed · 0 failed · 0 incomplete" in compact
    assert "Tier 2: Not run" in compact and "Tier 3: Not run" in compact
    assert "artifact" in compact.lower() and "index.html" in compact
    assert "%E2%82%AC" * 3000 not in compact


def test_compact_without_hosted_url_has_one_artifact_instruction(tmp_path):
    summary, _ = example(tmp_path)
    views.render_reports(tmp_path, summary, {})
    compact = (tmp_path / "summary.md").read_text()
    assert compact.lower().count("artifact") == 1
    assert "scan" in compact.lower() and "download" in compact.lower() and "index.html" in compact


def neutral_selection_summary():
    return {
        "total": 0,
        "passed": 0,
        "failed": 0,
        "incomplete": 0,
        "total_validators": 0,
        "total_errors": 0,
        "total_warnings": 0,
        "severity_counts": {"critical": 0, "high": 0, "medium": 0, "low": 0},
        "overall_status": "skipped",
        "overall_passed": False,
        "skills": [],
        "selection": {
            "mode": "changed",
            "base": "a" * 40,
            "base_ref": "c" * 40,
            "head": "b" * 40,
            "discovered": 2,
            "selected": [],
            "deleted": ["skills/removed"],
            "changed_files": 1,
        },
    }


def test_real_noop_marker_reports_no_scan_and_discloses_deleted_skills(tmp_path):
    summary = neutral_selection_summary()
    assert views.is_noop_summary(summary)
    views.render_reports(tmp_path, summary, {"llm_enabled": True})
    for name in ("summary.md", "details.md", "pr-comment.md"):
        text = (tmp_path / name).read_text()
        assert "NO SKILLS CHANGED" in text
        assert "Validation did not run" in text
        assert "Tier 1: Not run" in text and "LLM: Not run" in text
        assert "skills/removed" in text
        assert "PASSED" not in text
    assert not list(tmp_path.glob("skill-*"))


@pytest.mark.parametrize(
    "mutate",
    [
        lambda value: value.update(overall_passed=True),
        lambda value: value.update(error="Selection unavailable"),
        lambda value: value.update(total=1),
        lambda value: value.update(total_errors=False),
        lambda value: value["selection"].update(base="unknown"),
        lambda value: value["selection"].update(base_ref="unknown"),
        lambda value: value["selection"].update(selected=["skills/not-scanned"]),
        lambda value: value["selection"].update(deleted=["../outside"]),
        lambda value: value["selection"].update(changed_files=0),
        lambda value: value["selection"].update(discovered=0, deleted=[]),
    ],
)
def test_noop_requires_complete_consistent_selection_evidence(tmp_path, mutate):
    summary = neutral_selection_summary()
    mutate(summary)
    assert views.is_noop_summary(summary) is False
    views.render_reports(tmp_path, summary, {})
    assert "NO SKILLS CHANGED" not in (tmp_path / "summary.md").read_text()
    assert "PASSED" not in compact_header((tmp_path / "summary.md").read_text())


def test_quality_threshold_failure_is_not_replaced_by_advisory_finding(tmp_path):
    result = {
        "validator": "QUALITY",
        "status": "failed",
        "passed": False,
        "summary": {"errors": 1, "warnings": 1, "checks_performed": 1},
        "findings": [{"severity": "medium", "message": "Optional version missing", "suggestion": "Add a version"}],
        "legacy": {"errors": ["Quality score 98.2/100 is below the required 100/100"], "warnings": []},
        "gate_failures": ["Quality score 98.2/100 is below the required 100/100"],
    }
    summary, _ = example(tmp_path, result=result)
    views.render_reports(tmp_path, summary, {})
    for filename in ("summary.md", "details.md", "pr-comment.md"):
        assert "below the required 100/100" in visible_markdown((tmp_path / filename).read_text())
    assert "Optional version missing" not in visible_markdown((tmp_path / "summary.md").read_text())