Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

12.5 KB
Raw Blame
# SPDX-License-Identifier: Apache-2.0
"""Behavioral contracts for the native report and retained payload exports."""

import base64
import json
import re
from copy import deepcopy

from skillevaluator.evaluation.tier3_report import build_agent_eval_payload
from skillevaluator.models import ValidationResult
from skillevaluator.reporting.html import HTMLReporter


def _agent_result():
    result = ValidationResult(validator_name="AGENT_EVAL")
    result.metadata["agent_eval"] = build_agent_eval_payload(
        "demo",
        {"agent": {"overall_with_skill": 0.8, "overall_without_skill": 0.4}},
        use_llm_judge=False,
    )
    return result


def _embedded_payload(html):
    match = re.search(r'<script type="application/json" id="tier3-full"([^>]*)>(.*?)</script>', html, re.DOTALL)
    assert match
    raw = base64.b64decode(match[2]).decode() if 'data-encoding="base64"' in match[1] else match[2]
    return json.loads(raw)


def test_native_header_and_page_geometry_replace_sidebar_workspace():
    html = HTMLReporter(include_timestamp=False).render_all([ValidationResult(validator_name="Schema")])
    assert '<div class="logo-section">' in html
    assert ">SkillEvaluator</p>" in html
    assert '<aside class="report-sidebar"' not in html
    assert "body{font-family:ui-sans-serif" not in html
    assert ".container { max-width: 1200px; margin: 0 auto; padding: 2rem; }" in html
    assert "background: linear-gradient(135deg, var(--primary-color) 0%, var(--secondary-color) 100%);" in html
    assert re.search(r'<div class="dashboard(?:\s[^"]*)?">', html)
    assert "Validators Run" in html


def test_retained_canonical_export_is_distinct_from_fail_closed_display():
    result = _agent_result()
    raw = result.metadata["agent_eval"]
    raw.update(verdict="neutral", execution_status="failed", empty=[], nullable=None)
    raw["summary"]["execution_status"] = "failed"
    original = deepcopy(raw)
    result.add_error("Agent arm failed")
    html = HTMLReporter(include_timestamp=False).render_all([result])
    assert _embedded_payload(html) == original
    assert result.metadata["agent_eval"] == original
    assert HTMLReporter._tier3_report_data([result])["verdict"] != original["verdict"]
    assert "Tier 3 quality:" in html


def test_report_default_theme_is_dark_for_tier3_and_light_for_static():
    static = HTMLReporter(include_timestamp=False).render_all([ValidationResult(validator_name="Schema")])
    live = HTMLReporter(include_timestamp=False).render_all([_agent_result()])
    assert 'data-default-theme="light"' in static
    assert 'data-default-theme="dark"' in live
    assert "localStorage.getItem('skillevaluator-theme')" in live


def test_ci_report_suppresses_retained_payload_embedding():
    html = HTMLReporter(
        include_timestamp=False, ci_context={"canonical_json_url": "https://example.com/report.json"}
    ).render_all([_agent_result()])
    assert 'href="https://example.com/report.json"' in html
    assert 'id="report-data"' not in html
    assert 'id="tier3-full"' not in html
    assert "onclick=\"tier3DownloadJson('tier3-full'" not in html


def test_overview_links_to_full_insights_severity_cards_and_action_checks():
    result = _agent_result()
    payload = result.metadata["agent_eval"]
    payload.update(execution_status="succeeded", verdict="pass", expected_attempts=1, scored_attempts=1)
    payload["agents"]["agent"].update(execution_status="succeeded", expected_attempts=1, scored_attempts=1)
    payload["conclusions"] = [
        {"title": "Retained conclusion", "message": "Measured evidence", "severity": "pass", "source": "deterministic"}
    ]
    payload["recommendations"] = [
        {
            "title": "Retained action",
            "message": "Improve the instruction",
            "category": "Document",
            "source": "deterministic",
            "check": "Repeat the same case",
            "evidence_text": "case-one",
        }
    ]
    html = HTMLReporter(include_timestamp=False).render_all([result])
    overview = html.split('id="tier3-page-overview"', 1)[1].split('id="tier3-page-trials"', 1)[0]
    insights = html.split('id="tier3-page-suggestions"', 1)[1].split('id="tier3-page-provenance"', 1)[0]
    assert 'class="t3-sug-card ' not in overview
    assert "Retained action" in overview
    assert 'href="#tier3-page-suggestions"' in overview
    for section in (insights,):
        assert '<span class="t3-pill ok">pass</span>' in section
        assert '<span class="t3-sug-cat">Document</span>' in section
        assert "Check: Repeat the same case" in section
        assert "Evidence: case-one" in section


def test_native_tier2_intra_and_inter_tabs_keep_similarity_evidence():
    result = ValidationResult(validator_name="Similarity Check")
    result.add_success("catalog_compared", "Compared catalog entries", catalog_entry_count=5)
    html = HTMLReporter(include_timestamp=False).render_all([result])
    assert 'id="dedup-tab-intra"' in html
    assert 'id="dedup-tab-inter"' in html
    assert 'aria-controls="dedup-intra"' in html
    assert 'aria-controls="dedup-inter"' in html
    assert 'id="dedup-intra" class="dedup-subtab"' in html
    assert 'id="dedup-inter" class="dedup-subtab"' in html
    across = html.split('id="tier2-across"', 1)[1].split("</section>", 1)[0]
    assert "Catalog entries compared" in across
    assert "Compared catalog entries" in across


def test_failed_execution_actions_do_not_repeat_next_steps_or_expand_long_evidence():
    result = _agent_result()
    payload = result.metadata["agent_eval"]
    payload.update(execution_status="failed", verdict="fail")
    payload["next_steps"] = [
        {
            "kind": "Restore runner",
            "target": "evaluation",
            "problem": "A runner did not finish",
            "action": "Restore the runner and repeat the retained cases",
            "check": "Every case has a score",
            "evidence": "case-identity-" * 80,
            "strength": "hypothesis",
        }
    ]
    original = deepcopy(payload)
    result.add_error("Runner failed")
    html = HTMLReporter(include_timestamp=False).render_all([result])
    overview = html.split('id="tier3-page-overview"', 1)[1].split('id="tier3-page-trials"', 1)[0]
    assert re.sub(r"<[^>]+>", "", overview).count("Restore the runner and repeat the retained cases") == 1
    assert payload["next_steps"][0]["evidence"] not in overview
    assert payload["next_steps"][0]["problem"] not in overview
    for name, end in (("suggestions", "provenance"),):
        section = html.split(f'id="tier3-page-{name}"', 1)[1].split(f'id="tier3-page-{end}"', 1)[0]
        assert re.sub(r"<[^>]+>", "", section).count("Restore the runner and repeat the retained cases") == 1
        assert "Next steps for the run:" not in section
        assert "Scope: evaluation" in section
        assert "Strength: hypothesis" in section
        disclosure = re.search(
            r'<details class="t3-eval-evidence"([^>]*)>.*?<summary>Evidence</summary>(.*?)</details>',
            section,
            re.DOTALL,
        )
        assert disclosure and "open" not in disclosure[1]
        assert payload["next_steps"][0]["evidence"] in disclosure[2]
        problem = re.search(
            r'<details class="t3-eval-evidence"([^>]*)>\s*<summary>Recorded problem</summary>(.*?)</details>',
            section,
            re.DOTALL,
        )
        assert problem and "open" not in problem[1]
        assert payload["next_steps"][0]["problem"] in problem[2]
    assert _embedded_payload(html) == original
    assert payload == original


def test_only_fully_represented_failed_steps_are_suppressed():
    from skillevaluator.reporting import html as html_module

    represented = {
        "action": "Rerun",
        "target": "agent",
        "check": "Complete",
        "evidence": "case-a",
        "strength": "hypothesis",
        "agent": "a",
    }
    different_evidence = {**represented, "evidence": "case-b"}
    different_scope = {**represented, "target": "dataset"}
    different_agent = {**represented, "agent": "b"}
    different_problem = {**represented, "problem": "A distinct authored diagnosis"}
    steps = [represented, different_evidence, different_scope, different_agent, different_problem]
    view = {
        "next_steps": steps,
        "recommendations": [
            {
                "insight_kind": "execution_action",
                "message": "Rerun",
                "metric": "agent",
                "check": "Complete",
                "evidence_text": "case-a",
                "strength": "hypothesis",
                "agent": "a",
            }
        ],
    }
    assert html_module._unrepresented_insight_steps(view, failed=True) == steps[1:]
    assert html_module._unrepresented_insight_steps(view, failed=False) == steps


def test_failed_cards_preserve_distinct_problem_text_with_the_same_action_and_evidence():
    result = _agent_result()
    payload = result.metadata["agent_eval"]
    payload.update(execution_status="failed", verdict="fail")
    common = {
        "action": "Rerun the same case",
        "target": "evaluation",
        "check": "Every planned attempt is scored",
        "evidence": {"url": "https://example.test/shared-trace"},
        "agent": "agent",
        "case_id": "case-one",
        "condition": "with_skill",
    }
    problems = ["Runner exited; inspect <first> & retain the rest.", "Judge stopped; inspect the second cause."]
    payload["next_steps"] = [{**common, "problem": problem} for problem in problems]
    original = deepcopy(payload)
    html = HTMLReporter(include_timestamp=False).render_all([result])
    section = html.split('id="tier3-page-suggestions"', 1)[1].split('id="tier3-page-provenance"', 1)[0]
    from html import escape

    for problem in problems:
        assert escape(problem) in section
    assert "<first>" not in section
    assert section.count("<summary>Recorded problem</summary>") == 2
    assert section.count("https://example.test/shared-trace") == 2
    assert "Next steps for the run:" not in section
    assert _embedded_payload(html) == original
    assert payload == original


def test_historical_error_blob_problem_stays_in_diagnostics():
    result = _agent_result()
    payload = result.metadata["agent_eval"]
    errors = ["RunnerError: complete historical detail; retain semicolon", "JudgeError: complete cause"]
    payload.update(execution_status="failed", verdict="fail", execution_errors=errors)
    payload["next_steps"] = [{"action": "Rerun", "problem": "; ".join(errors)}]
    original = deepcopy(payload)
    html = HTMLReporter(include_timestamp=False).render_all([result])
    insights = html.split('id="tier3-page-suggestions"', 1)[1].split('id="tier3-page-provenance"', 1)[0]
    for error in errors:
        assert error not in insights
        assert error in html.split('id="tier3-execution-errors"', 1)[1]
    assert 'href="#tier3-execution-errors"' in insights
    assert _embedded_payload(html) == original


def test_failed_next_steps_keep_distinct_agent_case_and_condition_in_rendered_cards():
    result = _agent_result()
    payload = result.metadata["agent_eval"]
    payload.update(execution_status="failed", verdict="fail")
    common = {
        "kind": "Restore runner",
        "target": "evaluation",
        "problem": "Runner failed",
        "action": "Restore and rerun",
        "check": "Every attempt is scored",
        "evidence": "retained-log",
        "strength": "supported",
    }
    payload["next_steps"] = [
        {**common, "agent": "agent-one", "case_id": "case-one", "condition": "with_skill"},
        {**common, "agent": "agent-two", "case": "case-two", "condition": "without_skill"},
    ]
    original = deepcopy(payload)
    result.add_error("Runner failed")
    html = HTMLReporter(include_timestamp=False).render_all([result])
    overview = html.split('id="tier3-page-overview"', 1)[1].split('id="tier3-page-trials"', 1)[0]
    assert re.sub(r"<[^>]+>", "", overview).count("Restore and rerun") == 1
    for start, end in (("suggestions", "provenance"),):
        section = html.split(f'id="tier3-page-{start}"', 1)[1].split(f'id="tier3-page-{end}"', 1)[0]
        text = re.sub(r"<[^>]+>", "", section)
        assert text.count("Restore and rerun") == 2
        for association in (
            "Agent: agent-one",
            "Agent: agent-two",
            "Case: case-one",
            "Case: case-two",
            "Condition: with_skill",
            "Condition: without_skill",
        ):
            assert association in text
        assert "Next steps for the run:" not in text
    assert _embedded_payload(html) == original
    assert payload == original