Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
fix(tier3): mark never-activated skill evaluations inconclusive (#209)
Christopher Kevinefb5ab43f70d
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex da09de8f..559a9a9f 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -142,6 +142,15 @@ All notable changes to SkillEvaluator are documented in this file.### Fixed+- Tier 3 reports an inconclusive result when every observed positive with-skill+ trial has zero activation, and omits skill-lift comparisons for that agent.+ Activation counts are reported separately; negative controls and missing+ evidence do not imply that the skill was never loaded. The skill-execution+ score is zero when positive activation fails, so unrelated successful checks+ cannot inflate it to 0.5 (fixes #208). Benchmark cards and dashboard comparisons+ preserve this eligibility rule, and multi-agent HTML labels explain why lift+ is unavailable. Mixed-agent reports retain advice from the best eligible+ agent without attributing never-activated trials to skill quality.- Tier 3 now uses one versioned, equal-weight five-dimension overall-scorepolicy across Harbor rewards, pass@k, Skill Lift, reports, comparisons, andbest-agent selection. Previously, some paths averaged all six evaluatorsdiff --git a/docs/tier3-live-evaluation.mdx b/docs/tier3-live-evaluation.mdxindex 679e9d2d..9a460193 100644--- a/docs/tier3-live-evaluation.mdx+++ b/docs/tier3-live-evaluation.mdx@@ -39,6 +39,15 @@ human-readable dimensions:In `custom_only` mode, your own grader defines the score instead — see[Custom Graders & Tasks](custom-graders.mdx).+If every positive with-skill trial has a confirmed activation score of zero,+the result is **INCONCLUSIVE: skill never activated**. Tier 3 keeps the arm+scores for inspection but omits lift comparisons and skill-quality advice for+that agent. A positive case with failed activation receives a `skill_execution`+score of zero, even if unrelated commands succeeded. Activation counts exclude+negative controls; missing or unsupported activation evidence remains unknown.+When at least one positive trial activates the skill, the usual lift calculation+applies. This also makes activation failures visible with `--skip-baseline`.+## Before you startEvery row below is checkable in seconds; the linked guide covers setup indiff --git a/src/skillevaluator/evaluation/tier3_report.py b/src/skillevaluator/evaluation/tier3_report.pyindex f1267fcd..358eef65 100644--- a/src/skillevaluator/evaluation/tier3_report.py+++ b/src/skillevaluator/evaluation/tier3_report.py@@ -47,6 +47,26 @@ from skillevaluator.source_identity import resolve_evaluated_sourceVERDICT_PASS = "pass"VERDICT_FAIL = "fail"VERDICT_NEUTRAL = "neutral"+VERDICT_INCONCLUSIVE = "inconclusive"+++def _never_activated(agent: dict[str, Any]) -> bool:+ activation = agent.get("skill_activation")+ return isinstance(activation, dict) and activation.get("status") == "never_activated"+++def _activation_summary(info: dict[str, Any]) -> dict[str, Any]:+ from skillevaluator.tier3.harbor.activation import summarize_skill_activation++ # A retained summary covers all trials even when report loading is bounded.+ summary = info.get("skill_activation")+ if isinstance(summary, dict):+ return dict(summary)+ summary = summarize_skill_activation(info.get("rewards") or [])+ if info.get("rewards_complete") is False and summary.get("status") == "never_activated":+ summary = dict(summary, status="unknown", reason="Activation evidence is incomplete.")+ return summary+_AGENT_EVAL_VALIDATOR = "AGENT_EVAL"_AGENT_EVAL_DESCRIPTION = "Tier 3: Live Agent Evaluation (Harbor)"@@ -503,7 +523,6 @@ def agent_eval_result_from_directory(env_mode=env_mode,runtime_seconds=_runtime_seconds(engine_result),harbor_viewer=_harbor_viewer_from_engine_result(engine_result),- suggestions_v2=_load_suggestions_v2(run_dir, agents),run_dir=run_dir,comparison=_read_comparison(run_dir),evaluated_at=evaluated_at or _evaluated_at_from_run(run_dir, engine_result),@@ -725,6 +744,11 @@ def build_agent_eval_payload(report_budget,)best = agent_payloads.get(best_agent, {})+ if suggestions_v2 is None and run_dir is not None:+ # Findings are per-agent evidence. Resolve the canonical best agent+ # before loading them, so stale advice from a different agent cannot+ # become the run's recommendations.+ suggestions_v2 = _load_suggestions_v2(run_dir, {best_agent: agents[best_agent]} if best_agent else {})execution_errors = list(dict.fromkeys(@@ -787,11 +811,17 @@ def build_agent_eval_payload(harbor_viewer,)best_dimensions = best.get("dimensions", [])- evidence_links = list(harbor_summary.get("evidence_links") or [])+ # The full viewer summary remains diagnostic. Advice must only link to+ # agents whose trials were eligible for its quality assessment; identical+ # case IDs across agents do not make their traces interchangeable.+ advice_agents = {name for name, agent in agent_payloads.items() if not _never_activated(agent)}+ evidence_links = [link for link in harbor_summary.get("evidence_links") or [] if link.get("agent") in advice_agents]+ best_evidence_links = [link for link in evidence_links if link.get("agent") == best_agent]summary = {"schema_version": _SCHEMA_VERSION,"verdict": verdict,+ "skill_activation": best.get("skill_activation", {}),"skill_name": skill_name,"best_agent": best_agent,"agents_run": list(agent_payloads.keys()),@@ -830,9 +860,20 @@ def build_agent_eval_payload(failure_message = "; ".join(execution_errors) or "Tier 3 evaluation did not produce a complete scored run"deterministic_conclusions = [{"severity": "fail", "title": "Evaluation incomplete", "message": failure_message}]deterministic_suggestions = [failure_message]+ elif verdict == VERDICT_INCONCLUSIVE:+ deterministic_conclusions = [+ {+ "severity": "warn",+ "title": "Inconclusive: skill never activated",+ "message": "The skill was never activated in any positive with-skill trial. Scores are diagnostic only; no lift verdict was computed.",+ }+ ]+ deterministic_suggestions = ["Check skill loading and agent support, then rerun the evaluation."]else:deterministic_conclusions = _build_conclusions(- agent_payloads, best_dimensions, pass_threshold=_pass_threshold_from_policy(policy)+ {name: agent for name, agent in agent_payloads.items() if not _never_activated(agent)},+ best_dimensions,+ pass_threshold=_pass_threshold_from_policy(policy),)deterministic_suggestions = _suggestions_for_dimensions(best_dimensions)recommendations = _attach_harbor_evidence_to_recommendations(@@ -846,7 +887,7 @@ def build_agent_eval_payload(}for text in deterministic_suggestions],- evidence_links,+ best_evidence_links,)payload = {@@ -854,6 +895,7 @@ def build_agent_eval_payload("summary": summary,"skill_name": skill_name,"verdict": verdict,+ "skill_activation": summary["skill_activation"],"best_agent": best_agent,"agents_run": list(agent_payloads.keys()),"environment": env_mode,@@ -888,7 +930,10 @@ def build_agent_eval_payload("conclusions": list(deterministic_conclusions),"recommendations": recommendations,"suggestions": list(deterministic_suggestions),- "suggestions_v2": _attach_harbor_evidence_to_suggestions_v2(suggestions_v2 or [], evidence_links),+ "suggestions_v2": _attach_harbor_evidence_to_suggestions_v2(+ [] if _never_activated(best) else (suggestions_v2 or []),+ best_evidence_links,+ ),"metric_ids": metric_ids,"supported_metric_ids": list(AGENT_EVAL_EVALUATORS),"metric_labels": metric_labels,@@ -912,20 +957,25 @@ def build_agent_eval_payload(payload,deterministic_conclusions=deterministic_conclusions,deterministic_suggestions=deterministic_suggestions,- use_llm_judge=use_llm_judge and overall_score is not None,+ use_llm_judge=use_llm_judge and overall_score is not None and not _never_activated(best),)if evidence_links:payload["conclusions"] = _attach_harbor_evidence_to_conclusions(payload.get("conclusions") or [],evidence_links,)- payload["recommendations"] = _attach_harbor_evidence_to_recommendations(- payload.get("recommendations") or [],- evidence_links,- )+ linked_recommendations = []+ for recommendation in payload.get("recommendations") or []:+ links = (+ best_evidence_links+ if isinstance(recommendation, dict) and recommendation.get("source") == "deterministic"+ else evidence_links+ )+ linked_recommendations.extend(_attach_harbor_evidence_to_recommendations([recommendation], links))+ payload["recommendations"] = linked_recommendationspayload["suggestions_v2"] = _attach_harbor_evidence_to_suggestions_v2(payload.get("suggestions_v2") or [],- evidence_links,+ best_evidence_links,)payload = _sanitize_json_numbers(payload)_enforce_report_payload_budget(payload, report_budget, enforce_size=enforce_embedded_size)@@ -948,8 +998,25 @@ def _layer_llm_insights(try:from skillevaluator.evaluation.insights_judge import build_insights+ # In mixed runs, keep never-activated trials available as diagnostics but+ # exclude them from skill-quality advice and its grounding evidence.+ eligible_agents = {+ name: agent+ for name, agent in (payload.get("agents") or {}).items()+ if isinstance(agent, dict) and not _never_activated(agent)+ }+ insight_payload = {+ **payload,+ "agents": eligible_agents,+ "agents_run": list(eligible_agents),+ "trials": [+ trial+ for trial in payload.get("trials") or []+ if isinstance(trial, dict) and trial.get("agent") in eligible_agents+ ],+ }extra = build_insights(- payload,+ insight_payload,deterministic={"conclusions": deterministic_conclusions,"suggestions": deterministic_suggestions,@@ -1084,11 +1151,7 @@ def _compact_agent_conditions(agent: dict[str, Any]) -> bool:def bounded_count(condition: dict[str, Any], key: str) -> int:value = condition.get(key)- return (- value- if isinstance(value, int) and not isinstance(value, bool) and 0 <= value <= 2**63 - 1- else 0- )+ return value if isinstance(value, int) and not isinstance(value, bool) and 0 <= value <= 2**63 - 1 else 0compact: dict[str, dict[str, Any]] = {}canonical_names = {"with_skill", "without_skill"}@@ -1509,11 +1572,15 @@ def _build_agent(with_scores = info.get("with_skill") or {}without_scores = info.get("without_skill") or {}+ skill_activation = _activation_summary(info)+ never_activated = skill_activation.get("status") == "never_activated"condition_policies = {value.strip() if isinstance(value, str) and value.strip() else score_policyfor value in (info.get("score_policy_with_skill"), info.get("score_policy_without_skill"))}- comparable_scores = set(with_metrics) == set(baseline_metrics) and len(condition_policies) == 1+ comparable_scores = (+ not never_activated and set(with_metrics) == set(baseline_metrics) and len(condition_policies) == 1+ )lift_data = info.get("lift") or {}with_quality_available = _condition_quality_available(info, "with_skill")baseline_quality_available = _condition_quality_available(info, "without_skill")@@ -1529,6 +1596,16 @@ def _build_agent(info.get("dimensions_with_skill") or {},info.get("dimensions_without_skill") or {},)+ if never_activated:+ for evaluator in evaluators.values():+ evaluator["lift"] = None+ for dimension in dimensions:+ dimension.update(+ lift=None,+ verdict=VERDICT_INCONCLUSIVE,+ explanation="Skill never activated; scores are diagnostic only.",+ reasoning_bullets=["Skill never activated; no skill-effectiveness verdict was computed."],+ )with_dimension_values = [d["with_skill"] for d in dimensions]baseline_dimension_values = [d["baseline"] for d in dimensions]overall_ws = None@@ -1601,7 +1678,8 @@ def _build_agent(trials = _normalize_trials(info.get("rewards") or [], with_metrics)baseline_trials = _normalize_trials(info.get("rewards_baseline") or [], baseline_metrics)- _attach_baseline_pairs(trials, baseline_trials, with_metrics, comparable_overall=comparable_scores)+ if not never_activated:+ _attach_baseline_pairs(trials, baseline_trials, with_metrics, comparable_overall=comparable_scores)from skillevaluator.tier3.harbor.resource_usage import resource_comparisonresources = resource_comparison(@@ -1620,6 +1698,8 @@ def _build_agent(return {"name": name,+ "skill_activation": skill_activation,+ "verdict": VERDICT_INCONCLUSIVE if never_activated else None,"model": model,"execution_status": (info.get("execution_status")@@ -1676,6 +1756,9 @@ def _attach_agent_report_details(custom_lift=custom_lift if isinstance(custom_lift, dict) else {},report_budget=report_budget,)+ if _never_activated(agent_payload):+ for card in agent_payload["evaluator_cards"]:+ card["lift"] = Nonedef _build_evaluators(@@ -2958,6 +3041,8 @@ def _agent_quality_verdict(agent: dict[str, Any]) -> str:"""Classify one supported agent by the canonical dimension gate."""if agent.get("execution_status") != "succeeded":return VERDICT_NEUTRAL+ if _never_activated(agent):+ return VERDICT_INCONCLUSIVEdimensions = {str(dimension.get("id")): dimension@@ -2989,6 +3074,8 @@ def _overall_verdict_from_agents(agents: dict[str, dict[str, Any]]) -> str:return VERDICT_PASSif any(verdict == VERDICT_NEUTRAL for verdict in verdicts) or not verdicts:return VERDICT_NEUTRAL+ if all(verdict == VERDICT_INCONCLUSIVE for verdict in verdicts):+ return VERDICT_INCONCLUSIVEreturn VERDICT_FAIL@@ -3038,6 +3125,8 @@ def _canonical_agent_rank_from_info(def _pick_best_agent(agents: dict[str, dict[str, Any]]) -> str:eligible = {name: rank for name, agent in agents.items() if (rank := _canonical_agent_rank(agent)) is not None}+ activated = {name: rank for name, rank in eligible.items() if not _never_activated(agents[name])}+ eligible = activated or eligibleif not eligible:return ""if len(eligible) == 1:@@ -3200,7 +3289,9 @@ def _evaluated_at_from_run(run_dir: Path, engine_result: dict[str, Any] | None)def _load_suggestions_v2(run_dir: Path, agents: dict[str, dict[str, Any]]) -> list[dict[str, Any]]:"""Read evidence-backed suggestions from the best agent's findings.json."""suggestions: list[dict[str, Any]] = []- for agent_name in agents:+ for agent_name, info in agents.items():+ if _activation_summary(info).get("status") == "never_activated":+ continuefindings_file = run_dir / agent_name / "findings.json"if not findings_file.exists():continuediff --git a/src/skillevaluator/reporting/base.py b/src/skillevaluator/reporting/base.pyindex e43f200a..42322d72 100644--- a/src/skillevaluator/reporting/base.py+++ b/src/skillevaluator/reporting/base.py@@ -681,8 +681,8 @@ def _agent_eval_truth_state(payload: dict[str, Any] | None) -> tuple[str, str, bverdict, verdict_consistent = collect("verdict",- frozenset({"pass", "neutral", "fail", "skipped", "incomplete"}),- {"pass": 0, "skipped": 1, "neutral": 2, "incomplete": 3, "fail": 4},+ frozenset({"pass", "neutral", "fail", "skipped", "incomplete", "inconclusive"}),+ {"pass": 0, "skipped": 1, "neutral": 2, "inconclusive": 3, "incomplete": 3, "fail": 4},)execution_status, execution_consistent = collect("execution_status",@@ -1222,6 +1222,10 @@ def agent_eval_dimension_verdict(verdicts: list[str] = []has_partial_evidence = Falsefor agent in supported_agents:+ activation = agent.get("skill_activation")+ if isinstance(activation, dict) and activation.get("status") == "never_activated":+ verdicts.append("inconclusive")+ continuescores = _agent_eval_dimension_scores(agent, require_baseline=require_baseline)if scores is None:has_partial_evidence = True@@ -1239,6 +1243,8 @@ def agent_eval_dimension_verdict(return Noneif "neutral" in verdicts:return "neutral"+ if all(verdict == "inconclusive" for verdict in verdicts):+ return "inconclusive"return "fail"@@ -1293,8 +1299,23 @@ def _agent_eval_publication_aggregates_consistent(return Falseeligible_agents: list[tuple[str, float, float]] = []+ has_activation_eligible_agent = any(+ not (+ isinstance(agent.get("skill_activation"), dict)+ and agent["skill_activation"].get("status") == "never_activated"+ )+ and _agent_eval_safe_text(agent.get("execution_status")).casefold() == "succeeded"+ and _agent_eval_finite_score(agent.get("with_skill")) is not None+ for agent in agents.values()+ )for name in canonical_agent_names:agent = agents[name]+ if (+ has_activation_eligible_agent+ and isinstance(agent.get("skill_activation"), dict)+ and agent["skill_activation"].get("status") == "never_activated"+ ):+ continueif _agent_eval_safe_text(agent.get("execution_status")).casefold() != "succeeded":continuescore = _agent_eval_finite_score(agent.get("with_skill"))@@ -1653,7 +1674,7 @@ def _agent_eval_candidate_rank(if max(payload_attempts, _agent_eval_count(agent.get("scored_attempts"))) > 0:strong_agent_evidence = True- valid_verdict = truth_consistent and verdict in {"pass", "neutral", "fail"}+ valid_verdict = truth_consistent and verdict in {"pass", "neutral", "fail", "inconclusive"}dimension_verdict = agent_eval_dimension_verdict(payload)conservative_outcome_rank = max(({"pass": 0, "neutral": 1, "fail": 3}.get(candidate, -1) for candidate in (verdict, dimension_verdict)),@@ -2156,6 +2177,15 @@ def assess_tier3_evidence(selected_payload,"Tier 3 evidence records a failing verdict.",)+ if raw_verdict == "inconclusive" and dimension_verdict == "inconclusive" and execution_status == "succeeded":+ return Tier3EvidenceAssessment(+ "inconclusive",+ False,+ execution_status,+ raw_verdict,+ selected_payload,+ "Inconclusive: skill never activated. Scores are diagnostic only; no lift verdict was computed.",+ )if execution_status == "incomplete":return Tier3EvidenceAssessment("incomplete",diff --git a/src/skillevaluator/reporting/benchmark.py b/src/skillevaluator/reporting/benchmark.pyindex 44eb7534..b0992716 100644--- a/src/skillevaluator/reporting/benchmark.py+++ b/src/skillevaluator/reporting/benchmark.py@@ -421,6 +421,7 @@ class BenchmarkReporter(ReporterBase):_score_transition_values(_number(agent.get("baseline")),_number(agent.get("with_skill", agent.get("overall_score"))),+ skill_never_activated=_mapping(agent.get("skill_activation")).get("status") == "never_activated",)for agent in agents.values())@@ -433,7 +434,14 @@ class BenchmarkReporter(ReporterBase):dimension = _agent_dimension(agent, dim_id)if dimension and dimension.get("partial"):has_partial = True- row.append(_score_transition(dimension))+ row.append(+ _score_transition(+ dimension,+ skill_never_activated=(+ _mapping(agent.get("skill_activation")).get("status") == "never_activated"+ ),+ )+ )lines.append("| " + " | ".join(_md_cell(value, private_labels) for value in row) + " |")lines.extend(@@ -1060,18 +1068,22 @@ def _agent_dimension(agent: dict[str, Any], dim_id: str) -> dict[str, Any] | Nonreturn None-def _score_transition(dimension: dict[str, Any] | None) -> str:+def _score_transition(dimension: dict[str, Any] | None, *, skill_never_activated: bool = False) -> str:if not dimension:return "Not available"baseline = _number(dimension.get("baseline"))score = _number(dimension.get("with_skill", dimension.get("score")))- return _score_transition_values(baseline, score)+ return _score_transition_values(baseline, score, skill_never_activated=skill_never_activated)-def _score_transition_values(baseline: float | None, score: float | None) -> str:+def _score_transition_values(+ baseline: float | None, score: float | None, *, skill_never_activated: bool = False+) -> str:if score is None:return "Not available"skill_label = f"{score:.0%}"+ if skill_never_activated:+ return f"{skill_label} — INCONCLUSIVE: skill never activated; uplift unavailable"if baseline is None:return f"{skill_label} — baseline not run; uplift unavailable"delta = score - baseline@@ -1182,6 +1194,8 @@ def _tier_status(tier3 = assess_tier3_evidence(results, ae, expected_skill_name=expected_skill_name)if tier3.status == "fail":return "FAIL", f"{len(_agents(ae))} agent(s); {_dataset_summary(ae)['total_tasks']} task(s)"+ if tier3.status == "inconclusive":+ return "INCONCLUSIVE", tier3.reason or "Skill never activated."if not tier3.evidence_complete:evidence = tier3.reason or ("Required Tier 3 evidence is missing"diff --git a/src/skillevaluator/reporting/cli.py b/src/skillevaluator/reporting/cli.pyindex 2682c1d0..89de3fea 100644--- a/src/skillevaluator/reporting/cli.py+++ b/src/skillevaluator/reporting/cli.py@@ -344,6 +344,10 @@ class CLIReporter(ReporterBase):composite_text = f"{composite:+.2f}" if isinstance(composite, int | float) else "N/A"verdict_text = escape_markup(str(verdict).upper())console.print(f"\n [{vc}]Verdict: {verdict_text} (composite lift = {composite_text})[/{vc}]")+ if verdict == "inconclusive":+ console.print(+ " [yellow]Skill never activated. Scores are diagnostic only; no lift verdict was computed.[/yellow]"+ )if runtime:console.print(f" [dim]Runtime: {runtime:.1f}s[/dim]")harbor_viewer = normalize_harbor_viewer_for_display(agent_eval)@@ -377,12 +381,16 @@ class CLIReporter(ReporterBase):ws = scores.get("with_skill", 0.0)bl = scores.get("baseline", 0.0)lift = scores.get("lift", 0.0)- lc = "green" if lift > 0.01 else ("red" if lift < -0.01 else "dim")+ lc = (+ "green"+ if lift is not None and lift > 0.01+ else ("red" if lift is not None and lift < -0.01 else "dim")+ )table.add_row(escape_markup(str(name).replace("_", " ").title()),f"{ws:.2f}",f"{bl:.2f}",- f"[{lc}]{lift:+.2f}[/{lc}]",+ f"[{lc}]{lift:+.2f}[/{lc}]" if lift is not None else "N/A",)console.print(table)diff --git a/src/skillevaluator/reporting/console_ui.py b/src/skillevaluator/reporting/console_ui.pyindex 3375f0ea..230d11bd 100644--- a/src/skillevaluator/reporting/console_ui.py+++ b/src/skillevaluator/reporting/console_ui.py@@ -849,6 +849,8 @@ def summarize_tier3(result: ValidationResult) -> tuple[bool, bool, list[TierRow]# a fabricated N/N ratio here would overstate what we measured.agent_segments.append((f" eval cases {len(case_list)}", MUTED))rows = [TierRow("agent", agent_segments)]+ if payload.get("verdict") == "inconclusive" or summary.get("verdict") == "inconclusive":+ rows.append(TierRow("verdict", [("INCONCLUSIVE: skill never activated", TEXT)]))lift = _tier3_finite_number(summary.get("overall_lift"))with_score = _tier3_finite_number(summary.get("overall_score"))if (diff --git a/src/skillevaluator/reporting/dashboard_data.py b/src/skillevaluator/reporting/dashboard_data.pyindex 98e430bf..0cea8bca 100644--- a/src/skillevaluator/reporting/dashboard_data.py+++ b/src/skillevaluator/reporting/dashboard_data.py@@ -417,6 +417,7 @@ def parse_dashboard_report(payload: Any, source: str) -> DashboardData:"_report_index": report_index,"_agent_key": str(name),"_trial_keys": keys,+ "_skill_activation_status": _dict(agent.get("skill_activation")).get("status", "unknown"),"_dataset_digest": report.get("dataset_digest") or summary.get("dataset_digest"),"_attempt_policy": report.get("attempt_policy"),"_expected_usage_trials": max(@@ -508,6 +509,12 @@ def comparison_rows(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:)status = "comparable" if paired else "incomplete or mismatched coverage"reason = "" if paired else _ineligible_reason(with_row, without_row)+ if paired and with_row.get("_skill_activation_status") == "never_activated":+ # Absolute scores and usage remain diagnostic, but a complete+ # run that never exercised its skill cannot establish lift.+ paired = False+ status = "inconclusive"+ reason = "Inconclusive: skill never activated in with-skill trials"result["comparison_status"] = statusresult["comparison_reason"] = reasonresult["coverage"] = (diff --git a/src/skillevaluator/reporting/html.py b/src/skillevaluator/reporting/html.pyindex 82182d34..92a9b0d8 100644--- a/src/skillevaluator/reporting/html.py+++ b/src/skillevaluator/reporting/html.py@@ -790,6 +790,8 @@ def _sanitize_tier3_evaluator_card(value: dict[str, Any]) -> dict[str, Any] | Nodef _sanitize_tier3_agent(value: dict[str, Any]) -> dict[str, Any]:agent = dict(value)+ activation = agent.get("skill_activation")+ agent["skill_activation"] = dict(activation) if isinstance(activation, dict) else {}for key in ("model", "model_name", "llm_model", "display_name", "label"):if key not in agent:continue@@ -2203,7 +2205,7 @@ class HTMLReporter(ReporterBase):) -> bool:if agent_eval_publication_evidence_complete(candidate):return True- if payload_verdict(candidate) not in {"pass", "neutral", "fail"}:+ if payload_verdict(candidate) not in {"pass", "neutral", "fail", "inconclusive"}:return Falseexecution_status = payload_execution_status(candidate)if execution_status == "succeeded":diff --git a/src/skillevaluator/reporting/markdown.py b/src/skillevaluator/reporting/markdown.pyindex 3e968346..40164bf0 100644--- a/src/skillevaluator/reporting/markdown.py+++ b/src/skillevaluator/reporting/markdown.py@@ -309,6 +309,8 @@ class MarkdownReporter(ReporterBase):lines.append("")composite_text = _format_report_number(composite, signed=True)lines.append(f"**Verdict:** {verdict} (composite lift = {composite_text})")+ if verdict == "INCONCLUSIVE":+ lines.append("Skill never activated. Scores are diagnostic only; no lift verdict was computed.")runtime_number = _finite_report_number(runtime)runtime_text = f"{runtime_number:.1f}s" if runtime_number is not None else "N/A"lines.append(f"**Runtime:** {runtime_text}")diff --git a/src/skillevaluator/reporting/templates/report.html.j2 b/src/skillevaluator/reporting/templates/report.html.j2index 269a3f60..fb47ce7f 100644--- a/src/skillevaluator/reporting/templates/report.html.j2+++ b/src/skillevaluator/reporting/templates/report.html.j2@@ -2329,6 +2329,9 @@{% if tier3_is_skipped %}<strong>Live evaluation skipped:</strong> {{ tier3_skip_message }}Tier 3 is advisory and does not block required validation.+ {% elif tier3_verdict == 'inconclusive' %}+ <strong>Inconclusive: skill never activated.</strong>+ Scores are diagnostic only; no lift verdict was computed. Check skill loading and agent support, then rerun the evaluation.{% else %}<strong>Verdict at a glance:</strong>{{ tier3_verdict | upper }} with an overall score of@@ -2507,7 +2510,7 @@<span class="t3-mad-score" style="color:{{ d_color }};">{{ "%.2f" | format(d_score) if d_score is not none else "N/A" }}</span>{% if d_score is not none %}<div class="t3-mad-bar"><span style="width:{{ (d_score * 100) | round(0) }}%; background:{{ d_color }};"></span></div>{% endif %}</div>- <span class="t3-mad-lift" style="color:{{ lift_color }};">{% if d_lift is not none %}{{ "%+.2f" | format(d_lift) }} lift{% elif d_score is none and d.baseline is not none %}with-skill unavailable{% elif d_score is none %}N/A lift{% else %}no baseline lift{% endif %}</span>+ <span class="t3-mad-lift" style="color:{{ lift_color }};">{% if _agent.get('skill_activation', {}).get('status') == 'never_activated' %}inconclusive: skill never activated{% elif d_lift is not none %}{{ "%+.2f" | format(d_lift) }} lift{% elif d_score is none and d.baseline is not none %}with-skill unavailable{% elif d_score is none %}N/A lift{% else %}no baseline lift{% endif %}</span></div></td>{% else %}@@ -2530,7 +2533,7 @@<span class="t3-mad-score" style="color:{{ ws_color }};">{{ "%.2f" | format(ws) if ws is not none else "N/A" }}</span>{% if ws is not none %}<div class="t3-mad-bar"><span style="width:{{ (ws * 100) | round(0) }}%; background:{{ ws_color }};"></span></div>{% endif %}</div>- <span class="t3-mad-lift" style="color:{{ lf_color }};">{% if lf is not none %}{{ "%+.2f" | format(lf) }} lift{% elif ws is none and bl is not none %}with-skill unavailable{% elif ws is none %}N/A lift{% else %}no baseline lift{% endif %}</span>+ <span class="t3-mad-lift" style="color:{{ lf_color }};">{% if agent.get('skill_activation', {}).get('status') == 'never_activated' %}inconclusive: skill never activated{% elif lf is not none %}{{ "%+.2f" | format(lf) }} lift{% elif ws is none and bl is not none %}with-skill unavailable{% elif ws is none %}N/A lift{% else %}no baseline lift{% endif %}</span></div></td>{% endfor %}@@ -2566,7 +2569,7 @@<div class="t3-agent-name">{{ name }}</div><div class="t3-agent-score" style="color:{{ score_color }};">{{ "%.2f" | format(with_skill) if with_skill is not none else "N/A" }}</div><div style="color:{{ card_lift_color }};font-size:0.85rem;">- {% if lift is not none %}{{ "%+.2f" | format(lift) }} lift{% elif with_skill is none and agent.baseline is not none %}with-skill unavailable{% elif with_skill is none %}N/A lift{% else %}no baseline{% endif %}+ {% if agent.get('skill_activation', {}).get('status') == 'never_activated' %}inconclusive: skill never activated{% elif lift is not none %}{{ "%+.2f" | format(lift) }} lift{% elif with_skill is none and agent.baseline is not none %}with-skill unavailable{% elif with_skill is none %}N/A lift{% else %}no baseline{% endif %}</div><div style="color:var(--text-secondary);font-size:0.75rem;margin-top:0.25rem;">{{ agent.num_trials or 0 }} trials</div>{% if with_skill is not none %}<div class="t3-mini-bar"><span style="width:{{ (with_skill * 100) | round(0) }}%; background:{{ score_color }};"></span></div>{% endif %}@@ -3067,7 +3070,7 @@<div class="t3-kpi-sub">{% if agent.lift >= tier3_lift_pass_threshold %}positive{% elif agent.lift <= tier3_lift_fail_threshold %}regression{% else %}neutral{% endif %}</div>{% else %}<div class="t3-kpi-value" style="color:var(--text-secondary);">N/A</div>- <div class="t3-kpi-sub">{% if agent.with_skill is none and agent.baseline is not none %}with-skill unavailable{% elif agent.with_skill is none %}N/A lift{% else %}no baseline{% endif %}</div>+ <div class="t3-kpi-sub">{% if agent.get('skill_activation', {}).get('status') == 'never_activated' %}inconclusive: skill never activated{% elif agent.with_skill is none and agent.baseline is not none %}with-skill unavailable{% elif agent.with_skill is none %}N/A lift{% else %}no baseline{% endif %}</div>{% endif %}</div><div class="t3-kpi-card">diff --git a/src/skillevaluator/tier3/eval_core/checks.py b/src/skillevaluator/tier3/eval_core/checks.pyindex 381b77ea..01938d04 100644--- a/src/skillevaluator/tier3/eval_core/checks.py+++ b/src/skillevaluator/tier3/eval_core/checks.py@@ -5630,9 +5630,11 @@ def score_skill_execution(skill_tool_names: list[str] | None = None,acceptable_skills: Any = None,) -> dict[str, Any]:- """Compute the ``skill_execution`` eval score (average of sub-checks).+ """Compute execution credit, requiring activation for positive cases.- Returns ``{"score": float, "details": dict}`` with per-check results.+ Activated or unresolved cases retain the average of sub-checks. A definite+ activation failure receives no execution credit, while retaining all check+ details for diagnosis. Negative and unlabeled cases use their own rules."""if should_trigger is None:return {"score": 1.0, "details": {"message": "No expected_skill -- skipped"}}@@ -5658,9 +5660,6 @@ def score_skill_execution(if not expected_skill:return {"score": 1.0, "details": {"message": "No expected_skill -- skipped"}}- if not tool_calls:- return {"score": 0.0, "details": {"message": "No tool calls in trajectory"}}-checks: dict[str, dict[str, Any]] = {}scores: list[float] = []@@ -5673,6 +5672,9 @@ def score_skill_execution(checks["activation"] = rscores.append(r["score"])+ if not tool_calls:+ return {"score": 0.0, "details": {"message": "No tool calls in trajectory", **checks}}+r = check_script_execution(tool_calls, expected_script)checks["script_execution"] = rscores.append(r["score"])@@ -5689,6 +5691,13 @@ def score_skill_execution(checks["error_recovery"] = rscores.append(r["score"])+ # Nonapplicable script/recovery checks must not create execution credit+ # when the expected skill was never activated. Unknown activation remains+ # distinct from a demonstrated failure.+ activation = checks["activation"]+ if activation["passed"] is False and activation["score"] == 0.0:+ return {"score": 0.0, "details": checks}+avg = sum(scores) / len(scores) if scores else 0.0return {"score": round(avg, 4), "details": checks}diff --git a/src/skillevaluator/tier3/harbor/activation.py b/src/skillevaluator/tier3/harbor/activation.pynew file mode 100644index 00000000..b23f5501--- /dev/null+++ b/src/skillevaluator/tier3/harbor/activation.py@@ -0,0 +1,93 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++"""Summarize observed skill activation independently of execution scores."""++from __future__ import annotations++import math+from collections.abc import Iterable, Mapping+from typing import Any++from skillevaluator.tier3.harbor.metrics import (+ DEFAULT_METRIC_SET,+ LEGACY_METRIC_SET,+ metric_set_for_reward,+ metric_value,+)+++def _activation_state(reward: Mapping[str, Any]) -> str:+ details = reward.get("details")+ execution = details.get("skill_execution") if isinstance(details, Mapping) else None+ if not isinstance(execution, Mapping):+ return "unknown"+ message = execution.get("message")+ if execution.get("should_trigger") is False or (+ isinstance(message, str)+ and message+ in {+ "No expected_skill -- skipped",+ "Negative case, no skill identified",+ }+ ):+ return "not_applicable"+ standard_sets = {DEFAULT_METRIC_SET, LEGACY_METRIC_SET}+ declared = reward.get("metric_set") or reward.get("metric_set_version")+ metric_set, metrics = metric_set_for_reward(reward)+ if (+ (declared and str(declared) not in standard_sets)+ or metric_set not in standard_sets+ or "skill_execution" not in metrics+ or metric_value(reward, "skill_execution") is None+ ):+ return "unknown"+ activation = execution.get("activation")+ if not isinstance(activation, Mapping) or activation.get("supported") is False:+ return "unknown"+ score = activation.get("score")+ if isinstance(score, bool) or not isinstance(score, int | float) or not 0 <= score <= 1 or not math.isfinite(score):+ return "unknown"+ if activation.get("passed") is True and score > 0:+ return "activated"+ if activation.get("passed") is False and score == 0:+ return "never_activated"+ return "unknown"+++def summarize_skill_activation(rewards: Iterable[Mapping[str, Any]]) -> dict[str, Any]:+ """Count logical trials, preserving uncertainty and excluding negative controls.++ A multi-step trial activated the skill if any step did. Missing or unsupported+ evidence prevents a claim that the skill was *never* activated. Historical+ and custom-only artifacts without activation evidence remain unknown.+ """+ trials: dict[str, set[str]] = {}+ for index, reward in enumerate(rewards):+ trial = str(reward.get("_trial_root_name") or reward.get("_trial_name") or f"__row_{index}")+ trials.setdefault(trial, set()).add(_activation_state(reward))++ positive = activated = unknown = 0+ for states in trials.values():+ if states & {"activated", "never_activated"}:+ positive += 1+ if "activated" in states:+ activated += 1+ elif "unknown" in states:+ unknown += 1++ if activated:+ status, reason = "activated", "Skill activation observed in with-skill trials"+ elif unknown or not trials:+ status, reason = "unknown", "Skill activation evidence is unavailable or incomplete"+ elif positive:+ status, reason = "never_activated", "Inconclusive: skill never activated in with-skill trials"+ else:+ status, reason = "not_applicable", "No positive skill-activation cases were evaluated"+ return {+ "status": status,+ "positive_trials": positive,+ "activated_trials": activated,+ "unknown_trials": unknown,+ "reason": reason,+ }diff --git a/src/skillevaluator/tier3/harbor/collector.py b/src/skillevaluator/tier3/harbor/collector.pyindex 756e5b33..de812e5f 100644--- a/src/skillevaluator/tier3/harbor/collector.py+++ b/src/skillevaluator/tier3/harbor/collector.py@@ -30,6 +30,7 @@ from typing import Anyfrom skillevaluator.tier3.eval_core.atif_helpers import extract_tool_calls_as_dicts, get_skill_tool_callsfrom skillevaluator.tier3.eval_core.checks import check_negative_casefrom skillevaluator.tier3.eval_core.codex_tool_call_normalizer import atif_content_text+from skillevaluator.tier3.harbor.activation import summarize_skill_activationfrom skillevaluator.tier3.harbor.metrics import (CUSTOM_ONLY_METRIC_SET,CUSTOM_SCORE_POLICY,@@ -6438,6 +6439,7 @@ def collect_harbor_results("num_trials": len(with_logical_rewards),"num_reward_rows": len(with_collected_rewards),"mixed_metric_contracts": with_mixed_metric_contracts,+ "skill_activation": summarize_skill_activation(with_rewards),"pass_at_k": _public_pass_summary(with_pass),**with_execution,"job_failure": with_job_failure,@@ -6774,12 +6776,14 @@ def collect_harbor_results(with_overall_score = Nonewithout_overall_score = None+ skill_activation = summarize_skill_activation(with_rewards)lift: dict[str, Any] = {}paired_execution_succeeded = (with_execution.get("execution_status") == "succeeded"and without_execution.get("execution_status") == "succeeded")- if paired_execution_succeeded and with_scores and without_scores:+ lift_eligible = paired_execution_succeeded and skill_activation["status"] != "never_activated"+ if lift_eligible and with_scores and without_scores:lift = _compute_lift(with_scores,without_scores,@@ -6811,7 +6815,7 @@ def collect_harbor_results(custom_lift: dict[str, Any] = {}if (- paired_execution_succeeded+ lift_eligibleand with_logical_rewardsand without_logical_rewardsand (with_custom_scores or without_custom_scores or (not with_scores and not without_scores))@@ -6827,7 +6831,7 @@ def collect_harbor_results(_write_generated_root_json(agent_dir / "custom_lift.json", output_dir, custom_lift)pass_lift: dict[str, Any] = {}- if paired_execution_succeeded and set(with_metrics) == set(without_metrics) and with_pass and without_pass:+ if lift_eligible and set(with_metrics) == set(without_metrics) and with_pass and without_pass:pass_lift = {"with_skill": with_pass.get("rate", 0.0),"without_skill": without_pass.get("rate", 0.0),@@ -6887,6 +6891,7 @@ def collect_harbor_results("custom_without_skill": without_custom_scores,"dimensions_with_skill": dimension_scores(with_scores),"dimensions_without_skill": dimension_scores(without_scores),+ "skill_activation": skill_activation,"lift": lift,"custom_lift": custom_lift,"pass_at_k": {diff --git a/src/skillevaluator/tier3/harbor/report.py b/src/skillevaluator/tier3/harbor/report.pyindex aed1fdb8..69daecb5 100644--- a/src/skillevaluator/tier3/harbor/report.py+++ b/src/skillevaluator/tier3/harbor/report.py@@ -135,6 +135,8 @@ def _pick_best_agent(continuepersisted = persisted_agents.get(agent) if isinstance(persisted_agents, dict) else Noneranking_data = persisted if isinstance(persisted, dict) else data+ if not _findings_eligible(ranking_data):+ continuestandard_metrics = report_data.metrics_for_condition(ranking_data, "with_skill")with_scores = ranking_data.get("with_skill")if standard_metrics and (@@ -159,6 +161,9 @@ def _pick_best_agent(def _findings_eligible(agent_data: dict[str, Any]) -> bool:"""Return whether persisted execution truth permits quality findings."""+ activation = agent_data.get("skill_activation")+ if isinstance(activation, dict) and activation.get("status") == "never_activated":+ return Falseconditions = agent_data.get("conditions")if not isinstance(conditions, dict) or "with_skill" not in conditions:return agent_data.get("execution_status") == "succeeded"@@ -1120,6 +1125,8 @@ def display_findings_report(if not rewards_for_agent:continueloaded_agent = loaded_agents.get(agent)+ if isinstance(loaded_agent, dict) and not _findings_eligible(loaded_agent):+ continuecanonical_scores: dict[str, Any] = {}rewards_complete = Trueif isinstance(loaded_agent, dict):diff --git a/src/skillevaluator/tier3/harbor/report_data.py b/src/skillevaluator/tier3/harbor/report_data.pyindex 2990d686..d70448dd 100644--- a/src/skillevaluator/tier3/harbor/report_data.py+++ b/src/skillevaluator/tier3/harbor/report_data.py@@ -688,6 +688,8 @@ def load_agent_data(continuekey = "with_skill" if variant == "with-skill" else "without_skill"agent_info[key] = scores+ if variant == "with-skill" and isinstance(data.get("skill_activation"), dict):+ agent_info["skill_activation"] = data["skill_activation"]metric_key = "metrics_with_skill" if variant == "with-skill" else "metrics_without_skill"if isinstance(data.get("metrics"), list):agent_info[metric_key] = data["metrics"]@@ -864,6 +866,14 @@ def load_agent_data(if "with_skill" not in agent_info:continue+ if "skill_activation" not in agent_info:+ from skillevaluator.tier3.harbor.activation import summarize_skill_activation++ activation = summarize_skill_activation(agent_info.get("rewards") or [])+ if agent_info.get("rewards_complete") is False and activation["status"] == "never_activated":+ activation = dict(activation, status="unknown", reason="Activation evidence is incomplete.")+ agent_info["skill_activation"] = activation+active_conditions = list(condition_execution.values())execution_errors = [error for condition in active_conditions for error in condition.get("execution_errors", []) if errordiff --git a/src/skillevaluator/tier3/harbor/templates/eval.py b/src/skillevaluator/tier3/harbor/templates/eval.pyindex 2af1203c..073e236a 100644--- a/src/skillevaluator/tier3/harbor/templates/eval.py+++ b/src/skillevaluator/tier3/harbor/templates/eval.py@@ -8204,9 +8204,6 @@ def score_skill_execution(if not expected_skill:return {"score": 1.0, "details": {"message": "No expected_skill -- skipped"}}- if not tool_calls:- return {"score": 0.0, "details": {"message": "No tool calls in trajectory"}}-checks = {}scores = []@@ -8219,6 +8216,9 @@ def score_skill_execution(checks["activation"] = rscores.append(r["score"])+ if not tool_calls:+ return {"score": 0.0, "details": {"message": "No tool calls in trajectory", **checks}}+r = check_script_execution(tool_calls, expected_script)checks["script_execution"] = rscores.append(r["score"])@@ -8235,6 +8235,13 @@ def score_skill_execution(checks["error_recovery"] = rscores.append(r["score"])+ # Nonapplicable script/recovery checks must not create execution credit+ # when the expected skill was never activated. Unknown activation remains+ # distinct from a demonstrated failure.+ activation = checks["activation"]+ if activation["passed"] is False and activation["score"] == 0.0:+ return {"score": 0.0, "details": checks}+avg = sum(scores) / len(scores) if scores else 0.0return {"score": round(avg, 4), "details": checks}diff --git a/src/skillevaluator/tier3/result_display.py b/src/skillevaluator/tier3/result_display.pyindex 12cf2bb2..abe288be 100644--- a/src/skillevaluator/tier3/result_display.py+++ b/src/skillevaluator/tier3/result_display.py@@ -90,6 +90,11 @@ def _condition_status(data: Mapping[str, Any], variant: str) -> str:return str(condition.get("execution_status") or condition.get("status") or "")+def _never_activated(data: Mapping[str, Any]) -> bool:+ activation = data.get("skill_activation")+ return isinstance(activation, Mapping) and activation.get("status") == "never_activated"++def _condition_usable(data: Mapping[str, Any], variant: str) -> bool:status = _condition_status(data, variant)if status:@@ -403,6 +408,10 @@ def _render_agent_scores(custom_with = data.get("custom_with_skill") if isinstance(data.get("custom_with_skill"), Mapping) else {}custom_without = data.get("custom_without_skill") if isinstance(data.get("custom_without_skill"), Mapping) else {}custom_lift = data.get("custom_lift") if isinstance(data.get("custom_lift"), Mapping) else {}+ never_activated = _never_activated(data)+ if never_activated:+ lift = {}+ custom_lift = {}show_baseline = not baseline_skipped and bool(baseline_status or baseline_scores or lift)table = Table(@@ -527,7 +536,14 @@ def _render_agent_scores(# Repeat the stored evaluator lift prominently without changing either# table or deriving a headline from dimensions or incomplete comparisons.content: Table | Group = table- if (+ if never_activated:+ content = Group(+ Text("INCONCLUSIVE: skill never activated", style="bold yellow"),+ Text("Scores are diagnostic only; no lift verdict was computed.", style="dim"),+ Text(""),+ table,+ )+ elif (with_usableand baseline_usableand all(_finite_number(value) is not None for value in (with_overall, baseline_overall, overall.get("delta")))@@ -602,7 +618,9 @@ def _render_dimensions(else:baseline_cell, baseline_bar = _score_cell(baseline_numeric)delta = (- with_numeric - baseline_numeric if with_numeric is not None and baseline_numeric is not None else None+ with_numeric - baseline_numeric+ if not _never_activated(data) and with_numeric is not None and baseline_numeric is not None+ else None)row = [Text(DIMENSION_DISPLAY.get(dimension, dimension.title()), style="bold"),@@ -777,7 +795,11 @@ def render_evaluation_result(result: Mapping[str, Any], *, console: Console) ->display_findings_report(dict(result),str(result.get("skill_name") or ""),- [str(agent) for agent in agents],+ [+ str(agent)+ for agent, data in agents.items()+ if not isinstance(data, Mapping) or not _never_activated(data)+ ],Path(str(run_dir)) if run_dir else Path(),)or set()diff --git a/tests/reporting/test_activation_benchmark_html_regressions.py b/tests/reporting/test_activation_benchmark_html_regressions.pynew file mode 100644index 00000000..86236e9a--- /dev/null+++ b/tests/reporting/test_activation_benchmark_html_regressions.py@@ -0,0 +1,132 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++"""Public report tables must preserve each agent's activation eligibility."""++from __future__ import annotations++import re++import pytest++from skillevaluator.constants import DIMENSION_MAPPING+from skillevaluator.evaluation.tier3_report import _validation_result_from_payload, build_agent_eval_payload+from skillevaluator.reporting import BenchmarkReporter, HTMLReporter+from skillevaluator.tier3.harbor.metrics import DEFAULT_METRICS+++def _agent(*, activated: bool | None, score: float = 0.4, baseline: bool = True) -> dict:+ scores = dict.fromkeys(DEFAULT_METRICS, score)+ scores["skill_execution"] = float(activated) if activated is not None else score+ reward = {**scores, "entry_id": "case", "trial_id": "case__attempt1", "details": {}}+ if activated is not None:+ reward["details"] = {"skill_execution": {"activation": {"passed": activated, "score": float(activated)}}}+ agent = {+ "execution_status": "succeeded",+ "expected_attempts": 2 if baseline else 1,+ "scored_attempts": 2 if baseline else 1,+ "num_trials": 1,+ "num_trials_baseline": int(baseline),+ "with_skill": scores,+ "rewards": [reward],+ }+ if baseline:+ baseline_scores = dict.fromkeys(DEFAULT_METRICS, 0.8)+ agent["without_skill"] = baseline_scores+ agent["rewards_baseline"] = [{**baseline_scores, "trial_id": "baseline__attempt1"}]+ else:+ agent["conditions"] = {"without_skill": {"execution_status": "skipped"}}+ return agent+++def _render(agents: dict[str, dict]) -> tuple[dict, str, str]:+ payload = build_agent_eval_payload("demo", agents, use_llm_judge=False)+ assert payload is not None+ result = _validation_result_from_payload(payload)+ assert result is not None+ benchmark = BenchmarkReporter(include_timestamp=False).render_all([result])+ html = HTMLReporter(include_timestamp=False).render_all([result])+ return payload, benchmark, html+++def _benchmark_rows(benchmark: str) -> dict[str, list[str]]:+ section = benchmark.split("## Results at a Glance\n", 1)[1].split("\n## ", 1)[0]+ return {+ cells[0]: cells[1:]+ for line in section.splitlines()+ if line.startswith("| ")+ for cells in [[cell.strip() for cell in line.strip("|").split("|")]]+ if cells[0] != "Measure"+ }++[email protected]("baseline", [True, False])[email protected]("score", [0.2, 0.9])+def test_benchmark_never_activated_scores_do_not_reconstruct_uplift(baseline: bool, score: float) -> None:+ payload, benchmark, _ = _render({"unactivated": _agent(activated=False, score=score, baseline=baseline)})+ agent = payload["agents"]["unactivated"]+ assert agent["lift"] is None+ assert agent["verdict"] == "inconclusive"+ rows = _benchmark_rows(benchmark)+ scores = {"Overall": agent["with_skill"], **{d["id"].title(): d["with_skill"] for d in agent["dimensions"]}}+ assert len(rows) == len(DIMENSION_MAPPING) + 1+ for measure, cells in rows.items():+ assert f"{scores[measure]:.0%}" in cells[0]+ assert "INCONCLUSIVE: skill never activated" in cells[0]+ assert "uplift unavailable" in cells[0]+ assert "points" not in cells[0]+ assert "baseline not run" not in cells[0]+++def _mixed_agents() -> dict[str, dict]:+ return {+ "unactivated": _agent(activated=False, score=0.99),+ "activated": _agent(activated=True, score=0.5),+ "no-baseline": _agent(activated=True, score=0.7, baseline=False),+ }+++def test_benchmark_mixed_agents_keep_inconclusive_missing_baseline_and_measured_lift_distinct() -> None:+ payload, benchmark, _ = _render(_mixed_agents())+ assert payload["agents"]["unactivated"]["with_skill"] > payload["agents"]["activated"]["with_skill"]+ for cells in _benchmark_rows(benchmark).values():+ by_agent = dict(zip(payload["agents"], cells, strict=True))+ assert "INCONCLUSIVE: skill never activated" in by_agent["unactivated"]+ assert "points" not in by_agent["unactivated"]+ assert "→" in by_agent["activated"] and "points" in by_agent["activated"]+ assert "INCONCLUSIVE" not in by_agent["activated"]+ assert "baseline not run; uplift unavailable" in by_agent["no-baseline"]+++def test_html_mixed_agents_identify_activation_suppression_in_every_grid_cell() -> None:+ payload, _, html = _render(_mixed_agents())+ # The HTML grid interleaves one cell per agent for every dimension and Overall.+ labels = re.findall(r'<span class="t3-mad-lift"[^>]*>(.*?)</span>', html, flags=re.DOTALL)+ assert len(labels) == 3 * (len(DIMENSION_MAPPING) + 1)+ by_agent = {name: labels[index::3] for index, name in enumerate(payload["agents"])}+ assert by_agent["unactivated"] == ["inconclusive: skill never activated"] * (len(DIMENSION_MAPPING) + 1)+ assert all(re.fullmatch(r"[+-]\d+\.\d{2} lift", label) for label in by_agent["activated"])+ assert by_agent["no-baseline"] == ["no baseline lift"] * (len(DIMENSION_MAPPING) + 1)+++def test_unknown_activation_keeps_legacy_comparisons() -> None:+ payload, benchmark, _ = _render({"legacy": _agent(activated=None)})+ assert payload["agents"]["legacy"]["skill_activation"]["status"] == "unknown"+ for cells in _benchmark_rows(benchmark).values():+ assert "→" in cells[0] and "points" in cells[0]+ assert "INCONCLUSIVE" not in cells[0]++[email protected]("baseline", [True, False])+def test_html_grid_names_never_activation_with_or_without_baseline(baseline: bool) -> None:+ payload, _, html = _render(+ {+ "unactivated": _agent(activated=False, baseline=baseline),+ "activated": _agent(activated=True),+ }+ )+ labels = re.findall(r'<span class="t3-mad-lift"[^>]*>(.*?)</span>', html, flags=re.DOTALL)+ assert len(labels) == 2 * (len(DIMENSION_MAPPING) + 1)+ by_agent = {name: labels[index::2] for index, name in enumerate(payload["agents"])}+ assert by_agent["unactivated"] == ["inconclusive: skill never activated"] * (len(DIMENSION_MAPPING) + 1)+ assert all(re.fullmatch(r"[+-]\d+\.\d{2} lift", label) for label in by_agent["activated"])diff --git a/tests/reporting/test_dashboard_activation_regressions.py b/tests/reporting/test_dashboard_activation_regressions.pynew file mode 100644index 00000000..9f1ac125--- /dev/null+++ b/tests/reporting/test_dashboard_activation_regressions.py@@ -0,0 +1,252 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++"""Activation eligibility must survive canonical and retained dashboard loading."""++from __future__ import annotations++import copy+import json+from pathlib import Path++import pytest++from skillevaluator.evaluation.tier3_report import agent_eval_result_from_directory+from skillevaluator.reporting.dashboard_data import comparison_rows, load_dashboard_path, parse_dashboard_report+from skillevaluator.tier3.harbor.metrics import DEFAULT_METRICS+++def _report(status: str | None = "never_activated") -> dict:+ agent = {+ "name": "opencode",+ "model": "model-a",+ "execution_status": "succeeded",+ "with_skill": 0.32,+ "baseline": 0.8,+ "num_trials": 1,+ "num_trials_baseline": 1,+ "conditions": {+ arm: {"execution_status": "succeeded", "expected_attempts": 1, "scored_attempts": 1}+ for arm in ("with_skill", "without_skill")+ },+ "trials": [+ {+ "entry_id": "case-a",+ "trial_id": "case-a__random",+ "tokens": {"prompt": 20, "completion": 2, "cached": 0},+ "agent_seconds": 3,+ "cost_usd": 0.02,+ }+ ],+ "trials_baseline": [+ {+ "entry_id": "case-a",+ "trial_id": "case-a__other",+ "tokens": {"prompt": 10, "completion": 1, "cached": 0},+ "agent_seconds": 5,+ "cost_usd": 0.01,+ }+ ],+ "pass_at_k": {"with_skill": {"rate": 0.0}, "without_skill": {"rate": 1.0}},+ }+ if status is not None:+ agent["skill_activation"] = {"status": status}+ return {"skill_name": "calculator", "run_id": "run-a", "agents": {"opencode": agent}}+++def test_never_activated_comparison_keeps_diagnostics_without_deltas() -> None:+ rows = parse_dashboard_report(_report(), "report.json").rows+ comparison = comparison_rows(rows)[0]+ assert comparison["comparison_status"] == "inconclusive"+ assert comparison["comparison_reason"] == "Inconclusive: skill never activated in with-skill trials"+ assert comparison["with_score"] == 0.32+ assert comparison["without_score"] == 0.8+ assert comparison["with_pass_rate"] == 0.0+ assert comparison["without_pass_rate"] == 1.0+ assert comparison["with_total_tokens"] == 22+ assert comparison["without_total_tokens"] == 11+ assert comparison["with_duration_seconds"] == 3+ assert comparison["without_duration_seconds"] == 5+ assert comparison["with_cost_usd"] == 0.02+ assert comparison["without_cost_usd"] == 0.01+ assert all(value is None for key, value in comparison.items() if key.endswith(("_delta", "_percent")))+ assert all(row["_complete"] and row["status"] == "succeeded" for row in rows)++[email protected]("status", [None, "unknown", "activated", "not_applicable"])+def test_legacy_unknown_activated_and_negative_controls_remain_comparable(status: str | None) -> None:+ comparison = comparison_rows(parse_dashboard_report(_report(status), "report.json").rows)[0]+ assert comparison["comparison_status"] == "comparable"+ assert comparison["comparison_reason"] == ""+ assert comparison["score_delta"] == pytest.approx(-0.48)+ assert comparison["pass_rate_delta"] == -1+ assert comparison["total_tokens_delta"] == 11+++def test_mixed_agents_keep_eligible_comparisons_separate() -> None:+ report = _report()+ report["agents"]["activated"] = copy.deepcopy(_report("activated")["agents"]["opencode"])+ report["agents"]["activated"]["with_skill"] = 0.9+ # A top-level best-agent census must not disqualify other agents.+ report["skill_activation"] = {"status": "never_activated"}+ comparisons = {row["_agent_key"]: row for row in comparison_rows(parse_dashboard_report(report, "r").rows)}+ assert comparisons["opencode"]["score_delta"] is None+ assert comparisons["opencode"]["comparison_status"] == "inconclusive"+ assert comparisons["activated"]["score_delta"] == pytest.approx(0.1)+ assert comparisons["activated"]["comparison_status"] == "comparable"+++def test_missing_baseline_takes_precedence_over_activation() -> None:+ report = _report()+ agent = report["agents"]["opencode"]+ agent.pop("baseline")+ agent.pop("trials_baseline")+ agent["conditions"].pop("without_skill")+ comparison = comparison_rows(parse_dashboard_report(report, "r").rows)[0]+ assert comparison["comparison_status"] == "missing condition"+ assert comparison["comparison_reason"] == "no without-skill arm for this run and configuration"+ assert comparison["score_delta"] is None++[email protected]("condition", ["with_skill", "without_skill"])[email protected]("failure", ["failed", "incomplete"])+def test_execution_and_coverage_failures_take_precedence_over_activation(condition: str, failure: str) -> None:+ report = _report()+ details = report["agents"]["opencode"]["conditions"][condition]+ if failure == "failed":+ details["execution_status"] = "failed"+ else:+ details["expected_attempts"] = 2+ comparison = comparison_rows(parse_dashboard_report(report, "r").rows)[0]+ label = condition.replace("_", "-")+ assert comparison["comparison_status"] == "incomplete or mismatched coverage"+ assert comparison["comparison_reason"] == (+ f"{label} arm status failed" if failure == "failed" else f"{label} trial coverage incomplete"+ )+ assert comparison["score_delta"] is None+++def _write(path: Path, value: object) -> None:+ path.parent.mkdir(parents=True, exist_ok=True)+ path.write_text(json.dumps(value), encoding="utf-8")+++def _retained_run(path: Path, status: str, *, persist_census: bool) -> None:+ metrics = dict.fromkeys(DEFAULT_METRICS, 0.8)+ _write(path / "run_config.json", {"agents": {"opencode": {"agent": "opencode", "model": "model-a"}}})+ _write(path / "result.json", {"skill_name": "calculator", "run_id": path.name})+ for variant, prompt_tokens in (("with-skill", 20), ("without-skill", 10)):+ condition = path / "opencode" / variant+ summary = {+ "model": "model-a",+ "scores": metrics,+ "metrics": list(DEFAULT_METRICS),+ "num_trials": 1,+ "execution_status": "succeeded",+ "expected_attempts": 1,+ "scored_attempts": 1,+ "pass_at_k": {"k": 1, "rate": 1.0},+ }+ if persist_census and variant == "with-skill":+ summary["skill_activation"] = {"status": status}+ _write(condition / "summary.json", summary)+ details = {+ "skill_execution": {+ "should_trigger": status != "not_applicable",+ "activation": {"passed": status == "activated", "score": float(status == "activated")},+ }+ }+ reward = {**metrics, "entry_id": "case-a", "trial_id": "case-a__random", "details": details}+ if status == "unknown":+ # Custom-only rewards cannot establish activation eligibility.+ reward = {"custom_score": 0.8, "entry_id": "case-a", "trial_id": "case-a__random"}+ trial = condition / "trials" / "case-a__random"+ _write(trial / "reward.json", reward)+ _write(+ trial / "trajectory.json",+ {"steps": [], "final_metrics": {"total_prompt_tokens": prompt_tokens, "total_completion_tokens": 2}},+ )+ _write(+ trial / "result.json",+ {+ "agent_execution": {+ "started_at": "2026-10-18T00:00:00Z",+ "finished_at": "2026-10-18T00:00:03Z",+ }+ },+ )++[email protected]("status", ["never_activated", "activated", "not_applicable", "unknown"])[email protected]("persist_census", [False, True])+def test_retained_and_exported_reports_preserve_activation_eligibility(+ tmp_path: Path, status: str, persist_census: bool+) -> None:+ run = tmp_path / "run"+ _retained_run(run, status, persist_census=persist_census)+ result = agent_eval_result_from_directory(tmp_path, run, use_llm_judge=False)+ canonical = result.metadata["agent_eval"]+ assert canonical["agents"]["opencode"]["skill_activation"]["status"] == status+ report_path = tmp_path / "report.json"+ _write(report_path, canonical)+ for source in (run, run / "result.json", report_path):+ comparison = comparison_rows(load_dashboard_path(source).rows)[0]+ if status == "never_activated":+ assert canonical["agents"]["opencode"]["lift"] is None+ assert comparison["comparison_status"] == "inconclusive"+ assert "skill never activated" in comparison["comparison_reason"]+ assert comparison["score_delta"] is None+ assert comparison["pass_rate_delta"] is None+ assert comparison["total_tokens_delta"] is None+ else:+ assert comparison["comparison_status"] == "comparable"+ assert comparison["score_delta"] == pytest.approx(0)+ assert comparison["total_tokens_delta"] == 10+ assert comparison["with_score"] is not None+ assert comparison["without_score"] is not None+++def test_dashboard_app_displays_mixed_activation_without_reconstructing_lift(tmp_path: Path) -> None:+ st = pytest.importorskip("streamlit")+ pd = pytest.importorskip("pandas")+ pytest.importorskip("pyarrow")+ from streamlit.testing.v1 import AppTest++ report = _report()+ eligible = copy.deepcopy(_report("activated")["agents"]["opencode"])+ eligible.update({"name": "claude-code", "model": "model-b", "with_skill": 0.9})+ eligible["pass_at_k"] = {"with_skill": {"rate": 1.0}, "without_skill": {"rate": 0.0}}+ report["agents"]["claude-code"] = eligible+ path = tmp_path / "mixed-report.json"+ _write(path, report)+ st.cache_data.clear()+ try:+ app = AppTest.from_string(+ f"from skillevaluator.reporting.dashboard_app import main\nmain([{str(path)!r}])\n",+ default_timeout=60,+ ).run()+ assert not app.exception+ performance = app.dataframe[0].value+ assert performance["score"].tolist() == [0.32, 0.8, 0.9, 0.8]+ for view in ("Quality", "All metrics"):+ app.selectbox(key="comparison_metrics").set_value(view).run()+ assert not app.exception+ comparison = app.dataframe[1].value.set_index("model")+ zero = comparison.loc["model-a"]+ assert zero["comparison_status"] == "inconclusive"+ assert zero["comparison_reason"] == "Inconclusive: skill never activated in with-skill trials"+ assert pd.isna(zero["score_delta"])+ assert pd.isna(zero["pass_rate_delta"])+ assert zero["with_score"] == 0.32+ assert zero["without_score"] == 0.8+ activated = comparison.loc["model-b"]+ assert activated["comparison_status"] == "comparable"+ assert activated["score_delta"] == pytest.approx(0.1)+ assert activated["pass_rate_delta"] == 1+ if view == "All metrics":+ assert pd.isna(zero["total_tokens_delta"])+ assert zero["with_total_tokens"] == 22+ assert activated["total_tokens_delta"] == 11+ assert len(app.get("download_button")) == 2+ finally:+ st.cache_data.clear()diff --git a/tests/reporting/test_dashboard_app.py b/tests/reporting/test_dashboard_app.pyindex cf71c901..08512c94 100644--- a/tests/reporting/test_dashboard_app.py+++ b/tests/reporting/test_dashboard_app.py@@ -148,6 +148,25 @@ def test_performance_table_shows_per_trial_usage(tmp_path: Path) -> None:assert config["total_tokens_per_trial"]["label"] == "Tokens / trial"for field in ("total_tokens_per_trial", "duration_seconds_per_trial", "cost_usd_per_trial"):assert config[field]["type_config"]["type"] == "number", field+++def test_never_activated_dashboard_retains_scores_without_comparison_deltas(tmp_path: Path) -> None:+ path = _write_report(tmp_path / "never-activated.json")+ report = json.loads(path.read_text(encoding="utf-8"))+ report["agents"]["opencode"]["skill_activation"] = {"status": "never_activated"}+ path.write_text(json.dumps(report), encoding="utf-8")+ app = _app([path])+ app.selectbox(key="comparison_metrics").set_value("Quality").run()+ assert not app.exception+ performance = app.dataframe[0].value+ comparison = app.dataframe[1].value+ assert performance["score"].tolist() == [0.9, 0.6]+ assert comparison["with_score"].iloc[0] == 0.9+ assert comparison["without_score"].iloc[0] == 0.6+ assert comparison["comparison_status"].iloc[0] == "inconclusive"+ assert "skill never activated" in comparison["comparison_reason"].iloc[0]+ assert comparison["score_delta"].isna().all()+ assert comparison["pass_rate_delta"].isna().all()rows = dashboard_app.load_dashboard_path(path).rowsheader = next(csv.reader(io.StringIO(rows_csv(rows, dashboard_app._PERFORMANCE_COLUMNS))))assert header.index("total_tokens_per_trial") == header.index("score") + 1diff --git a/tests/reporting/test_skill_activation_reporting.py b/tests/reporting/test_skill_activation_reporting.pynew file mode 100644index 00000000..1acb7317--- /dev/null+++ b/tests/reporting/test_skill_activation_reporting.py@@ -0,0 +1,333 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++"""Never-activated trials must stay inconclusive across saved report surfaces."""++from __future__ import annotations++import io+import json++import pytest+from rich.console import Console++from skillevaluator.evaluation.tier3_report import (+ _load_suggestions_v2,+ _validation_result_from_payload,+ agent_eval_result_from_directory,+ build_agent_eval_payload,+)+from skillevaluator.reporting import HTMLReporter, MarkdownReporter+from skillevaluator.reporting.base import assess_tier3_evidence+from skillevaluator.reporting.cli import CLIReporter+from skillevaluator.reporting.console_ui import summarize_tier3+from skillevaluator.tier3.harbor.metrics import DEFAULT_METRICS+from skillevaluator.tier3.harbor.report_data import load_agent_data+from skillevaluator.tier3.result_display import render_result+++def _agent(*, activated: bool = False, score: float = 0.4) -> dict:+ reward = {+ **dict.fromkeys(DEFAULT_METRICS, score),+ "entry_id": "case",+ "trial_id": "case__attempt1",+ "details": {"skill_execution": {"activation": {"passed": activated, "score": float(activated)}}},+ }+ return {+ "execution_status": "succeeded",+ "expected_attempts": 2,+ "scored_attempts": 2,+ "num_trials": 1,+ "num_trials_baseline": 1,+ "with_skill": dict.fromkeys(DEFAULT_METRICS, score),+ "without_skill": dict.fromkeys(DEFAULT_METRICS, 0.8),+ "rewards": [reward],+ "rewards_baseline": [{**reward, "trial_id": "baseline__attempt1"}],+ "lift": {"overall": {"delta": -0.4}},+ "pass_lift": {"delta": -1.0},+ "custom_with_skill": {"custom_grade": 0.2},+ "custom_without_skill": {"custom_grade": 0.9},+ "custom_lift": {"custom_grade": {"delta": -0.7}},+ }++[email protected]("score", [0.2, 0.9])+def test_no_activation_suppresses_all_causal_lift_without_erasing_scores(score: float) -> None:+ payload = build_agent_eval_payload("demo", {"opencode": _agent(score=score)}, use_llm_judge=False)+ assert payload is not None+ assert payload["execution_status"] == "succeeded"+ assert payload["verdict"] == "inconclusive"+ assert payload["overall_score"] == score+ assert payload["overall_lift"] is None+ assert payload["composite_lift"] is None+ assert payload["summary"]["skill_activation"]["status"] == "never_activated"+ assert all(item["lift"] is None for item in payload["evaluators"].values())+ assert all(item["lift"] is None and item["verdict"] == "inconclusive" for item in payload["dimensions"])+ assert all(item["lift"] is None for item in payload["evaluator_cards"])+ assert payload["pass_at_k"]["lift"] == {}+ assert all("lift_scores" not in trial for trial in payload["trials"])+ assert all(item["severity"] != "fail" for item in payload["conclusions"])+++def test_inconclusive_renders_consistently_in_html_markdown_cli_and_evidence_assessment() -> None:+ payload = build_agent_eval_payload("demo", {"opencode": _agent()}, use_llm_judge=False)+ result = _validation_result_from_payload(payload)+ assert result is not None+ assessment = assess_tier3_evidence([result])+ assert assessment.status == "inconclusive"+ assert not assessment.evidence_complete+ assert assessment.execution_status == "succeeded"+ assert any("INCONCLUSIVE" in text for row in summarize_tier3(result)[2] for text, _ in row.segments)+ html = HTMLReporter(include_timestamp=False).render_all([result])+ markdown = MarkdownReporter(include_timestamp=False).render_all([result])+ assert "Inconclusive: skill never activated." in html+ assert "**Verdict:** INCONCLUSIVE" in markdown+ assert "Skill never activated." in markdown+ output = io.StringIO()+ CLIReporter._print_agent_eval_tables(payload, Console(file=output, width=140, color_system=None))+ assert "Verdict: INCONCLUSIVE" in output.getvalue()+++def test_mixed_agents_select_activated_evidence_even_when_unactivated_score_is_higher() -> None:+ payload = build_agent_eval_payload(+ "demo", {"unactivated": _agent(score=1.0), "activated": _agent(activated=True, score=0.8)}, use_llm_judge=False+ )+ assert payload is not None+ assert payload["best_agent"] == "activated"+ assert payload["verdict"] == "pass"+ assert payload["agents"]["unactivated"]["verdict"] == "inconclusive"+ assert payload["agents"]["unactivated"]["lift"] is None+ assert payload["overall_lift"] == 0.0+++def test_mixed_agents_discard_unattributed_stale_skill_advice(tmp_path) -> None:+ agents = {"unactivated": _agent(score=1.0), "activated": _agent(activated=True, score=0.8)}+ stale_advice = [{"suggestion": "Remove the skill because it regressed quality."}]+ agent_dir = tmp_path / "unactivated"+ agent_dir.mkdir()+ (agent_dir / "findings.json").write_text(json.dumps({"suggestions_v2": stale_advice}))+ suggestions = _load_suggestions_v2(tmp_path, agents)+ assert suggestions == []+ payload = build_agent_eval_payload("demo", agents, suggestions_v2=suggestions, use_llm_judge=False)+ assert payload is not None+ assert payload["suggestions_v2"] == []+ assert payload["verdict"] == "pass"++[email protected]("with_best_advice", [True, False])+def test_mixed_agents_load_advice_only_from_selected_best_agent(tmp_path, with_best_advice: bool) -> None:+ agents = {+ "a-unactivated": _agent(score=1.0),+ "b-activated": _agent(activated=True, score=0.7),+ "z-best": _agent(activated=True, score=0.8),+ }+ for name, info in agents.items():+ for condition, score_key, reward_key in (+ ("with-skill", "with_skill", "rewards"),+ ("without-skill", "without_skill", "rewards_baseline"),+ ):+ condition_dir = tmp_path / name / condition+ trial_dir = condition_dir / "trials" / "case__attempt1"+ trial_dir.mkdir(parents=True)+ (condition_dir / "summary.json").write_text(+ json.dumps(+ {+ "scores": info[score_key],+ "execution_status": "succeeded",+ "num_trials": 1,+ "expected_attempts": 1,+ "scored_attempts": 1,+ }+ )+ )+ (trial_dir / "reward.json").write_text(json.dumps(info[reward_key][0]))+ if name != "z-best" or with_best_advice:+ (tmp_path / name / "findings.json").write_text(+ json.dumps({"suggestions_v2": [{"suggestion": f"Advice from {name}", "dimension": "accuracy"}]})+ )++ result = agent_eval_result_from_directory(tmp_path / "demo", tmp_path, use_llm_judge=False)+ assert result is not None+ payload = result.metadata["agent_eval"]+ assert payload["best_agent"] == "z-best"+ recommendations = [item["recommendation"] for item in payload["suggestions_v2"]]+ assert recommendations == (["Advice from z-best"] if with_best_advice else [])+ html = HTMLReporter(include_timestamp=False).render_all([result])+ assert ("Advice from z-best" in html) is with_best_advice+ assert "Advice from a-unactivated" not in html+ assert "Advice from b-activated" not in html++[email protected]("activated", [True, False])+def test_llm_insights_use_eligible_trials_without_hiding_diagnostics(monkeypatch, activated: bool) -> None:+ calls = []+ prompts = []++ def build_insights(payload, **kwargs):+ from skillevaluator.evaluation.insights_judge import InsightsJudge++ calls.append(payload)+ prompt = InsightsJudge().create_user_prompt(canonical=payload, deterministic=kwargs["deterministic"])+ prompts.append(json.loads(prompt.split("\n\n", 1)[1]))+ return {"conclusions": [], "recommendations": [{"message": "Advice from activated evidence."}]}++ monkeypatch.setattr("skillevaluator.evaluation.insights_judge.build_insights", build_insights)+ payload = build_agent_eval_payload(+ "demo",+ {"unactivated": _agent(score=1.0), "activated": _agent(activated=activated, score=0.8)},+ use_llm_judge=True,+ )+ assert payload is not None+ assert {trial["agent"] for trial in payload["trials"]} == {"activated", "unactivated"}+ if activated:+ assert len(calls) == 1+ assert calls[0]["agents_run"] == ["activated"]+ assert set(calls[0]["agents"]) == {"activated"}+ assert {trial["agent"] for trial in calls[0]["trials"]} == {"activated"}+ assert prompts[0]["skill"]["agents_run"] == ["activated"]+ assert {trial["agent"] for trial in prompts[0]["trials"]} == {"activated"}+ assert "Advice from activated evidence." in payload["suggestions"]+ else:+ assert calls == []+ assert "Advice from activated evidence." not in payload["suggestions"]++[email protected]("best_has_trace", [True, False])+def test_mixed_agent_advice_links_only_eligible_traces_and_keeps_full_diagnostics(monkeypatch, best_has_trace) -> None:+ agents = {+ "never": _agent(score=0.1),+ "activated": _agent(activated=True, score=0.4),+ "best": _agent(activated=True, score=0.8),+ }+ for name, agent in agents.items():+ if name == "best" and not best_has_trace:+ continue+ agent["rewards"][0]["harbor_viewer"] = {+ "job_url": "https://harbor.example.test/jobs/run",+ "trial_url": f"https://harbor.example.test/jobs/run/trials/{name}",+ }++ def build_insights(_payload, **_kwargs):+ return {+ "conclusions": [{"title": "Eligible case evidence", "source": "llm", "evidence_case_ids": ["case"]}],+ "recommendations": [+ {"message": "Improve case using activated evidence.", "source": "llm", "evidence_case_ids": ["case"]}+ ],+ }++ monkeypatch.setattr("skillevaluator.evaluation.insights_judge.build_insights", build_insights)+ payload = build_agent_eval_payload(+ "demo",+ agents,+ suggestions_v2=[{"recommendation": "Advice from best", "evidence_case_ids": ["case"]}],+ use_llm_judge=True,+ )+ assert payload is not None+ assert payload["best_agent"] == "best"+ linked_agents = set(agents) if best_has_trace else {"never", "activated"}+ assert {link["agent"] for link in payload["harbor_viewer"]["evidence_links"]} == linked_agents+ for field in ("conclusions", "recommendations"):+ llm_advice = [item for item in payload[field] if item.get("source") == "llm"]+ assert len(llm_advice) == 1+ assert llm_advice[0]["evidence"]["agent"] in {"activated", "best"}+ assert "/trials/never" not in llm_advice[0]["evidence"]["url"]+ assert all(item["evidence"]["agent"] != "never" for item in payload[field] if item.get("evidence"))+ deterministic_advice = [item for item in payload["recommendations"] if item.get("source") == "deterministic"]+ assert deterministic_advice+ if best_has_trace:+ assert payload["suggestions_v2"][0]["harbor_evidence"]["agent"] == "best"+ assert payload["suggestions_v2"][0]["harbor_evidence"]["url"].endswith("/trials/best")+ assert all(item["evidence"]["agent"] == "best" for item in deterministic_advice)+ else:+ assert "harbor_evidence" not in payload["suggestions_v2"][0]+ assert all("evidence" not in item for item in deterministic_advice)+++def test_missing_or_incomplete_activation_evidence_does_not_assert_never_activated() -> None:+ for change in ({"rewards": []}, {"rewards_complete": False}):+ payload = build_agent_eval_payload("demo", {"opencode": {**_agent(), **change}}, use_llm_judge=False)+ assert payload is not None+ assert payload["verdict"] != "inconclusive"+++def test_negative_only_activation_cases_are_not_inconclusive() -> None:+ agent = _agent()+ agent["rewards"][0]["details"]["skill_execution"] = {"should_trigger": False}+ payload = build_agent_eval_payload("demo", {"opencode": agent}, use_llm_judge=False)+ assert payload is not None+ assert payload["skill_activation"]["status"] == "not_applicable"+ assert payload["verdict"] != "inconclusive"+++def test_no_baseline_does_not_turn_never_activated_scores_into_quality_verdict() -> None:+ agent = _agent(score=1.0)+ agent.pop("without_skill")+ agent.pop("rewards_baseline")+ agent["conditions"] = {"without_skill": {"execution_status": "skipped"}}+ payload = build_agent_eval_payload("demo", {"opencode": agent}, use_llm_judge=False)+ assert payload is not None+ assert payload["verdict"] == "inconclusive"+ assert payload["overall_score"] == 1.0+ assert payload["overall_lift"] is None+++def test_saved_summary_preserves_census_when_trial_loading_is_incomplete(tmp_path) -> None:+ condition = tmp_path / "opencode" / "with-skill"+ condition.mkdir(parents=True)+ summary = {+ "scores": dict.fromkeys(DEFAULT_METRICS, 0.4),+ "execution_status": "succeeded",+ "num_trials": 12,+ "expected_attempts": 12,+ "scored_attempts": 12,+ "skill_activation": {+ "status": "never_activated",+ "positive_trials": 12,+ "activated_trials": 0,+ "unknown_trials": 0,+ "reason": "Skill never activated.",+ },+ }+ (condition / "summary.json").write_text(json.dumps(summary))+ agents = load_agent_data(tmp_path)+ assert agents["opencode"]["skill_activation"]["positive_trials"] == 12+ payload = build_agent_eval_payload("demo", agents, use_llm_judge=False)+ assert payload is not None and payload["verdict"] == "inconclusive"+++def test_historical_saved_trials_rebuild_activation_census_for_all_report_consumers(tmp_path) -> None:+ condition = tmp_path / "opencode" / "with-skill"+ trial = condition / "trials" / "case__attempt1"+ trial.mkdir(parents=True)+ agent = _agent()+ (condition / "summary.json").write_text(+ json.dumps({"scores": agent["with_skill"], "execution_status": "succeeded", "num_trials": 1})+ )+ (trial / "reward.json").write_text(json.dumps(agent["rewards"][0]))+ agents = load_agent_data(tmp_path)+ assert agents["opencode"]["rewards_complete"]+ assert agents["opencode"]["skill_activation"]["status"] == "never_activated"+++def test_terminal_does_not_restore_dimension_lift_from_raw_scores() -> None:+ agent = _agent()+ agent.update(+ skill_activation={"status": "never_activated"},+ dimensions_with_skill={"effectiveness": {"score": 0.4}},+ dimensions_without_skill={"effectiveness": {"score": 0.8}},+ )+ output = render_result({"execution_status": "succeeded", "agents": {"opencode": agent}})+ assert "INCONCLUSIVE: skill never activated" in output+ assert "OVERALL SKILL LIFT" not in output+ assert "-0.40" not in output++[email protected]("activation", [None, [], "unknown", {"status": None}, {"status": []}, {"status": "unknown"}])+def test_html_tolerates_malformed_persisted_activation_summary(activation) -> None:+ payload = build_agent_eval_payload("demo", {"opencode": _agent(activated=True)}, use_llm_judge=False)+ assert payload is not None+ payload["agents"]["opencode"]["skill_activation"] = activation+ result = _validation_result_from_payload(payload)+ assert result is not None+ html = HTMLReporter(include_timestamp=False).render_all([result])+ assert "Agent Scorecards" in htmldiff --git a/tests/test_harbor_skill_activation.py b/tests/test_harbor_skill_activation.pynew file mode 100644index 00000000..c4de8655--- /dev/null+++ b/tests/test_harbor_skill_activation.py@@ -0,0 +1,150 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++"""Activation evidence must precede a causal skill-lift verdict."""++from __future__ import annotations++import json+from pathlib import Path++import pytest++from skillevaluator.tier3.harbor.activation import summarize_skill_activation+from skillevaluator.tier3.harbor.collector import collect_harbor_results+from skillevaluator.tier3.harbor.metrics import CUSTOM_ONLY_METRIC_SET, CUSTOM_SCORE_POLICY, DEFAULT_METRICS+from skillevaluator.tier3.harbor.report import _findings_eligible+++def _reward(score: float = 0.0, *, passed: bool | None = False, **activation: object) -> dict:+ return {+ **dict.fromkeys(DEFAULT_METRICS, 0.8),+ "details": {"skill_execution": {"activation": {"score": score, "passed": passed, **activation}}},+ }+++def test_zero_activation_is_separate_from_execution_score() -> None:+ summary = summarize_skill_activation([_reward(), _reward()])+ assert summary["status"] == "never_activated"+ assert summary["positive_trials"] == 2+ assert summary["activated_trials"] == 0+ assert summary["unknown_trials"] == 0+++def test_unactivated_skill_does_not_generate_quality_advice() -> None:+ assert not _findings_eligible(+ {"execution_status": "succeeded", "skill_activation": summarize_skill_activation([_reward()])}+ )+++def test_custom_grader_details_do_not_imply_standard_activation() -> None:+ reward = {+ "metric_set": CUSTOM_ONLY_METRIC_SET,+ "score_policy": CUSTOM_SCORE_POLICY,+ "custom_metrics": {"quality": 0.8},+ "details": _reward()["details"],+ }+ assert summarize_skill_activation([reward])["status"] == "unknown"+++def test_malformed_message_is_not_an_activation_skip() -> None:+ reward = _reward()+ reward["details"]["skill_execution"] = {"message": {"malformed": True}}+ assert summarize_skill_activation([reward])["status"] == "unknown"++[email protected]("score", [0.5, 0.75, 1.0])+def test_any_positive_activation_preserves_lift(score: float) -> None:+ summary = summarize_skill_activation([_reward(), _reward(score, passed=True)])+ assert summary["status"] == "activated"+ assert summary["activated_trials"] == 1+++ "unknown",+ [{}, _reward(0.5, passed=None, supported=False), _reward(float("nan")), _reward(True), _reward(-1.0)],+)+def test_unknown_or_malformed_evidence_does_not_assert_never_activated(unknown: dict) -> None:+ summary = summarize_skill_activation([_reward(), unknown])+ assert summary["status"] == "unknown"+ assert summary["unknown_trials"] == 1+++ "detail",+ [{"should_trigger": False}, {"message": "No expected_skill -- skipped"}],+)+def test_negative_and_unlabelled_cases_do_not_count_as_activation_opportunities(detail: dict) -> None:+ negative = {"details": {"skill_execution": detail}}+ assert summarize_skill_activation([negative])["status"] == "not_applicable"+ summary = summarize_skill_activation([_reward(), negative])+ assert summary["status"] == "never_activated"+ assert summary["positive_trials"] == 1+++def test_multistep_trial_counts_once_and_later_activation_is_preserved() -> None:+ rows = [+ {**_reward(), "_trial_root_name": "trial-a", "_step_name": "one"},+ {**_reward(1.0, passed=True), "_trial_root_name": "trial-a", "_step_name": "two"},+ {**_reward(), "_trial_root_name": "trial-b"},+ ]+ summary = summarize_skill_activation(rows)+ assert summary["status"] == "activated"+ assert summary["positive_trials"] == 2+ assert summary["activated_trials"] == 1+++def _write_job(jobs: Path, variant: str, rewards: list[dict]) -> None:+ job = jobs / f"demo-opencode-{variant}"+ for index, reward in enumerate(rewards):+ verifier = job / f"case-{index}__trial" / "verifier"+ verifier.mkdir(parents=True)+ (verifier / "reward.json").write_text(json.dumps({"entry_id": f"case-{index}", **reward}))+ (job / "result.json").write_text(+ json.dumps(+ {+ "n_total_trials": len(rewards),+ "stats": {+ "n_trials": len(rewards),+ "n_errors": 0,+ "evals": {+ "opencode__model": {+ "n_trials": len(rewards),+ "n_errors": 0,+ "reward_stats": {"reward": {"0.8": [f"case-{i}__trial" for i in range(len(rewards))]}},+ }+ },+ },+ }+ )+ )++[email protected]("skip_baseline", [False, True])+def test_collector_preserves_scores_but_suppresses_unactivated_lift(tmp_path: Path, skip_baseline: bool) -> None:+ jobs, output = tmp_path / "jobs", tmp_path / "output"+ _write_job(jobs, "with", [_reward()])+ if not skip_baseline:+ _write_job(jobs, "without", [_reward()])+ result = collect_harbor_results(+ "demo", ["opencode"], output, jobs, skip_baseline=skip_baseline, expected_case_ids=["case-0"]+ )+ agent = result["agents"]["opencode"]+ assert agent["execution_status"] == "succeeded"+ assert agent["with_skill"]["accuracy"] == 0.8+ assert agent["skill_activation"]["status"] == "never_activated"+ assert agent["lift"] == {}+ assert agent["custom_lift"] == {}+ assert agent["pass_at_k"]["lift"] == {}+ summary = json.loads((output / "opencode/with-skill/summary.json").read_text())+ assert summary["skill_activation"] == agent["skill_activation"]+ assert not (output / "opencode/lift.json").exists()+++def test_collector_keeps_lift_when_one_trial_activates(tmp_path: Path) -> None:+ jobs, output = tmp_path / "jobs", tmp_path / "output"+ _write_job(jobs, "with", [_reward(), _reward(0.75, passed=True)])+ _write_job(jobs, "without", [_reward(), _reward()])+ result = collect_harbor_results("demo", ["opencode"], output, jobs, expected_case_ids=["case-0", "case-1"])+ assert result["agents"]["opencode"]["skill_activation"]["activated_trials"] == 1+ assert result["agents"]["opencode"]["lift"]["overall"]["delta"] == 0.0diff --git a/tests/tier3/test_skill_activation_scoring.py b/tests/tier3/test_skill_activation_scoring.pynew file mode 100644index 00000000..0e5666fc--- /dev/null+++ b/tests/tier3/test_skill_activation_scoring.py@@ -0,0 +1,120 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++"""Expected skill activation is required for positive execution credit."""++from __future__ import annotations++from types import ModuleType++import pytest++from skillevaluator.tier3.eval_core import checks+from skillevaluator.tier3.eval_core.codex_tool_call_normalizer import UNSUPPORTED_NATIVE_CODEX_EXEC++[email protected](params=("host", "verifier"))+def scorer(request: pytest.FixtureRequest, harbor_eval_template: ModuleType) -> ModuleType:+ return checks if request.param == "host" else harbor_eval_template+++ "call",+ [+ {"action": "Read", "action_input": {"file_path": "/workspace/README.md"}},+ {"action": "Read", "action_input": {"file_path": "/skills/other/SKILL.md"}},+ {"action": "Read", "action_input": {"file_path": "/skills/demo/SKILL.md"}, "is_error": True},+ {"action": "Bash", "action_input": {"command": "python /skills/demo/scripts/run.py"}},+ ],+ ids=("unrelated-work", "wrong-skill", "failed-read", "script-without-activation"),+)[email protected]("expected_script", (None, "run.py"))+def test_zero_activation_cannot_receive_execution_credit(+ scorer: ModuleType, call: dict, expected_script: str | None+) -> None:+ result = scorer.score_skill_execution([call], "demo", expected_script)++ assert result["score"] == 0.0+ assert result["details"]["activation"]["passed"] is False+ assert result["details"]["activation"]["score"] == 0.0+ # Keep the independent evidence visible even though it cannot override activation.+ assert result["details"]["error_recovery"]["score"] == 1.0+ assert set(result["details"]) == {"activation", "script_execution", "workflow_order", "error_recovery"}+++def test_empty_positive_trajectory_retains_zero_activation_evidence(scorer: ModuleType) -> None:+ result = scorer.score_skill_execution([], "demo")++ assert result["score"] == 0.0+ assert result["details"]["activation"]["passed"] is False+ assert result["details"]["activation"]["score"] == 0.0+ assert result["details"]["message"] == "No tool calls in trajectory"+++ ("call", "options", "activation_score", "execution_score"),+ [+ ({"action": "Read", "action_input": {"file_path": "/skills/demo/SKILL.md"}}, {}, 1.0, 1.0),+ ({"action": "Bash", "action_input": {"command": "cat /skills/demo/SKILL.md"}}, {}, 0.75, 0.9375),+ (+ {"action": "Read", "action_input": {"file_path": "/skills/alternate/SKILL.md"}},+ {"acceptable_skills": ["alternate"]},+ 0.75,+ 0.9375,+ ),+ ({"action": "Skill", "action_input": {"skill": "demo"}}, {"skill_tool_names": ["demo"]}, 1.0, 1.0),+ ],+ ids=("target-read", "shell-partial-credit", "acceptable-alternate", "native-skill"),+)+def test_supported_activation_preserves_subcheck_average(+ scorer: ModuleType, call: dict, options: dict, activation_score: float, execution_score: float+) -> None:+ result = scorer.score_skill_execution([call], "demo", **options)++ assert result["details"]["activation"]["score"] == activation_score+ assert result["score"] == execution_score+++def test_unsupported_activation_is_unknown_and_not_forced_to_zero(scorer: ModuleType) -> None:+ call = {+ "action": "functions.exec",+ "action_input": {"code": "await tools.exec_command(args)"},+ "normalization_status": UNSUPPORTED_NATIVE_CODEX_EXEC,+ }+ result = scorer.score_skill_execution([call], "demo", "run.py")++ assert result["details"]["activation"]["passed"] is None+ assert result["details"]["activation"]["supported"] is False+ assert result["score"] == 0.5++[email protected]("expected_skill", (None, ""))+def test_negative_control_does_not_require_positive_activation(scorer: ModuleType, expected_skill: str | None) -> None:+ result = scorer.score_skill_execution(+ [{"action": "Read", "action_input": {"file_path": "/workspace/README.md"}}],+ expected_skill,+ should_trigger=False,+ evaluated_skill="demo",+ require_evaluated_skill=True,+ )++ assert result["score"] == 1.0+ assert result["details"]["negative_check"]["passed"] is True+ assert "activation" not in result["details"]++[email protected]("expected_skill", (None, "", "demo"))+def test_unlabeled_case_remains_skipped(scorer: ModuleType, expected_skill: str | None) -> None:+ result = scorer.score_skill_execution([], expected_skill, should_trigger=None)++ assert result["score"] == 1.0+ assert "skipped" in result["details"]["message"]+ assert "activation" not in result["details"]+++def test_missing_expected_skill_remains_skipped(scorer: ModuleType) -> None:+ result = scorer.score_skill_execution([], "", should_trigger=True)++ assert result["score"] == 1.0+ assert "skipped" in result["details"]["message"]+ assert "activation" not in result["details"]