Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

Polish terminal reports and keep CI progress durable (#353)

Narendran Raghavanf440bbd5f613
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 963f987c..ee8f3923 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,15 @@ All notable changes to SkillEvaluator are documented in this file.
- When CI redaction masks a credential inside a reported PII value, remove the
finding's digest as well.
+- Align terminal grade and lift colors across compact, full, and direct Tier 3
+ reports; displayed zero lift is neutral. Narrow terminals stack complete
+ condition score/bar groups. Full-report Details retain successful quality and
+ lift evidence without opening every success panel. Artifact footers show the
+ emitted filenames, with links only on capable interactive terminals.
+- Compact validation now writes durable tier/check starts, stage changes and
+ measured milestones to stderr in CI, with deduplicated events and a heartbeat
+ after 60 seconds without progress. Final report panels remain on stdout.
+
- Align final terminal and HTML command outcomes: Passed required checks,
Changes required, or Incomplete. Keep publication eligibility and Best agent
score separate. Tier counts exclude explicit N/A checks from execution;
diff --git a/docs/reports.mdx b/docs/reports.mdx
index 93a20b2b..ab23b964 100644
--- a/docs/reports.mdx
+++ b/docs/reports.mdx
@@ -47,7 +47,20 @@ into unrelated content.
For `validate`, that compact default is a pipeline view — a per-check ticker
plus a summary. Pass `--verbose` to get the full per-check detail stream in
-the terminal instead.
+the terminal instead. Final panels are written to stdout. When output is
+redirected or running in CI, tier and check starts, stage changes, measured
+completion counts, and completion lines remain visible on stderr. Unchanged
+updates are omitted; a running stage emits a heartbeat after 60 seconds without
+progress. Counts describe observed work, without estimating a completion
+percentage. Interruption tears down the progress display and its worker.
+
+Terminal grades use green for A/B, amber for C, and red for D/F; failed,
+incomplete, and unavailable evidence retains its status treatment. Lift that
+rounds to zero is neutral. At narrow widths, condition labels, numeric scores,
+and their complete bars stack together. The full terminal summary includes
+successful quality and lift evidence in Details. Artifact paths use the emitted
+filenames and become clickable only on capable interactive terminals; plain,
+no-color, redirected, and string-rendered output retains readable paths.
| Format | What it is | Reach for it when |
| --- | --- | --- |
diff --git a/src/skillevaluator/cli.py b/src/skillevaluator/cli.py
index 3d705ff5..f78c6d4a 100644
--- a/src/skillevaluator/cli.py
+++ b/src/skillevaluator/cli.py
@@ -2463,7 +2463,12 @@ def validate(
_show_tier3_progress()
reporter = (
- ViewProgressReporter(_on_engine_tail, on_cases=_on_case_rows, pass_threshold=pass_threshold)
+ ViewProgressReporter(
+ _on_engine_tail,
+ on_cases=_on_case_rows,
+ pass_threshold=pass_threshold,
+ progress_reporter=view.progress_reporter,
+ )
if quiet
else None
)
@@ -2973,7 +2978,9 @@ def _workflow_report_options(func):
help="Override the packaged quality-default.yaml with a custom YAML file.",
)
@click.option("--pii-waivers", type=PII_WAIVERS_PATH, default=None, help=_PII_WAIVERS_HELP)
[email protected]("--profile", default=None, help="Validation policy [default: default; external is a compatibility alias].")
+ "--profile", default=None, help="Validation policy [default: default; external is a compatibility alias]."
+)
@click.option("--previous-version", default=None, help="Previous version for version checks.")
@click.option(
"--skillspector-baseline", type=SKILLSPECTOR_BASELINE_PATH, default=None, help=_SKILLSPECTOR_BASELINE_HELP
diff --git a/src/skillevaluator/reporting/cli.py b/src/skillevaluator/reporting/cli.py
index ef3b26c8..c45922af 100644
--- a/src/skillevaluator/reporting/cli.py
+++ b/src/skillevaluator/reporting/cli.py
@@ -22,6 +22,7 @@ Consoles are created with ``emoji=False`` so ``:name:`` codes in that text
from __future__ import annotations
+import math
from io import StringIO
from typing import TYPE_CHECKING
@@ -35,6 +36,7 @@ from skillevaluator.constants import (
)
from skillevaluator.reporting.base import (
ReporterBase,
+ _agent_eval_finite_number,
additional_errors,
get_skip_reason,
is_advisory_agent_eval_skip,
@@ -50,6 +52,7 @@ from skillevaluator.reporting.harbor_viewer import (
)
from skillevaluator.reporting.presentation import FinalRenderContext, build_final_render_context, trial_coverage
from skillevaluator.reporting.run_status import Status, result_status
+from skillevaluator.reporting.terminal_styles import AMBER, FAINT, GREEN, RED, TEXT, grade_style, lift_style
from skillevaluator.utils.rich_markup import escape_markup
if TYPE_CHECKING:
@@ -226,7 +229,7 @@ class CLIReporter(ReporterBase):
qs = result.metadata.get("quality_scores") if result.metadata else None
qs_all = result.metadata.get("quality_scores_all") if result.metadata else None
if qs and qs.get("dimensions"):
- grade_colors = {"A": "green", "B": "green", "C": "yellow", "D": "red", "F": "red"}
+ grade_colors = {grade: grade_style(grade, status=result_status(result).value) for grade in "ABCDF"}
skills_list = qs_all or [qs]
self._print_quality_table(qs, skills_list, grade_colors, console)
@@ -269,7 +272,7 @@ class CLIReporter(ReporterBase):
"""Format a dimension score cell with color coding."""
d = dims.get(dname, {})
s = d.get("score", 0)
- c = "green" if s >= 80 else ("yellow" if s >= 60 else "red")
+ c = GREEN if s >= 80 else (AMBER if s >= 60 else RED)
return f"[{c}]{s:.0f}[/{c}]"
@staticmethod
@@ -295,7 +298,7 @@ class CLIReporter(ReporterBase):
console.print(f"\n [{gc}]Overall: {avg_score:.1f}/100 (Grade: {avg_grade_text})[/{gc}]")
console.print(f" [dim]Skill Type:[/dim] {stype}\n")
- table = Table(title="Quality Scores by Skill", border_style="cyan", show_header=True)
+ table = Table(title="Quality Scores by Skill", border_style=FAINT, show_header=True)
table.add_column("Skill", style="bold")
table.add_column("Grade", justify="center")
table.add_column("Score", justify="right")
@@ -348,7 +351,7 @@ class CLIReporter(ReporterBase):
console.print(f" [dim]{escape_markup(str(summary))}[/dim]")
console.print()
- table = Table(title="Rubric Evaluation", border_style="cyan", show_header=True)
+ table = Table(title="Rubric Evaluation", border_style=FAINT, show_header=True)
table.add_column("Criterion", style="bold")
table.add_column("Score", justify="center")
table.add_column("Pass", justify="center")
@@ -375,10 +378,17 @@ class CLIReporter(ReporterBase):
composite = agent_eval.get("composite_lift")
runtime = agent_eval.get("runtime_seconds", 0.0)
- vc = "green" if verdict == "pass" else ("red" if verdict == "fail" else "yellow")
- composite_text = f"{composite:+.2f}" if isinstance(composite, int | float) else "N/A"
+ vc = GREEN if verdict == "pass" else (RED if verdict == "fail" else AMBER)
+ scored_lift = (
+ isinstance(composite, int | float) and not isinstance(composite, bool) and math.isfinite(composite)
+ )
+ composite_text = f"{composite:+.2f}" if scored_lift else "N/A"
+ composite_style = lift_style(composite) if scored_lift else "dim"
verdict_text = escape_markup(str(verdict).upper())
- console.print(f"\n [{vc}]Verdict: {verdict_text} (composite lift = {composite_text})[/{vc}]")
+ console.print(
+ f"\n [{vc}]Verdict: {verdict_text}[/{vc}] "
+ f"(composite lift = [{composite_style}]{composite_text}[/{composite_style}])"
+ )
if verdict == "inconclusive":
console.print(
" [yellow]Skill never activated. Scores are diagnostic only; no lift verdict was computed.[/yellow]"
@@ -390,12 +400,12 @@ class CLIReporter(ReporterBase):
console.print(" [dim]Harbor artifacts:[/dim]")
if harbor_viewer.get("job_url"):
console.print(
- f" [dim]Harbor logs:[/dim] [cyan]{escape_markup(harbor_viewer['job_url'])}[/cyan]",
+ f" [dim]Harbor logs:[/dim] [dim]{escape_markup(harbor_viewer['job_url'])}[/dim]",
soft_wrap=True,
)
if harbor_viewer.get("analysis_url"):
console.print(
- f" [dim]Harbor analysis:[/dim] [cyan]{escape_markup(harbor_viewer['analysis_url'])}[/cyan]",
+ f" [dim]Harbor analysis:[/dim] [dim]{escape_markup(harbor_viewer['analysis_url'])}[/dim]",
soft_wrap=True,
)
console.print(f" {trial_coverage(agent_eval).text}")
@@ -405,7 +415,7 @@ class CLIReporter(ReporterBase):
if evaluators:
table = Table(
title="Evaluator Scores (Skill Lift)",
- border_style="cyan",
+ border_style=FAINT,
show_header=True,
)
table.add_column("Evaluator", style="bold")
@@ -428,7 +438,7 @@ class CLIReporter(ReporterBase):
else "NO SCORE"
)
else:
- lc = "green" if lift > 0.01 else ("red" if lift < -0.01 else "dim")
+ lc = lift_style(lift)
lift_text = f"[{lc}]{lift:+.2f}[/{lc}]"
table.add_row(
escape_markup(str(name).replace("_", " ").title()),
@@ -468,7 +478,7 @@ class CLIReporter(ReporterBase):
url = safe_url(evidence.get("url"))
if url:
link_text = escape_markup(harbor_evidence_link_text(evidence))
- console.print(f" [dim]{link_text}:[/dim] [cyan]{escape_markup(url)}[/cyan]", soft_wrap=True)
+ console.print(f" [dim]{link_text}:[/dim] [dim]{escape_markup(url)}[/dim]", soft_wrap=True)
if printed:
console.print()
@@ -476,7 +486,7 @@ class CLIReporter(ReporterBase):
if any(v.get("score") is not None for v in insights.values()):
table = Table(
title="LLM-as-Judge Insights",
- border_style="cyan",
+ border_style=FAINT,
show_header=True,
)
table.add_column("Dimension", style="bold")
@@ -624,7 +634,7 @@ class CLIReporter(ReporterBase):
border_style="green",
title_style="bold green",
)
- table.add_column("Validator", style="cyan")
+ table.add_column("Validator", style=TEXT)
table.add_column("Status", justify="center")
table.add_column("Details")
@@ -673,6 +683,26 @@ class CLIReporter(ReporterBase):
parts.append(f"{s.warnings} warnings")
details = ", ".join(parts) if parts else "Failed"
+ if outcome in {"pass", "passed"}:
+ quality = result.metadata.get("quality_scores")
+ if isinstance(quality, dict):
+ score = _agent_eval_finite_number(quality.get("overall_score"))
+ if score is not None:
+ grade = str(quality.get("grade", "?"))
+ details += f"; quality {escape_markup(grade)} {score:.1f}/100"
+ agent = result.metadata.get("agent_eval")
+ if isinstance(agent, dict):
+ raw_summary = agent.get("summary")
+ summary = raw_summary if isinstance(raw_summary, dict) else agent
+ lift = _agent_eval_finite_number(summary.get("overall_lift", agent.get("composite_lift")))
+ if (
+ lift is not None
+ and agent.get("verdict") != "inconclusive"
+ and summary.get("verdict") != "inconclusive"
+ ):
+ details += f"; lift [{lift_style(lift)}]{lift:+.2f}[/]"
+ elif agent.get("verdict") == "inconclusive":
+ details += "; inconclusive (skill never activated)"
table.add_row(escape_markup(str(result.validator_name)), status, details)
console.print(table)
diff --git a/src/skillevaluator/reporting/console_ui.py b/src/skillevaluator/reporting/console_ui.py
index a12ccc43..eed1a27c 100644
--- a/src/skillevaluator/reporting/console_ui.py
+++ b/src/skillevaluator/reporting/console_ui.py
@@ -39,6 +39,7 @@ from pathlib import Path
from typing import TYPE_CHECKING, TextIO
from rich import box
+from rich.cells import cell_len
from rich.console import Console, Group
from rich.live import Live
from rich.panel import Panel
@@ -50,6 +51,19 @@ from skillevaluator.reporting.base import (
source_validation_errors,
source_validation_preview,
)
+from skillevaluator.reporting.terminal_styles import (
+ AMBER,
+ BRIGHT,
+ FAINT,
+ GREEN,
+ INK,
+ MUTED,
+ RED,
+ TEXT,
+ artifact_text,
+ grade_style,
+ lift_style,
+)
from skillevaluator.utils.rich_markup import strip_terminal_controls
if TYPE_CHECKING:
@@ -57,15 +71,6 @@ if TYPE_CHECKING:
from skillevaluator.models.result import ValidationResult
-GREEN = "#76B900" # NVIDIA green: success + identity moments only
-RED = "#F5524C" # failure highlights only
-TEXT = "#D9DEE3" # body
-MUTED = "#8A8F98" # labels / secondary
-FAINT = "#565B66" # chrome, pending, bar remainders
-BRIGHT = "#EDEDED" # the actively-running element
-INK = "#101403" # pill/badge foreground on green fills
-AMBER = "#E5A33B" # the "what to fix" section only
-
_TIER2_SCAN_FAILURE_CHECKS = frozenset(
{
CONTENT_DEDUP_LLM_REFUSED_CHECK,
@@ -129,6 +134,7 @@ class ViewProgressReporter:
on_cases=None,
pass_threshold: float | None = None,
poll_seconds: float = 2.0,
+ progress_reporter=None,
):
self._on_tail = on_tail
self._max_lines = max_lines
@@ -138,6 +144,9 @@ class ViewProgressReporter:
self._pass_threshold = 0.5 if pass_threshold is None else pass_threshold
self._poll_seconds = poll_seconds
self._plan = None
+ self._progress_reporter = progress_reporter
+ self._last_case_rows = None
+ self._last_event = None
self._stop = threading.Event()
self._poller: threading.Thread | None = None
@@ -147,6 +156,8 @@ class ViewProgressReporter:
def start(self, plan) -> None:
"""Track the run plan; once it names the run directory, poll case results."""
+ if self._progress_reporter is not None:
+ self._progress_reporter.start(plan)
if self._on_cases is None or not getattr(plan, "output_dir", None):
return
self._plan = plan
@@ -179,7 +190,8 @@ class ViewProgressReporter:
progress=(done, plan.matrix_trials) if plan.matrix_trials else None,
show_agent=False,
)
- if rows:
+ if rows and rows != self._last_case_rows:
+ self._last_case_rows = rows
self._on_cases(rows)
def _poll_loop(self) -> None:
@@ -191,10 +203,17 @@ class ViewProgressReporter:
def set_secret_values(self, values) -> None:
self._secrets.update(str(v) for v in values if v)
+ if self._progress_reporter is not None:
+ self._progress_reporter.set_secret_values(values)
def emit(self, event) -> None:
from skillevaluator.tier3.harbor.progress import redact_progress_detail
+ if self._progress_reporter is not None:
+ self._progress_reporter.emit(event)
+ if event == self._last_event:
+ return
+ self._last_event = event
state = str(getattr(event.state, "value", event.state))
glyph = {"complete": "✓", "ready": "✓", "failed": "✗", "degraded": "⚠", "skipped": "·"}.get(state)
if glyph is None:
@@ -240,6 +259,7 @@ class TierRow:
glyph_style: str = MUTED
label_style: str = MUTED
fit: str = "" # "wrap": word-wrap the last segment to the panel; "rule": a full-width divider
+ atomic_groups: list[list[tuple[str, str]]] | None = None
def detail_row(label: str, value: str, note: str = "") -> TierRow:
@@ -257,9 +277,9 @@ def lift_row(lift: float, with_score: float, baseline: float) -> TierRow:
filled = max(0, min(BAR, round(score * BAR)))
return [("█" * filled, color), ("░" * (BAR - filled), FAINT)]
- lift_style = f"bold {GREEN}" if lift >= 0 else f"bold {RED}"
+ delta_style = f"bold {lift_style(lift)}"
segments: list[tuple[str, str]] = [
- (f"{lift:+.2f}", lift_style),
+ (f"{lift:+.2f}", delta_style),
(" with-skill ", MUTED),
(f"{with_score:.2f} ", f"bold {TEXT}"),
*bar(with_score, GREEN),
@@ -267,7 +287,7 @@ def lift_row(lift: float, with_score: float, baseline: float) -> TierRow:
(f"{baseline:.2f} ", f"bold {TEXT}"),
*bar(baseline, MUTED),
]
- return TierRow("lift", segments)
+ return TierRow("lift", segments, atomic_groups=[segments[:1], segments[1:5], segments[5:]])
_CASES_PER_LINE = 8
@@ -405,7 +425,7 @@ def fix_section_rows(items: list[FixItem]) -> list[TierRow]:
)
rows += [TierRow("", [("→ ", AMBER), (action, MUTED)], fit="wrap") for action in item.actions]
if len(items) > _FIX_ITEMS_SHOWN:
- rows.append(TierRow("", [(f"+{len(items) - _FIX_ITEMS_SHOWN} more in report.html", FAINT)]))
+ rows.append(TierRow("", [(f"+{len(items) - _FIX_ITEMS_SHOWN} more in detailed reports", FAINT)]))
return rows
@@ -443,6 +463,14 @@ def tier3_fix_rows(findings: list[dict] | None, *, agents: int = 1) -> list[Tier
def _fit_row(row: TierRow, room: int) -> list[list[tuple[str, str]]]:
"""Lay a row out in ``room`` columns so nothing wraps under the label column."""
+ if row.atomic_groups:
+ lines: list[list[tuple[str, str]]] = [[]]
+ for group in row.atomic_groups:
+ if lines[-1] and cell_len("".join(c for c, _ in [*lines[-1], *group])) > room:
+ lines.append([])
+ group = [(group[0][0].lstrip(), group[0][1]), *group[1:]]
+ lines[-1].extend(group)
+ return lines
if row.fit == "rule":
return [[("─" * room, row.segments[0][1])]]
if row.fit != "wrap" or not row.segments:
@@ -458,13 +486,13 @@ def _wrap_segments(segments: list[tuple[str, str]], room: int) -> list[list[tupl
lines: list[list[tuple[str, str]]] = [[]]
used = 0
for chunk, style in segments:
- if used and used + len(chunk) > room:
+ if used and used + cell_len(chunk) > room:
lines.append([])
used = 0
if chunk.strip() in {"·", ""}:
continue
lines[-1].append((chunk, style))
- used += len(chunk)
+ used += cell_len(chunk)
return lines
@@ -522,14 +550,27 @@ class ValidateView:
self.verdict: Verdict | None = None
self.links: list[tuple[str, str]] = []
self._live: Live | None = None
+ self.progress_reporter = None
+ self._progress_stage: dict[int, set[str]] = {}
self._static_printed = 0 # blocks already flushed in non-tty mode
# ── lifecycle ────────────────────────────────────────────────
+ def _live_capable(self) -> bool:
+ """Live redraws need a real, capable TTY, as Harbor progress requires.
+
+ FORCE_COLOR/TTY_COMPATIBLE make Rich treat a CI pipe as a terminal; that
+ stream still needs durable stderr progress, not cursor-control frames.
+ """
+ console = self.console
+ return bool(
+ console.is_terminal and not console.is_dumb_terminal and getattr(console.file, "isatty", lambda: False)()
+ )
+
def start(self) -> None:
if not self.enabled:
return
- if self.console.is_terminal:
+ if self._live_capable():
self._live = Live(
self,
console=self.console,
@@ -538,6 +579,10 @@ class ValidateView:
)
self._live.start()
else:
+ from skillevaluator.tier3.harbor.progress import PlainProgressReporter
+
+ self.progress_reporter = PlainProgressReporter(stream=sys.stderr)
+ self.progress_reporter.start_progress()
self.console.print(self._header())
self.console.print()
@@ -546,7 +591,9 @@ class ValidateView:
return
block = self.blocks[index]
block.status = RUNNING
- block.started_at = time.time()
+ if block.started_at is None:
+ block.started_at = time.time()
+ self._progress_event(index, "running")
self._refresh()
def tier_progress(self, index: int, rows: list[TierRow]) -> None:
@@ -554,6 +601,17 @@ class ValidateView:
if not self.enabled:
return
self.blocks[index].rows = rows
+ meaningful = [row for row in rows if row.label in {"checks", "status", "stages", "progress"}]
+ for row in meaningful:
+ detail = "".join(chunk for chunk, _ in row.segments).strip()
+ if row.label == "checks":
+ detail = next((chunk.removeprefix("▸ ") for chunk, _ in row.segments if chunk.startswith("▸ ")), detail)
+ elif row.label == "progress" and row.segments:
+ # The bar cells only make sense in Live; durable logs keep the count.
+ detail = row.segments[-1][0].strip()
+ stage = f"Tier {self.blocks[index].number}/{row.label}"
+ self._progress_stage.setdefault(index, set()).add(stage)
+ self._progress_event(index, "running", detail, stage=stage)
self._refresh()
def tier_done(self, index: int, *, failed: bool = False, rows: list[TierRow] | None = None) -> None:
@@ -563,6 +621,7 @@ class ValidateView:
block.status = FAIL if failed else PASS
block.duration = time.time() - block.started_at if block.started_at else None
block.rows = rows or []
+ self._complete_progress(index, "failed" if failed else "complete")
self._refresh()
self._flush_static()
@@ -571,6 +630,7 @@ class ValidateView:
return
block = self.blocks[index]
block.status = SKIP
+ self._complete_progress(index, "skipped", reason)
# Multi-line reasons (setup instructions) continue on unlabeled rows.
lines = reason.split("\n") or [""]
block.rows = [TierRow("skipped" if index == 0 else "", [(line, MUTED)]) for index, line in enumerate(lines)]
@@ -580,27 +640,50 @@ class ValidateView:
def finish(self, verdict: Verdict, links: list[tuple[str, str]]) -> None:
if not self.enabled:
return
+ if self.progress_reporter is not None:
+ self.progress_reporter.close()
self.verdict = verdict
- self.links = links
+ # Keep artifacts out of Live's refresh/stop buffers. After its render
+ # hook is removed, normal printing can distinguish TTY from capture.
+ self.links = []
if self._live is not None:
- self._live.update(self)
- self._live.stop()
- self._live = None
+ live = self._live
+ try:
+ live.update(self)
+ finally:
+ try:
+ live.stop()
+ finally:
+ self._live = None
else:
if len(self.blocks) >= 2:
self.console.print(self._gate())
self.console.print()
self.console.print(self._verdict_panel(verdict))
self.console.print()
- for line in self._footer_lines():
- self.console.print(line)
+ self.links = links
+ for line in self._footer_lines():
+ self.console.print(line)
def stop(self) -> None:
"""Tear down the live display (safe on errors / KeyboardInterrupt)."""
+ if self.progress_reporter is not None:
+ self.progress_reporter.close()
if self._live is not None:
self._live.stop()
self._live = None
+ def _progress_event(self, index, state, detail=None, *, stage=None):
+ if self.progress_reporter is not None:
+ from skillevaluator.tier3.harbor.progress import ProgressEvent
+
+ self.progress_reporter.emit(ProgressEvent(stage or f"Tier {self.blocks[index].number}", state, detail))
+
+ def _complete_progress(self, index, state, detail=None):
+ for stage in sorted(self._progress_stage.pop(index, set())):
+ self._progress_event(index, state, stage=stage)
+ self._progress_event(index, state, detail)
+
# ── rendering ────────────────────────────────────────────────
def __rich_console__(self, console, options): # rich renderable protocol
@@ -705,10 +788,16 @@ class ValidateView:
lines = []
room = self.width - 8 - LABEL
for row in block.rows:
+ label = strip_terminal_controls(row.label)
+ if cell_len(label) > LABEL:
+ lines.append(Text(label, style=row.label_style))
+ label = ""
for i, segments in enumerate(_fit_row(row, room)):
t = Text()
t.append(f"{row.glyph if i == 0 else ' '} ", style=row.glyph_style)
- t.append(f"{strip_terminal_controls(row.label if i == 0 else ''):<{LABEL}}", style=row.label_style)
+ t.append(
+ (label if i == 0 else "") + " " * (LABEL - cell_len(label if i == 0 else "")), style=row.label_style
+ )
for chunk, style in segments:
t.append(strip_terminal_controls(chunk), style=style)
lines.append(t)
@@ -728,9 +817,7 @@ class ValidateView:
def _verdict_panel(self, verdict: Verdict) -> Panel:
headline = strip_terminal_controls(verdict.headline)
if verdict.passed:
- body = _pill("✓ PASS", GREEN) + Text.assemble(
- (" ", ""), (headline, f"bold {TEXT}"), (" · exit 0", MUTED)
- )
+ body = Text.assemble(_pill("✓ PASS", GREEN), (" ", ""), (headline, f"bold {TEXT}"), (" · exit 0", MUTED))
return Panel(
Group(body, *(Text(strip_terminal_controls(line), style=MUTED) for line in verdict.details)),
box=box.ROUNDED,
@@ -739,8 +826,11 @@ class ValidateView:
padding=(0, 2),
)
color = AMBER if verdict.incomplete else RED
- line1 = _pill("INCOMPLETE" if verdict.incomplete else "✗ FAIL", color, ink="#1C0605") + Text.assemble(
- (" ", ""), (headline, f"bold {TEXT}"), (" · exit 1", MUTED)
+ line1 = Text.assemble(
+ _pill("INCOMPLETE" if verdict.incomplete else "✗ FAIL", color, ink="#1C0605"),
+ (" ", ""),
+ (headline, f"bold {TEXT}"),
+ (" · exit 1", MUTED),
)
parts: list = [line1, *(Text(strip_terminal_controls(line), style=MUTED) for line in verdict.details)]
if verdict.fix:
@@ -754,11 +844,8 @@ class ValidateView:
def _footer_lines(self):
for label, target in self.links:
- yield Text.assemble(
- (" ", ""),
- (f"{strip_terminal_controls(label):<{LABEL}}", MUTED),
- (strip_terminal_controls(target), MUTED),
- )
+ prefix = Text.assemble((" ", ""), (f"{strip_terminal_controls(label):<{LABEL}}", MUTED))
+ yield artifact_text(target, self.console, prefix=prefix)
def _refresh(self) -> None:
if self._live is not None:
@@ -766,7 +853,7 @@ class ValidateView:
def _flush_static(self) -> None:
"""Non-tty mode: print blocks once, as they complete."""
- if self._live is not None or self.console.is_terminal:
+ if self._live is not None or self._live_capable():
return
for block in self.blocks[self._static_printed :]:
if block.status in (PENDING, RUNNING):
@@ -917,9 +1004,7 @@ def sanitize_tier3_console_text(value: object, *, limit: int = _TIER3_TEXT_MAX_C
sample = value[:sample_limit]
flattened = " ".join(
"".join(
- " "
- if character.isspace() or unicodedata.category(character).startswith("C")
- else character
+ " " if character.isspace() or unicodedata.category(character).startswith("C") else character
for character in sample
).split()
)
@@ -1102,7 +1187,13 @@ def summarize_tier1(results: list[ValidationResult], lineup: list[str] | None =
quality = _quality_stats(results)
if quality:
grade, score = quality
- rows.append(TierRow("quality", [(grade, f"bold {GREEN}"), (f" {score:.1f} / 100", TEXT)]))
+ quality_result = next((r for r in results if _quality_stats([r])), None)
+ from skillevaluator.reporting.run_status import result_status
+
+ status = result_status(quality_result).value if quality_result else "unavailable"
+ rows.append(
+ TierRow("quality", [(grade, f"bold {grade_style(grade, status=status)}"), (f" {score:.1f} / 100", TEXT)])
+ )
items = [_tier1_fix(r) for r in broken]
if incomplete:
items.append(_setup_fix(incomplete))
@@ -1316,12 +1407,7 @@ def summarize_tier3(
rows.append(TierRow("verdict", [("INCONCLUSIVE: skill never activated", TEXT)]))
lift = _tier3_finite_number(summary.get("overall_lift"))
with_score = _tier3_finite_number(summary.get("overall_score"))
- if (
- lift is not None
- and -1.0 <= lift <= 1.0
- and with_score is not None
- and 0.0 <= with_score <= 1.0
- ):
+ if lift is not None and -1.0 <= lift <= 1.0 and with_score is not None and 0.0 <= with_score <= 1.0:
baseline = with_score - lift
if 0.0 <= baseline <= 1.0:
rows.append(lift_row(lift, with_score, baseline))
diff --git a/src/skillevaluator/reporting/terminal_styles.py b/src/skillevaluator/reporting/terminal_styles.py
new file mode 100644
index 00000000..9fe469e1
--- /dev/null
+++ b/src/skillevaluator/reporting/terminal_styles.py
@@ -0,0 +1,121 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Lightweight semantic terminal styles shared by report presentations."""
+
+from __future__ import annotations
+
+GREEN = "#76B900"
+RED = "#F5524C"
+TEXT = "default"
+MUTED = "#8A8F98"
+FAINT = "#565B66"
+BRIGHT = "default"
+INK = "#101403"
+AMBER = "#E5A33B"
+
+
+def grade_style(grade: str, *, status: str = "passed") -> str:
+ """Keep unavailable or incomplete evidence from looking like a passing grade."""
+ if status in {"failed", "fail"}:
+ return RED
+ if status in {"incomplete", "degraded"}:
+ return AMBER
+ if status not in {"passed", "pass", "succeeded", "complete"}:
+ return MUTED
+ return {"A": GREEN, "B": GREEN, "C": AMBER, "D": RED, "F": RED}.get(grade, MUTED)
+
+
+def lift_style(value: float) -> str:
+ """Style the displayed two-decimal lift; rounded zero is neutral."""
+ displayed = float(f"{value:.2f}")
+ return GREEN if displayed > 0 else RED if displayed < 0 else MUTED
+
+
+def _artifact_uri(target: str, *, windows: bool) -> str | None:
+ """Build a terminal-only URI without changing canonical URL normalization."""
+ from pathlib import Path, PurePosixPath, PureWindowsPath
+ from urllib.parse import quote, unquote, urlsplit
+
+ from skillevaluator.utils.rich_markup import strip_terminal_controls
+
+ def has_controls(value: str) -> bool:
+ return any(ord(char) < 32 or 127 <= ord(char) <= 159 for char in value)
+
+ if strip_terminal_controls(target) != target or has_controls(target):
+ return None
+ windows_path = PureWindowsPath(target)
+ if windows_path.drive:
+ # A foreign Windows path must not become a relative POSIX file URI;
+ # drive-relative paths and UNC shares are deliberately left as text.
+ if windows and len(windows_path.drive) == 2 and windows_path.is_absolute():
+ # PurePath.as_uri() is deprecated from Python 3.14; build the same URI.
+ drive = windows_path.drive
+ try:
+ return "file:///" + drive + quote(windows_path.as_posix()[len(drive) :])
+ except UnicodeEncodeError:
+ # An optional hyperlink must not prevent the path from being shown.
+ return None
+ return None
+ try:
+ parts = urlsplit(target)
+ if parts.scheme:
+ decoded = urlsplit(unquote(target))
+ if (
+ parts.scheme not in {"http", "https"}
+ or not parts.hostname
+ or parts.username is not None
+ or parts.password is not None
+ or decoded.username is not None
+ or decoded.password is not None
+ or any(char.isspace() for char in target)
+ or has_controls(unquote(target))
+ ):
+ return None
+ # Access validates malformed/out-of-range ports before linking.
+ _ = parts.port
+ return target
+ except ValueError:
+ return None
+ if windows and PurePosixPath(target).is_absolute():
+ return None
+ return Path(target).expanduser().absolute().as_uri()
+
+
+def artifact_text(target: str, console, *, label: str | None = None, prefix=None):
+ """Defer link styling until render, keeping captured strings inert."""
+ import os
+
+ from rich.measure import Measurement
+ from rich.style import Style
+ from rich.text import Text
+
+ from skillevaluator.utils.rich_markup import strip_terminal_controls
+
+ text = Text()
+ if prefix is not None:
+ text.append_text(prefix)
+ start = len(text)
+ text.append(strip_terminal_controls(label if label is not None else target), style=MUTED)
+ url = _artifact_uri(target, windows=os.name == "nt")
+
+ def capable(output) -> bool:
+ return (
+ output.is_terminal
+ and not output.no_color
+ and not output.is_dumb_terminal
+ and bool(getattr(output.file, "isatty", lambda: False)())
+ )
+
+ class ArtifactLine:
+ def __rich_console__(self, output, options):
+ rendered = text.copy()
+ # Rich's normal print() owns one buffer context. capture() adds an
+ # outer context, even when its underlying stream is a real TTY.
+ # Check at render time so prebuilt/footer renderables remain safe.
+ if url and capable(console) and capable(output) and getattr(output, "_buffer_index", 0) <= 1:
+ rendered.stylize(Style(link=url), start)
+ yield from rendered.__rich_console__(output, options)
+
+ def __rich_measure__(self, output, options):
+ return Measurement.get(output, options, text)
+
+ return ArtifactLine()
diff --git a/src/skillevaluator/tier3/case_summary.py b/src/skillevaluator/tier3/case_summary.py
index 9cc34824..cd5c4eed 100644
--- a/src/skillevaluator/tier3/case_summary.py
+++ b/src/skillevaluator/tier3/case_summary.py
@@ -21,6 +21,7 @@ from rich.box import SIMPLE
from rich.table import Table
from rich.text import Text
+from skillevaluator.reporting.terminal_styles import AMBER
from skillevaluator.tier3.harbor.progress import redact_progress_detail, secret_values_from_environment
from skillevaluator.utils.redaction import redact_sensitive_text
from skillevaluator.utils.secret_redaction import Redactor
@@ -589,8 +590,8 @@ def render_case_summary(
heading.append(" (consider these fixes)", style="dim")
console.print(heading)
for i, fix in enumerate(fixes, 1):
- head = Text(f" {i}. ", style="bold")
- head.append(safe(fix["case"]), style="bold cyan")
+ head = Text(f" {i}. ", style=f"bold {AMBER}")
+ head.append(safe(fix["case"]), style="bold")
n = fix["attempts_lost"]
lost = (
(f" · {n} attempt(s) missed the goal" if fix["kind"] == "goal not met" else f" · lost {n} attempt(s)")
diff --git a/src/skillevaluator/tier3/harbor/progress.py b/src/skillevaluator/tier3/harbor/progress.py
index b3055b80..533b67a0 100644
--- a/src/skillevaluator/tier3/harbor/progress.py
+++ b/src/skillevaluator/tier3/harbor/progress.py
@@ -20,6 +20,7 @@ from dataclasses import dataclass
from time import monotonic
from typing import Literal, Protocol, TextIO, runtime_checkable
+from skillevaluator.reporting.terminal_styles import AMBER, FAINT, GREEN, MUTED, RED, TEXT
from skillevaluator.tier3.harbor.secret_redaction import redact_secrets_in_log_line
from skillevaluator.utils.redaction import credential_uri_secret_values
from skillevaluator.utils.rich_markup import strip_terminal_controls
@@ -177,15 +178,19 @@ class PlainProgressReporter:
self._stop = threading.Event()
self._thread: threading.Thread | None = None
self._plan_rendered = False
+ self._last_progress_at: float | None = None
+ self._last_events: dict[str, tuple[str, str]] = {}
@property
def is_active(self) -> bool:
return self._started_at is not None and not self._stop.is_set()
- def start(self, plan: Tier3RunPlan) -> None:
+ def start_progress(self) -> None:
+ """Start the shared lifecycle before a plan (for the validation tiers)."""
with self._lock:
if self._started_at is None:
self._started_at = monotonic()
+ self._last_progress_at = self._started_at
self._stop.clear()
self._thread = threading.Thread(
target=self._heartbeat_loop,
@@ -193,6 +198,10 @@ class PlainProgressReporter:
daemon=True,
)
self._thread.start()
+
+ def start(self, plan: Tier3RunPlan) -> None:
+ self.start_progress()
+ with self._lock:
self._render_plan(plan)
def set_secret_values(self, values: list[str] | tuple[str, ...] | set[str]) -> None:
@@ -202,17 +211,26 @@ class PlainProgressReporter:
def emit(self, event: ProgressEvent) -> None:
with self._lock:
detail = self._event_detail(event)
+ stage = self._safe_text(event.stage)
+ current = (event.state, detail)
+ if self._last_events.get(stage) == current:
+ return
+ self._last_events[stage] = current
+ self._last_progress_at = monotonic()
if event.state == "running":
- self._active[event.stage] = detail
+ self._active[stage] = detail
elif event.state in _TERMINAL_STATES:
- self._active.pop(event.stage, None)
+ self._active.pop(stage, None)
suffix = f" - {detail}" if detail else ""
- self._write_line(f"[{self._elapsed()}] {event.stage}: {event.state}{suffix}")
+ self._write_line(f"[{self._elapsed()}] {stage}: {event.state}{suffix}")
def heartbeat(self) -> None:
with self._lock:
- if not self._active:
+ if not self._active or self._stop.is_set():
return
+ if self._last_progress_at is not None and monotonic() - self._last_progress_at < self._refresh_interval:
+ return
+ self._last_progress_at = monotonic()
stages = ", ".join(sorted(self._active))
self._write_line(f"[{self._elapsed()}] still running: {stages}")
@@ -237,6 +255,8 @@ class PlainProgressReporter:
self._thread = None
self._started_at = None
self._plan_rendered = False
+ self._last_events.clear()
+ self._last_progress_at = None
def _render_plan(self, plan: Tier3RunPlan) -> None:
first_plan = not self._plan_rendered
@@ -288,7 +308,13 @@ class PlainProgressReporter:
return f"{hours:02d}:{minutes:02d}:{seconds:02d}"
def _heartbeat_loop(self) -> None:
- while not self._stop.wait(self._refresh_interval):
+ while True:
+ delay = self._refresh_interval
+ with self._lock:
+ if self._active and self._last_progress_at is not None:
+ delay = max(MIN_REFRESH_INTERVAL_SECONDS, delay - (monotonic() - self._last_progress_at))
+ if self._stop.wait(delay):
+ return
self.heartbeat()
def _safe_text(self, value: object) -> str:
@@ -298,8 +324,14 @@ class PlainProgressReporter:
return self._safe_text(event.detail) if event.detail else ""
def _write_line(self, line: str) -> None:
- self._stream.write(f"{line}\n")
- self._stream.flush()
+ if self._stop.is_set():
+ return
+ try:
+ self._stream.write(f"{line}\n")
+ self._stream.flush()
+ except (OSError, ValueError):
+ # A closed pipe must stop the heartbeat without gating evaluation.
+ self._stop.set()
class RichProgressReporter(PlainProgressReporter):
@@ -340,6 +372,7 @@ class RichProgressReporter(PlainProgressReporter):
def heartbeat(self) -> None:
with self._lock:
+ self._last_progress_at = monotonic()
if self._active:
self._refresh_live()
@@ -386,19 +419,19 @@ class RichProgressReporter(PlainProgressReporter):
else:
mode_label = "Skill-lift evaluation" if plan.baseline is not False else "Agent evaluation"
title = Text.assemble(
- ("Harbor Run Configuration", "bold cyan"),
+ ("Harbor Run Configuration", f"bold {TEXT}"),
" · ",
(self._safe_text(plan.skill_name), "bold"),
" · ",
- (mode_label, "magenta"),
+ (mode_label, MUTED),
)
- table = Table(title=title, box=box.ROUNDED, expand=True, show_lines=False)
+ table = Table(title=title, box=box.ROUNDED, border_style=FAINT, expand=True, show_lines=False)
table.add_column("Stage", style="bold", no_wrap=True)
table.add_column("State", width=12, no_wrap=True)
table.add_column("Detail", ratio=1)
table.add_column("Elapsed", justify="right", no_wrap=True)
- table.add_row("Environment", Text("configured", style="cyan"), Text(self._safe_text(plan.environment)), "")
+ table.add_row("Environment", Text("configured", style=TEXT), Text(self._safe_text(plan.environment)), "")
model_map = dict(plan.agent_models)
agents = ", ".join(
f"{self._safe_text(agent)}={self._safe_text(model_map[agent])}"
@@ -406,11 +439,11 @@ class RichProgressReporter(PlainProgressReporter):
else self._safe_text(agent)
for agent in plan.agents
)
- table.add_row("Agents / models", Text("configured", style="cyan"), Text(agents or "none"), "")
+ table.add_row("Agents / models", Text("configured", style=TEXT), Text(agents or "none"), "")
if plan.provider:
table.add_row(
"Provider",
- Text("configured", style="cyan"),
+ Text("configured", style=TEXT),
Text(self._safe_text(plan.provider)),
"",
)
@@ -433,13 +466,13 @@ class RichProgressReporter(PlainProgressReporter):
)
known = " · ".join(f"{label} {value}" for label, value in values if value is not None)
if known:
- table.add_row("Run plan", Text("configured", style="cyan"), Text(known), "")
+ table.add_row("Run plan", Text("configured", style=TEXT), Text(known), "")
state_styles = {
- "ready": "green",
- "complete": "green",
- "failed": "bold red",
- "degraded": "bold yellow",
- "delegated": "yellow",
+ "ready": GREEN,
+ "complete": GREEN,
+ "failed": f"bold {RED}",
+ "degraded": f"bold {AMBER}",
+ "delegated": AMBER,
"skipped": "dim",
}
for stage, (event, finished_at) in self._live_events.items():
@@ -449,7 +482,7 @@ class RichProgressReporter(PlainProgressReporter):
else stage.replace("-", " ").title()
)
state = (
- Spinner("dots", text=Text("running", style="cyan"))
+ Spinner("dots", text=Text("running", style=TEXT))
if event.state == "running"
else Text(event.state, style=state_styles.get(event.state, "white"))
)
diff --git a/src/skillevaluator/tier3/harbor/report.py b/src/skillevaluator/tier3/harbor/report.py
index 69daecb5..a6801d39 100644
--- a/src/skillevaluator/tier3/harbor/report.py
+++ b/src/skillevaluator/tier3/harbor/report.py
@@ -363,7 +363,7 @@ def _render_findings_body(findings: list[dict[str, Any]]) -> Any:
body.append(f"{finding['score']:.2f}\n", style=style)
question = METRIC_QUESTIONS.get(finding["metric"])
if question:
- body.append(f" {question}\n", style="dim cyan")
+ body.append(f" {question}\n", style="dim")
reason_style = "dim green" if finding["severity"] == "ok" else "dim"
for reason in finding["reasons"]:
@@ -1086,6 +1086,7 @@ def display_findings_report(
from rich.panel import Panel
from rich.text import Text
+ from skillevaluator.reporting.terminal_styles import AMBER, FAINT
from skillevaluator.utils.rich_markup import escape_markup, strip_terminal_controls
console = Console(emoji=False)
@@ -1102,7 +1103,7 @@ def display_findings_report(
best_agent_label = _agent_model_label(best_agent, harbor_result, agents_data)
console.print(
f" [dim]Findings from best performing agent/model combination for your skill:[/dim] "
- f"[bold cyan]{escape_markup(best_agent_label)}[/bold cyan]"
+ f"[bold]{escape_markup(best_agent_label)}[/bold]"
)
console.print()
else:
@@ -1166,16 +1167,18 @@ def display_findings_report(
if suggestions:
rendered_messages.update(str(suggestion).strip() for suggestion in suggestions if str(suggestion).strip())
suggestions = add_evidence_links_to_suggestions(suggestions, rewards)
- body.append(" \U0001f4a1 SUGGESTIONS\n", style="bold cyan")
+ body.append(" \U0001f4a1 SUGGESTIONS\n", style=f"bold {AMBER}")
for i, suggestion in enumerate(suggestions, 1):
- body.append(f" {i}. {strip_terminal_controls(str(suggestion))}\n", style="white")
+ body.append(f" {i}. ", style=AMBER)
+ body.append(f"{strip_terminal_controls(str(suggestion))}\n")
else:
- body.append(" \U0001f4a1 NEXT STEPS\n", style="bold cyan")
+ body.append(" \U0001f4a1 NEXT STEPS\n", style="bold")
suggestions = _passing_skill_suggestions(findings, rewards)
rendered_messages.update(str(suggestion).strip() for suggestion in suggestions if str(suggestion).strip())
suggestions = add_evidence_links_to_suggestions(suggestions, rewards)
for i, suggestion in enumerate(suggestions, 1):
- body.append(f" {i}. {strip_terminal_controls(str(suggestion))}\n", style="white")
+ body.append(f" {i}. ", style=AMBER)
+ body.append(f"{strip_terminal_controls(str(suggestion))}\n")
rendered_messages.update(str(suggestion).strip() for suggestion in suggestions if str(suggestion).strip())
for agent, (agent_findings, _agent_rewards) in agent_reports.items():
@@ -1202,7 +1205,7 @@ def display_findings_report(
f"[bold]{escape_markup(f'{skill_name} / {panel_agent_label}')} \u2014 Findings[/bold]",
emoji=False,
),
- border_style="cyan",
+ border_style=FAINT,
padding=(1, 1),
)
)
diff --git a/src/skillevaluator/tier3/result_display.py b/src/skillevaluator/tier3/result_display.py
index 715722de..7071dc2d 100644
--- a/src/skillevaluator/tier3/result_display.py
+++ b/src/skillevaluator/tier3/result_display.py
@@ -20,6 +20,7 @@ from rich.panel import Panel
from rich.table import Table
from rich.text import Text
+from skillevaluator.reporting.terminal_styles import AMBER, FAINT, GREEN, RED, TEXT, lift_style
from skillevaluator.tier3.harbor.metrics import (
CUSTOM_ONLY_METRIC_SET,
DEFAULT_METRIC_SET,
@@ -52,10 +53,10 @@ def _finite_number(value: object) -> float | None:
def _score_style(score: float) -> str:
if score >= 0.8:
- return "green"
+ return GREEN
if score >= 0.6:
- return "yellow"
- return "red"
+ return AMBER
+ return RED
def _score_bar(score: float) -> str:
@@ -88,7 +89,7 @@ def _delta_cell(value: object) -> Text:
numeric = _finite_number(value)
if numeric is None:
return Text("NO SCORE", style="dim")
- style = "green" if numeric > 0 else "red" if numeric < 0 else "dim"
+ style = lift_style(numeric)
return Text(f"{numeric:+.2f}", style=f"bold {style}")
@@ -405,6 +406,39 @@ def _resolved_skill_name(result: Mapping[str, Any]) -> str:
return "Skill Evaluation"
+def _responsive_conditions(table: Table, console: Console) -> Table:
+ """Stack complete labeled score/bar groups when paired columns do not fit."""
+ if getattr(console, "width", 98) >= 98:
+ return table
+ compact = Table(show_header=True, header_style="bold dim", box=SIMPLE, padding=(0, 1), show_edge=False, expand=True)
+ compact.add_column(str(table.columns[0].header), ratio=1, overflow="fold")
+ compact.add_column("Conditions", no_wrap=True, min_width=26)
+ columns = [list(column.cells) for column in table.columns]
+ for index, row in enumerate(table.rows):
+ cells = [column[index] for column in columns]
+ conditions = Text()
+ if not any(str(cell).strip() for cell in cells[1:]):
+ # Agent header rows carry no scores; keep them as a single heading line.
+ compact.add_row(cells[0], conditions, end_section=row.end_section)
+ continue
+ for offset, label in ((1, "With Skill"), (3, "No Skill")):
+ if offset >= len(cells):
+ continue
+ if conditions:
+ conditions.append("\n")
+ conditions.append(f"{label:<10} ", style="dim")
+ conditions.append(cells[offset])
+ if offset + 1 < len(cells):
+ conditions.append(" ")
+ conditions.append(cells[offset + 1])
+ if len(cells) > 5:
+ conditions.append("\nLift ", style="dim")
+ conditions.append(cells[5])
+ compact.add_row(cells[0], conditions, end_section=row.end_section)
+ compact.caption = table.caption
+ return compact
+
+
def _render_agent_scores(
*,
console: Console,
@@ -440,7 +474,7 @@ def _render_agent_scores(
show_edge=False,
expand=True,
)
- table.add_column("Evaluator", style="white", min_width=14, ratio=2)
+ table.add_column("Evaluator", style=TEXT, min_width=14, ratio=2)
table.add_column("With Skill", justify="right", no_wrap=True, width=10)
table.add_column("", no_wrap=True, width=10)
if show_baseline:
@@ -484,7 +518,7 @@ def _render_agent_scores(
with_value = custom_with.get(metric) if with_usable else None
baseline_value = custom_without.get(metric) if baseline_usable else None
with_score, with_bar = _score_cell(with_value, unavailable=with_unavailable)
- row = [Text(f"custom: {safe(metric)}", style="cyan"), with_score, with_bar]
+ row = [Text(f"custom: {safe(metric)}", style=TEXT), with_score, with_bar]
if show_baseline:
persisted = custom_lift.get(metric) if isinstance(custom_lift.get(metric), Mapping) else {}
delta = (
@@ -538,7 +572,8 @@ def _render_agent_scores(
model = _resolved_agent_model(result, agent, data)
skill = _resolved_skill_name(result)
- title = Text("Results by Evaluator", style="bold cyan")
+ table = _responsive_conditions(table, console)
+ title = Text("Results by Evaluator", style=f"bold {TEXT}")
subtitle = Text()
subtitle.append(" / ".join(part for part in (safe(skill), safe(agent)) if part), style="dim")
trials = data.get("num_trials_with")
@@ -588,7 +623,7 @@ def _render_agent_scores(
content,
title=title,
subtitle=subtitle if subtitle.plain else None,
- border_style="cyan",
+ border_style=FAINT,
padding=(1, 1),
)
)
@@ -608,7 +643,7 @@ def _render_dimensions(
show_edge=False,
expand=True,
)
- table.add_column("Dimension", style="white", min_width=15, ratio=2, no_wrap=True)
+ table.add_column("Dimension", style=TEXT, min_width=15, ratio=2, no_wrap=True)
table.add_column("With Skill", justify="right", no_wrap=True, width=10)
table.add_column("", no_wrap=True, width=10)
table.add_column("No Skill", justify="right", no_wrap=True, width=9)
@@ -619,7 +654,7 @@ def _render_dimensions(
if agent_index:
table.add_section()
if len(agents) > 1:
- table.add_row(Text(f"Agent: {safe(agent)}", style="bold cyan"), "", "", "", "", "")
+ table.add_row(Text(f"Agent: {safe(agent)}", style=f"bold {TEXT}"), "", "", "", "", "")
data = raw if isinstance(raw, Mapping) else {}
for dimension in DIMENSION_DISPLAY:
with_dimensions = data.get("dimensions_with_skill")
@@ -677,11 +712,12 @@ def _render_dimensions(
"Dimensions are report-only rollups; custom metrics do not change them."
)
table.caption = caption
+ table = _responsive_conditions(table, console)
console.print(
Panel(
table,
title="[bold]Results by Dimension[/bold]",
- border_style="cyan",
+ border_style=FAINT,
padding=(1, 0),
)
)
@@ -769,7 +805,7 @@ def _render_feedback_and_suggestions(
body.append("\n")
body.append("Suggestions\n", style="bold")
for index, (_severity, message) in enumerate(suggestions[:5], start=1):
- body.append(f" {index}. ", style="cyan")
+ body.append(f" {index}. ", style=AMBER)
body.append(f"{safe(message)}\n")
if len(suggestions) > 5:
location = "in the HTML report" if has_html_report else "not shown"
@@ -778,8 +814,8 @@ def _render_feedback_and_suggestions(
console.print(
Panel(
body,
- title=Text("Feedback & Suggestions", style="bold cyan"),
- border_style="cyan",
+ title=Text("Feedback & Suggestions", style=f"bold {TEXT}"),
+ border_style=FAINT,
padding=(1, 1),
)
)
@@ -964,7 +1000,7 @@ def render_evaluation_result(
Panel(
artifacts,
title="[bold]Artifacts[/bold]",
- border_style="cyan",
+ border_style=FAINT,
padding=(0, 1),
)
)
diff --git a/tests/reporting/test_terminal_polish.py b/tests/reporting/test_terminal_polish.py
new file mode 100644
index 00000000..c1db700b
--- /dev/null
+++ b/tests/reporting/test_terminal_polish.py
@@ -0,0 +1,572 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Operator-visible terminal evidence, wrapping and progress contracts."""
+
+import io
+import math
+
+import pytest
+from rich.console import Console
+from rich.text import Text
+
+from skillevaluator.models import ValidationResult
+from skillevaluator.reporting.cli import CLIReporter
+from skillevaluator.reporting.console_ui import ValidateView, Verdict, detail_row, lift_row, summarize_tier1
+
+
+ ("grade", "style"), [("A", "#76B900"), ("B", "#76B900"), ("C", "#E5A33B"), ("D", "#F5524C"), ("F", "#F5524C")]
+)
+def test_compact_quality_grade_has_semantic_color(grade, style):
+ result = ValidationResult(
+ validator_name="Quality", passed=True, metadata={"quality_scores": {"grade": grade, "overall_score": 65}}
+ )
+ rows = summarize_tier1([result])[1]
+ assert style in next(row for row in rows if row.label == "quality").segments[0][1]
+
+
[email protected]("lift", [0.0, 0.004, -0.004])
+def test_displayed_zero_lift_is_neutral(lift):
+ assert "#76B900" not in lift_row(lift, 0.8, 0.8).segments[0][1]
+ assert "#F5524C" not in lift_row(lift, 0.8, 0.8).segments[0][1]
+
+
[email protected]("width", [98, 80, 60])
+def test_each_lift_condition_keeps_label_number_and_complete_bar(width):
+ out = io.StringIO()
+ view = ValidateView(
+ "試験🧪", [(3, "Live Agent Eval", "live")], console=Console(file=out, width=width, color_system=None)
+ )
+ view.tier_done(0, rows=[detail_row("長い条件名🧪非常に長いラベル", "測定結果 0.85"), lift_row(0.25, 0.85, 0.60)])
+ lines = out.getvalue().splitlines()
+ for label, score in [("with-skill", "0.85"), ("baseline", "0.60")]:
+ condition = next(line for line in lines if label in line)
+ assert score in condition
+ assert sum(condition.count(c) for c in "█░") >= 10
+ assert all(Text(line).cell_len <= width for line in lines)
+
+
+def test_full_summary_keeps_successful_quality_and_lift_evidence():
+ quality = ValidationResult(
+ validator_name="Quality", passed=True, metadata={"quality_scores": {"grade": "B", "overall_score": 87.5}}
+ )
+ live = ValidationResult(
+ validator_name="AGENT_EVAL", passed=True, metadata={"agent_eval": {"verdict": "pass", "composite_lift": 0.25}}
+ )
+ output = io.StringIO()
+ CLIReporter(console=Console(file=output, width=120)).print_all([quality, live])
+ text = output.getvalue()
+ assert "B" in text and "87.5/100" in text and "+0.25" in text
+ assert "Quality Scores by Skill" not in text
+
+
+def test_noninteractive_progress_is_durable_deduplicated_and_separate(monkeypatch):
+ progress = io.StringIO()
+ output = io.StringIO()
+ monkeypatch.setattr("sys.stderr", progress)
+ view = ValidateView("demo", [(1, "Static", "static")], console=Console(file=output, width=80))
+ try:
+ view.start()
+ view.tier_start(0)
+ view.tier_progress(0, [detail_row("checks", "schema running")])
+ view.tier_progress(0, [detail_row("checks", "schema running")])
+ assert "Tier 1" in progress.getvalue() and "schema running" in progress.getvalue()
+ assert progress.getvalue().count("schema running") == 1
+ view.tier_done(0)
+ view.finish(Verdict(True, "Passed required checks"), [("report", "/tmp/actual-name.html")])
+ finally:
+ view.stop()
+ assert "complete" in progress.getvalue()
+ assert "Passed required checks" in output.getvalue()
+ assert "still running" not in output.getvalue()
+ assert "\x1b]8" not in output.getvalue()
+
+
+def test_plain_progress_heartbeat_waits_for_idle_and_deduplicates(monkeypatch):
+ from skillevaluator.tier3.harbor import progress
+
+ clock = [100.0]
+ monkeypatch.setattr(progress, "monotonic", lambda: clock[0])
+ output = io.StringIO()
+ reporter = progress.PlainProgressReporter(stream=output)
+ # Manual fake-clock steps must not race the autonomous heartbeat worker.
+ monkeypatch.setattr(reporter, "_heartbeat_loop", reporter._stop.wait)
+ try:
+ reporter.start(progress.Tier3RunPlan("demo", "docker", ()))
+ worker = reporter._thread
+ event = progress.ProgressEvent("agent", "running", "one measured trial")
+ reporter.emit(event)
+ clock[0] = 130
+ reporter.emit(event)
+ clock[0] = math.nextafter(160.0, 100.0)
+ reporter.heartbeat()
+ assert "still running" not in output.getvalue()
+ clock[0] = 160
+ reporter.heartbeat()
+ assert output.getvalue().count("still running") == 1
+ clock[0] = 180
+ reporter.emit(progress.ProgressEvent("agent", "running", "two measured trials"))
+ clock[0] = 220
+ reporter.heartbeat()
+ assert output.getvalue().count("still running") == 1
+ clock[0] = math.nextafter(240.0, 180.0)
+ reporter.heartbeat()
+ assert output.getvalue().count("still running") == 1
+ clock[0] = 240.0
+ reporter.heartbeat()
+ assert output.getvalue().count("still running") == 2
+ assert output.getvalue().count("one measured trial") == 1
+ finally:
+ reporter.close()
+ assert not reporter.is_active
+ assert not worker.is_alive()
+ assert reporter._thread is None
+
+
+def test_plain_progress_closed_stream_stops_worker():
+ from skillevaluator.tier3.harbor.progress import PlainProgressReporter, ProgressEvent, Tier3RunPlan
+
+ output = io.StringIO()
+ reporter = PlainProgressReporter(stream=output)
+ reporter.start(Tier3RunPlan("demo", "docker", ()))
+ output.close()
+ reporter.emit(ProgressEvent("agent", "running"))
+ reporter.close()
+ assert reporter._thread is None
+ assert not reporter.is_active
+
+
[email protected]("width", [98, 80, 60])
+def test_direct_tier3_preserves_conditions_and_bars_at_narrow_width(width):
+ from skillevaluator.tier3.result_display import render_evaluation_result
+
+ output = io.StringIO()
+ result = {
+ "execution_status": "succeeded",
+ "agents": {
+ "試験🧪" * 10: {
+ "execution_status": "succeeded",
+ "with_skill": {"accuracy": 0.85},
+ "without_skill": {"accuracy": 0.60},
+ "lift": {
+ "accuracy": {"delta": 0.25},
+ "overall": {"with_skill": 0.85, "without_skill": 0.60, "delta": 0.25},
+ },
+ }
+ },
+ }
+ render_evaluation_result(result, console=Console(file=output, width=width, color_system=None))
+ lines = output.getvalue().splitlines()
+ for number in ("0.85", "0.60"):
+ condition = next(line for line in lines if number in line)
+ assert sum(condition.count(c) for c in "█░") >= 10
+ assert "+0.25" in output.getvalue()
+ assert all(Text(line).cell_len <= width for line in lines)
+
+
+ ("is_tty", "no_color", "legacy_windows", "expect_link"),
+ [(True, False, False, True), (True, True, False, False), (False, False, False, False), (True, False, True, False)],
+ ids=["modern-tty", "no-color", "redirected", "legacy-windows-tty"],
+)
+def test_interactive_footer_links_require_capability_and_preserve_real_path(
+ monkeypatch, tmp_path, is_tty, no_color, legacy_windows, expect_link
+):
+ monkeypatch.setenv("TERM", "xterm-256color")
+
+ class TTY(io.StringIO):
+ def isatty(self):
+ return True
+
+ report_path = tmp_path / "actual [name].html"
+ expected_uri = report_path.as_uri()
+ assert expected_uri.endswith("/actual%20%5Bname%5D.html")
+ stream = TTY() if is_tty else io.StringIO()
+ console = Console(
+ file=stream,
+ force_terminal=True,
+ no_color=no_color,
+ color_system="truecolor",
+ legacy_windows=legacy_windows,
+ width=max(120, len(str(report_path)) + 30),
+ )
+ view = ValidateView("demo", [], console=console)
+ view.links = [("report", str(report_path))]
+ console.print(next(view._footer_lines()))
+ text = stream.getvalue()
+ assert str(report_path) in text
+ assert ("\x1b]8" in text) is expect_link
+ assert (expected_uri in text) is expect_link
+
+
+def test_noninteractive_check_ticker_reports_only_active_check(monkeypatch):
+ from skillevaluator.reporting.console_ui import check_ticker_row
+
+ output = io.StringIO()
+ monkeypatch.setattr("sys.stderr", output)
+ view = ValidateView("demo", [(1, "Static", "static")], console=Console(file=io.StringIO()))
+ try:
+ view.start()
+ view.tier_start(0)
+ view.tier_progress(0, [check_ticker_row(["schema", "quality", "license"], [], "schema")])
+ assert "schema" in output.getvalue()
+ assert "license" not in output.getvalue()
+ finally:
+ view.stop()
+
+
+ "target", ["https://[invalid", "javascript:alert(1)", "/tmp/name\x1b]8;;https://evil\x1b\\.html"]
+)
+def test_unsafe_footer_target_is_readable_without_hyperlink(monkeypatch, target):
+ from skillevaluator.reporting.terminal_styles import artifact_text
+
+ monkeypatch.setenv("TERM", "xterm-256color")
+
+ class TTY(io.StringIO):
+ def isatty(self):
+ return True
+
+ stream = TTY()
+ console = Console(file=stream, force_terminal=True, no_color=False)
+ console.print(artifact_text(target, console))
+ assert "\x1b]8" not in stream.getvalue()
+
+
+ ("passed", "extra", "expected"),
+ [
+ (False, {}, "#F5524C"),
+ (False, {"incomplete_scans": ["gitleaks"]}, "#E5A33B"),
+ (True, {"skipped": True}, "#8A8F98"),
+ ],
+)
+def test_quality_grade_cannot_override_failed_incomplete_or_skipped_status(passed, extra, expected):
+ result = ValidationResult(
+ validator_name="Quality",
+ passed=passed,
+ metadata={
+ **extra,
+ "quality_scores": {"grade": "A", "overall_score": 95},
+ },
+ )
+ row = next(row for row in summarize_tier1([result])[1] if row.label == "quality")
+ assert expected in row.segments[0][1]
+
+
+def test_full_agent_verdict_keeps_displayed_zero_lift_neutral():
+ console = Console(file=io.StringIO(), width=100)
+ rendered = []
+ original_print = console.print
+
+ def capture(value="", *args, **kwargs):
+ if isinstance(value, str) and "composite lift" in value:
+ rendered.append(Text.from_markup(value))
+ original_print(value, *args, **kwargs)
+
+ console.print = capture
+ CLIReporter._print_agent_eval_tables({"verdict": "pass", "composite_lift": 0.004}, console)
+ text = rendered[0]
+ style = text.get_style_at_offset(console, text.plain.index("+0.00"))
+ assert style.color.name == "#8a8f98"
+
+
+def test_verdict_badge_background_does_not_fill_headline():
+ from rich.console import Group
+
+ view = ValidateView("demo", [])
+ for verdict in (Verdict(True, "Passed required checks"), Verdict(False, "Changes required")):
+ panel = view._verdict_panel(verdict)
+ assert isinstance(panel.renderable, Group)
+ body = panel.renderable.renderables[0]
+ style = body.get_style_at_offset(view.console, body.plain.index(verdict.headline))
+ assert style.bgcolor is None
+
+
+def test_real_tty_capture_excludes_links_but_ordinary_print_keeps_them(monkeypatch):
+ import os
+
+ from skillevaluator.reporting.terminal_styles import artifact_text
+
+ if not hasattr(os, "openpty"):
+ pytest.skip("PTY capture regression runs on POSIX; Windows URI policy is tested separately")
+ monkeypatch.setenv("TERM", "xterm-256color")
+ master, slave = os.openpty()
+ try:
+ with os.fdopen(slave, "w") as stream:
+ console = Console(file=stream, color_system="truecolor", no_color=False, width=120)
+ line = artifact_text("/tmp/actual [name].html", console)
+ with console.capture() as capture:
+ console.print(line)
+ assert "actual [name].html" in capture.get()
+ assert "\x1b]8" not in capture.get()
+ console.print(line)
+ assert "\x1b]8" in os.read(master, 65536).decode()
+ finally:
+ os.close(master)
+
+
[email protected]("legacy_windows", [False, True], ids=["modern-tty", "legacy-windows-tty"])
+def test_footer_links_follow_capture_at_render_time(monkeypatch, tmp_path, legacy_windows):
+ monkeypatch.setenv("TERM", "xterm-256color")
+
+ class TTY(io.StringIO):
+ def isatty(self):
+ return True
+
+ report_path = tmp_path / "actual [name].html"
+ expected_uri = report_path.as_uri()
+ assert expected_uri.endswith("/actual%20%5Bname%5D.html")
+ stream = TTY()
+ console = Console(
+ file=stream,
+ force_terminal=True,
+ color_system="truecolor",
+ no_color=False,
+ legacy_windows=legacy_windows,
+ width=max(120, len(str(report_path)) + 30),
+ )
+ view = ValidateView("demo", [], console=console)
+ view.links = [("report", str(report_path))]
+ view.verdict = Verdict(True, "Passed required checks")
+ with console.capture() as capture:
+ console.print(view)
+ assert str(report_path) in capture.get()
+ assert "\x1b]8" not in capture.get()
+ assert expected_uri not in capture.get()
+ console.print(view)
+ assert str(report_path) in stream.getvalue()
+ assert ("\x1b]8" in stream.getvalue()) is (not legacy_windows)
+ assert (expected_uri in stream.getvalue()) is (not legacy_windows)
+
+
+ "target",
+ [
+ "https://user:[email protected]/report.html",
+ "https://[email protected]/report.html",
+ "https://example.com/line\nbreak.html",
+ "https://example.com/tab\tname.html",
+ "https://example.com/%0areport.html",
+ ],
+)
+def test_remote_credential_and_control_targets_are_not_linked(monkeypatch, target):
+ from skillevaluator.reporting.terminal_styles import artifact_text
+
+ monkeypatch.setenv("TERM", "xterm-256color")
+
+ class TTY(io.StringIO):
+ def isatty(self):
+ return True
+
+ output = TTY()
+ console = Console(file=output, color_system="truecolor", no_color=False, width=120)
+ console.print(artifact_text(target, console, label="Readable report path"))
+ assert "Readable report path" in output.getvalue()
+ assert "\x1b]8" not in output.getvalue()
+
+
+ ("target", "windows", "expected"),
+ [
+ (r"C:\reports\actual [name].html", True, "file:///C:/reports/actual%20%5Bname%5D.html"),
+ ("C:/reports/actual.html", True, "file:///C:/reports/actual.html"),
+ (r"C:\reports\actual.html", False, None),
+ (r"C:report.html", True, None),
+ ("/tmp/report.html", True, None),
+ ("javascript:alert(1)", True, None),
+ ("https://example.com/report.html", True, "https://example.com/report.html"),
+ ],
+)
+def test_artifact_uri_uses_only_native_absolute_windows_paths(target, windows, expected):
+ from skillevaluator.reporting.terminal_styles import _artifact_uri
+
+ assert _artifact_uri(target, windows=windows) == expected
+
+
[email protected]("captured", [False, True])
+def test_real_tty_live_lifecycle_prints_footer_once_with_capture_safe_links(monkeypatch, captured):
+ import os
+ import threading
+
+ from skillevaluator.utils.rich_markup import strip_terminal_controls
+
+ if not hasattr(os, "openpty"):
+ pytest.skip("Live PTY regression runs on POSIX")
+ monkeypatch.setenv("TERM", "xterm-256color")
+ master, slave = os.openpty()
+ chunks = []
+
+ def drain():
+ while True:
+ try:
+ chunk = os.read(master, 65536)
+ except OSError:
+ return
+ if not chunk:
+ return
+ chunks.append(chunk)
+
+ reader = threading.Thread(target=drain, daemon=True)
+ try:
+ with os.fdopen(slave, "w") as stream:
+ # Drain concurrently: a real Live frame can fill a PTY's small buffer.
+ reader.start()
+ console = Console(file=stream, color_system="truecolor", no_color=False, width=98)
+ view = ValidateView("demo", [], console=console)
+
+ def lifecycle():
+ view.start()
+ worker = view._live._refresh_thread
+ try:
+ view.finish(Verdict(True, "Passed required checks"), [("report", "/tmp/actual.html")])
+ finally:
+ view.stop()
+ assert view._live is None
+ worker.join(timeout=1)
+ assert not worker.is_alive()
+
+ if captured:
+ with console.capture() as capture:
+ lifecycle()
+ output = capture.get()
+ else:
+ lifecycle()
+ reader.join(timeout=1)
+ assert not reader.is_alive()
+ if not captured:
+ output = b"".join(chunks).decode()
+ assert strip_terminal_controls(output).count("/tmp/actual.html") == 1
+ assert ("\x1b]8" in output) is (not captured)
+ finally:
+ os.close(master)
+
+
+ "value", [10**400, -(10**400), True, False, float("nan"), float("inf"), float("-inf"), None, "95"]
+)
+def test_success_summary_omits_unusable_quality_evidence_without_changing_payload(value):
+ import json
+
+ result = ValidationResult(
+ validator_name="CHECK",
+ passed=True,
+ metadata={
+ "quality_scores": {"grade": "A", "overall_score": value},
+ },
+ )
+ before = json.dumps(result.metadata, sort_keys=True)
+ output = io.StringIO()
+ CLIReporter(console=Console(file=output, width=120, color_system=None)).print_summary([result])
+ text = output.getvalue()
+ assert "PASS" in text
+ assert "quality" not in text.casefold()
+ assert "/100" not in text
+ assert json.dumps(result.metadata, sort_keys=True) == before
+
+
+ "value", [10**400, -(10**400), True, False, float("nan"), float("inf"), float("-inf"), None, "0.25"]
+)
+def test_success_summary_omits_unusable_lift_evidence_without_changing_payload(value):
+ import json
+
+ result = ValidationResult(
+ validator_name="CHECK",
+ passed=True,
+ metadata={
+ "agent_eval": {"verdict": "pass", "summary": {"overall_lift": value}},
+ },
+ )
+ before = json.dumps(result.metadata, sort_keys=True)
+ output = io.StringIO()
+ CLIReporter(console=Console(file=output, width=120, color_system=None)).print_summary([result])
+ assert "PASS" in output.getvalue()
+ assert "lift" not in output.getvalue().casefold()
+ assert json.dumps(result.metadata, sort_keys=True) == before
+
+
+def test_success_summary_keeps_valid_numeric_zero_quality_and_lift():
+ result = ValidationResult(
+ validator_name="CHECK",
+ passed=True,
+ metadata={
+ "quality_scores": {"grade": "F", "overall_score": 0},
+ "agent_eval": {"verdict": "pass", "summary": {"overall_lift": 0}},
+ },
+ )
+ output = io.StringIO()
+ CLIReporter(console=Console(file=output, width=120, color_system=None)).print_summary([result])
+ assert "quality F 0.0/100" in output.getvalue()
+ assert "lift +0.00" in output.getvalue()
+
+
[email protected]("env", ["FORCE_COLOR", "TTY_COMPATIBLE"])
+def test_forced_terminal_pipe_keeps_durable_progress_instead_of_live(monkeypatch, env):
+ # CI often forces colour, so Rich treats the pipe as a terminal; it is not a TTY.
+ monkeypatch.setenv(env, "1")
+ progress = io.StringIO()
+ output = io.StringIO()
+ monkeypatch.setattr("sys.stderr", progress)
+ view = ValidateView("demo", [(1, "Static", "static")], console=Console(file=output, width=80))
+ assert view.console.is_terminal
+ try:
+ view.start()
+ assert view._live is None
+ view.tier_start(0)
+ view.tier_progress(0, [detail_row("checks", "schema running")])
+ view.tier_done(0)
+ finally:
+ view.stop()
+ assert "Tier 1" in progress.getvalue() and "schema running" in progress.getvalue()
+ assert "\x1b[?25l" not in output.getvalue()
+
+
+def test_durable_tier3_progress_logs_the_count_without_bar_cells(monkeypatch):
+ from skillevaluator.reporting.console_ui import tier3_case_rows
+
+ progress = io.StringIO()
+ monkeypatch.setattr("sys.stderr", progress)
+ view = ValidateView("demo", [(3, "Live Agent Eval", "live")], console=Console(file=io.StringIO()))
+ marks = {"codex": {"with_skill": {"001": ["pass"]}, "without_skill": {"001": []}}}
+ try:
+ view.start()
+ view.tier_start(0)
+ view.tier_progress(0, tier3_case_rows(marks, agents=["codex"], k=4, progress=(3, 8), show_agent=False))
+ finally:
+ view.stop()
+ assert "Tier 3/progress: running - 3/8 attempts" in progress.getvalue()
+ assert "█" not in progress.getvalue() and "░" not in progress.getvalue()
+
+
+def test_windows_artifact_uri_avoids_deprecated_pure_path_as_uri():
+ import warnings
+
+ from skillevaluator.reporting.terminal_styles import _artifact_uri
+
+ with warnings.catch_warnings():
+ warnings.simplefilter("error", DeprecationWarning)
+ assert _artifact_uri(r"D:\a\b #%?;é.html", windows=True) == "file:///D:/a/b%20%23%25%3F%3B%C3%A9.html"
+
+
[email protected]("surrogate", ["\udcff", "\ud800"])
+def test_windows_artifact_paths_with_invalid_unicode_remain_unlinked(surrogate):
+ from skillevaluator.reporting.terminal_styles import _artifact_uri
+
+ assert _artifact_uri("C:\\reports\\surrogate" + surrogate + ".html", windows=True) is None
+
+
+def test_compact_dimension_table_keeps_agent_headers_free_of_condition_labels():
+ from skillevaluator.tier3 import result_display
+
+ dimension = next(iter(result_display.DIMENSION_DISPLAY))
+ agent = {
+ "execution_status": "succeeded",
+ "dimensions_with_skill": {dimension: {"score": 0.9}},
+ "dimensions_without_skill": {dimension: {"score": 0.7}},
+ }
+ output = io.StringIO()
+ result_display._render_dimensions(
+ console=Console(file=output, width=80, color_system=None), agents={"codex": agent, "claude": agent}, safe=str
+ )
+ header = next(line for line in output.getvalue().splitlines() if "Agent: codex" in line)
+ assert "With Skill" not in header
diff --git a/tests/reporting/test_tier3_validate_cases.py b/tests/reporting/test_tier3_validate_cases.py
index 15f89154..ff1941f4 100644
--- a/tests/reporting/test_tier3_validate_cases.py
+++ b/tests/reporting/test_tier3_validate_cases.py
@@ -68,7 +68,7 @@ def test_fix_rows():
assert rows[1].label == "what to fix"
assert [r.label for r in rows if r.glyph in "123" and r.glyph != " "] == ["001, 002"] * 3
assert "Goal not met: asked for a file" in text and "claude-code" not in text
- assert "→ add the file" in text and "+1 more in report.html" in text
+ assert "→ add the file" in text and "+1 more in detailed reports" in text
assert "(claude-code)" in _text(tier3_fix_rows([finding], agents=2))
long = {
**finding,
diff --git a/tests/test_tier3_progress.py b/tests/test_tier3_progress.py
index bb8b4873..435bb794 100644
--- a/tests/test_tier3_progress.py
+++ b/tests/test_tier3_progress.py
@@ -9,6 +9,7 @@ import errno
import importlib
import io
import json
+import math
import os
import tempfile
import threading
@@ -355,19 +356,34 @@ def test_rich_reporter_stops_live_display_during_exception_cleanup(monkeypatch:
assert live.stop_calls == 1
-def test_plain_reporter_emits_durable_plan_events_and_elapsed_heartbeat() -> None:
+def test_plain_reporter_emits_durable_plan_events_and_elapsed_heartbeat(monkeypatch: pytest.MonkeyPatch) -> None:
progress = _progress_module()
+ clock = [0.0]
+ monkeypatch.setattr(progress, "monotonic", lambda: clock[0])
output = io.StringIO()
reporter = progress.PlainProgressReporter(stream=output, refresh_interval=60)
+ # Drive heartbeat() ourselves; the real worker only waits for cleanup.
+ monkeypatch.setattr(reporter, "_heartbeat_loop", reporter._stop.wait)
- reporter.start(_plan(progress))
- reporter.emit(progress.ProgressEvent(stage="configuration", state="running"))
- reporter.emit(progress.ProgressEvent(stage="configuration", state="ready", detail="eval config loaded"))
- reporter.emit(progress.ProgressEvent(stage="docker-images", state="delegated", detail="build delegated to Harbor"))
- reporter.emit(progress.ProgressEvent(stage="agent:codex", state="running", detail="with-skill + baseline"))
- reporter.heartbeat()
- reporter.emit(progress.ProgressEvent(stage="agent:codex", state="complete"))
- reporter.close()
+ try:
+ reporter.start(_plan(progress))
+ worker = reporter._thread
+ reporter.emit(progress.ProgressEvent(stage="configuration", state="running"))
+ reporter.emit(progress.ProgressEvent(stage="configuration", state="ready", detail="eval config loaded"))
+ reporter.emit(
+ progress.ProgressEvent(stage="docker-images", state="delegated", detail="build delegated to Harbor")
+ )
+ reporter.emit(progress.ProgressEvent(stage="agent:codex", state="running", detail="with-skill + baseline"))
+ clock[0] = math.nextafter(60.0, 0.0)
+ reporter.heartbeat()
+ assert "still running" not in output.getvalue()
+ clock[0] = 60.0
+ reporter.heartbeat()
+ reporter.emit(progress.ProgressEvent(stage="agent:codex", state="complete"))
+ finally:
+ reporter.close()
+ assert not worker.is_alive()
+ assert not reporter.is_active
rendered = output.getvalue()
assert "Tier 3 live evaluation: demo-skill" in rendered
@@ -382,7 +398,7 @@ def test_plain_reporter_emits_durable_plan_events_and_elapsed_heartbeat() -> Non
assert "timeout=1.5x" in rendered
assert "configuration: ready - eval config loaded" in rendered
assert "docker-images: delegated - build delegated to Harbor" in rendered
- assert "still running" in rendered
+ assert "[00:01:00] still running: agent:codex" in rendered
assert "agent:codex" in rendered
assert "\x1b[" not in rendered
assert "\r" not in rendered
diff --git a/tests/test_tier3_result_display.py b/tests/test_tier3_result_display.py
index 6c5c6bee..b6ae4aa4 100644
--- a/tests/test_tier3_result_display.py
+++ b/tests/test_tier3_result_display.py
@@ -115,7 +115,7 @@ def test_overall_lift_headline_is_per_agent_and_highlighted_without_a_background
assert headline.plain == f"OVERALL SKILL LIFT {delta:+.2f} · {name}"
style = headline.get_style_at_offset(console, headline.plain.index(f"{delta:+.2f}"))
assert style.bold
- assert style.color.name == ("green" if delta > 0 else "red")
+ assert style.color.name == ("#76b900" if delta > 0 else "#f5524c")
assert style.bgcolor is None